Merge remote-tracking branch 'upstream/master' into feature/classifier-threshold-tuning

.github/ISSUE_TEMPLATE/01_bugs.md

@@ -10,7 +10,7 @@ about: Use this template if you came across a bug or unexpected behaviour differ
 <!-- Include a code example or the steps that led to the problem. Please try to be as specific as possible. -->
 
 ## Your Environment
-<!-- Include details of your environment. If you're using spaCy 1.7+, you can also type `python -m spacy info --markdown` and copy-paste the result here.-->
+<!-- Include details of your environment. You can also type `python -m spacy info --markdown` and copy-paste the result here.-->
 * Operating System:
 * Python Version Used:
 * spaCy Version Used:

.github/azure-steps.yml

@@ -10,6 +10,7 @@ steps:
     inputs:
       versionSpec: ${{ parameters.python_version }}
       architecture: ${{ parameters.architecture }}
+      allowUnstable: true
 
   - bash: |
       echo "##vso[task.setvariable variable=python_version]${{ parameters.python_version }}"
@@ -27,6 +28,7 @@ steps:
 
   - script: python -m mypy spacy
     displayName: 'Run mypy'
+    condition: ne(variables['python_version'], '3.6')
 
   - task: DeleteFiles@1
     inputs:
@@ -54,12 +56,12 @@ steps:
     condition: eq(${{ parameters.gpu }}, true)
 
   - script: |
-      ${{ parameters.prefix }} python -m pytest --pyargs spacy
+      ${{ parameters.prefix }} python -m pytest --pyargs spacy -W error
     displayName: "Run CPU tests"
     condition: eq(${{ parameters.gpu }}, false)
 
   - script: |
-      ${{ parameters.prefix }} python -m pytest --pyargs spacy -p spacy.tests.enable_gpu
+      ${{ parameters.prefix }} python -m pytest --pyargs spacy -W error -p spacy.tests.enable_gpu
     displayName: "Run GPU tests"
     condition: eq(${{ parameters.gpu }}, true)
 

.github/no-response.yml

@@ -1,13 +0,0 @@
-# Configuration for probot-no-response - https://github.com/probot/no-response
-
-# Number of days of inactivity before an Issue is closed for lack of response
-daysUntilClose: 14
-# Label requiring a response
-responseRequiredLabel: more-info-needed
-# Comment to post when closing an Issue for lack of response. Set to `false` to disable
-closeComment: >
-  This issue has been automatically closed because there has been no response
-  to a request for more information from the original author. With only the
-  information that is currently in the issue, there's not enough information
-  to take action. If you're the original author, feel free to reopen the issue
-  if you have or find the answers needed to investigate further.

.github/workflows/issue-manager.yml

@@ -15,7 +15,7 @@ jobs:
   issue-manager:
     runs-on: ubuntu-latest
     steps:
-      - uses: tiangolo/issue-manager@0.2.1
+      - uses: tiangolo/issue-manager@0.4.0
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
           config: >
@@ -25,5 +25,11 @@ jobs:
                 "message": "This issue has been automatically closed because it was answered and there was no follow-up discussion.",
                 "remove_label_on_comment": true,
                 "remove_label_on_close": true
+              },
+              "more-info-needed": {
+                "delay": "P7D",
+                "message": "This issue has been automatically closed because there has been no response to a request for more information from the original author. With only the information that is currently in the issue, there's not enough information to take action. If you're the original author, feel free to reopen the issue if you have or find the answers needed to investigate further.",
+                "remove_label_on_comment": true,
+                "remove_label_on_close": true
               }
             }

.gitignore

@@ -24,6 +24,7 @@ quickstart-training-generator.js
 cythonize.json
 spacy/*.html
 *.cpp
+*.c
 *.so
 
 # Vim / VSCode / editors

.pre-commit-config.yaml

@@ -6,7 +6,7 @@ repos:
       language_version: python3.7
       additional_dependencies: ['click==8.0.4']
 -   repo: https://gitlab.com/pycqa/flake8
-    rev: 3.9.2
+    rev: 5.0.4
     hooks:
     - id: flake8
       args:

azure-pipelines.yml

@@ -31,8 +31,8 @@ jobs:
         inputs:
           versionSpec: "3.7"
       - script: |
-          pip install flake8==3.9.2
+          pip install flake8==5.0.4
-          python -m flake8 spacy --count --select=E901,E999,F821,F822,F823 --show-source --statistics
+          python -m flake8 spacy --count --select=E901,E999,F821,F822,F823,W605 --show-source --statistics
         displayName: "flake8"
 
   - job: "Test"
@@ -85,6 +85,15 @@ jobs:
         Python310Mac:
           imageName: "macos-latest"
           python.version: "3.10"
+        Python311Linux:
+          imageName: 'ubuntu-latest'
+          python.version: '3.11.0-rc.2'
+        Python311Windows:
+          imageName: 'windows-latest'
+          python.version: '3.11.0-rc.2'
+        Python311Mac:
+          imageName: 'macos-latest'
+          python.version: '3.11.0-rc.2'
       maxParallel: 4
     pool:
       vmImage: $(imageName)

extra/DEVELOPER_DOCS/Code Conventions.md

@@ -191,6 +191,8 @@ def load_model(name: str) -> "Language":
     ...
 ```
 
+Note that we typically put the `from typing` import statements on the first line(s) of the Python module.
+
 ## Structuring logic
 
 ### Positional and keyword arguments
@@ -275,6 +277,27 @@ If you have to use `try`/`except`, make sure to only include what's **absolutely
 +     return [v.strip() for v in value.split(",")]
 ```
 
+### Numeric comparisons
+
+For numeric comparisons, as a general rule we always use `<` and `>=` and avoid the usage of `<=` and `>`. This is to ensure we consistently
+apply inclusive lower bounds and exclusive upper bounds, helping to prevent off-by-one errors.
+
+One exception to this rule is the ternary case. With a chain like
+
+```python
+if value >= 0 and value < max:
+    ...
+```
+
+it's fine to rewrite this to the shorter form
+
+```python
+if 0 <= value < max:
+    ...
+```
+
+even though this requires the usage of the `<=` operator.
+
 ### Iteration and comprehensions
 
 We generally avoid using built-in functions like `filter` or `map` in favor of list or generator comprehensions.

extra/DEVELOPER_DOCS/Satellite Packages.md

@@ -0,0 +1,82 @@
+# spaCy Satellite Packages
+
+This is a list of all the active repos relevant to spaCy besides the main one, with short descriptions, history, and current status. Archived repos will not be covered.
+
+## Always Included in spaCy
+
+These packages are always pulled in when you install spaCy. Most of them are direct dependencies, but some are transitive dependencies through other packages.
+
+- [spacy-legacy](https://github.com/explosion/spacy-legacy): When an architecture in spaCy changes enough to get a new version, the old version is frozen and moved to spacy-legacy. This allows us to keep the core library slim while also preserving backwards compatability.
+- [thinc](https://github.com/explosion/thinc): Thinc is the machine learning library that powers trainable components in spaCy. It wraps backends like Numpy, PyTorch, and Tensorflow to provide a functional interface for specifying architectures.
+- [catalogue](https://github.com/explosion/catalogue): Small library for adding function registries, like those used for model architectures in spaCy.
+- [confection](https://github.com/explosion/confection): This library contains the functionality for config parsing that was formerly contained directly in Thinc.
+- [spacy-loggers](https://github.com/explosion/spacy-loggers): Contains loggers beyond the default logger available in spaCy's core code base. This includes loggers integrated with third-party services, which may differ in release cadence from spaCy itself.
+- [wasabi](https://github.com/explosion/wasabi): A command line formatting library, used for terminal output in spaCy.
+- [srsly](https://github.com/explosion/srsly): A wrapper that vendors several serialization libraries for spaCy. Includes parsers for JSON, JSONL, MessagePack, (extended) Pickle, and YAML.
+- [preshed](https://github.com/explosion/preshed): A Cython library for low-level data structures like hash maps, used for memory efficient data storage.
+- [cython-blis](https://github.com/explosion/cython-blis): Fast matrix multiplication using BLIS without depending on system libraries. Required by Thinc, rather than spaCy directly.
+- [murmurhash](https://github.com/explosion/murmurhash): A wrapper library for a C++ murmurhash implementation, used for string IDs in spaCy and preshed.
+- [cymem](https://github.com/explosion/cymem): A small library for RAII-style memory management in Cython. 
+
+## Optional Extensions for spaCy
+
+These are repos that can be used by spaCy but aren't part of a default installation. Many of these are wrappers to integrate various kinds of third-party libraries.
+
+- [spacy-transformers](https://github.com/explosion/spacy-transformers): A wrapper for the [HuggingFace Transformers](https://huggingface.co/docs/transformers/index) library, this handles the extensive conversion necessary to coordinate spaCy's powerful `Doc` representation, training pipeline, and the Transformer embeddings. When released, this was known as `spacy-pytorch-transformers`, but it changed to the current name when HuggingFace update the name of their library as well.
+- [spacy-huggingface-hub](https://github.com/explosion/spacy-huggingface-hub): This package has a CLI script for uploading a packaged spaCy pipeline (created with `spacy package`) to the [Hugging Face Hub](https://huggingface.co/models).
+- [spacy-alignments](https://github.com/explosion/spacy-alignments): A wrapper for the tokenizations library (mentioned below) with a modified build system to simplify cross-platform wheel creation. Used in spacy-transformers for aligning spaCy and HuggingFace tokenizations.
+- [spacy-experimental](https://github.com/explosion/spacy-experimental): Experimental components that are not quite ready for inclusion in the main spaCy library. Usually there are unresolved questions around their APIs, so the experimental library allows us to expose them to the community for feedback before fully integrating them. 
+- [spacy-lookups-data](https://github.com/explosion/spacy-lookups-data): A repository of linguistic data, such as lemmas, that takes up a lot of disk space. Originally created to reduce the size of the spaCy core library. This is mainly useful if you want the data included but aren't using a pretrained pipeline; for the affected languages, the relevant data is included in pretrained pipelines directly.
+- [coreferee](https://github.com/explosion/coreferee): Coreference resolution for English, French, German and Polish, optimised for limited training data and easily extensible for further languages. Used as a spaCy pipeline component.
+- [spacy-stanza](https://github.com/explosion/spacy-stanza): This is a wrapper that allows the use of Stanford's Stanza library in spaCy. 
+- [spacy-streamlit](https://github.com/explosion/spacy-streamlit): A wrapper for the Streamlit dashboard building library to help with integrating [displaCy](https://spacy.io/api/top-level/#displacy).
+- [spacymoji](https://github.com/explosion/spacymoji): A library to add extra support for emoji to spaCy, such as including character names.
+- [thinc-apple-ops](https://github.com/explosion/thinc-apple-ops): A special backend for OSX that uses Apple's native libraries for improved performance.
+- [os-signpost](https://github.com/explosion/os-signpost): A Python package that allows you to use the `OSSignposter` API in OSX for performance analysis.
+- [spacy-ray](https://github.com/explosion/spacy-ray): A wrapper to integrate spaCy with Ray, a distributed training framework. Currently a work in progress.
+
+## Prodigy
+
+[Prodigy](https://prodi.gy) is Explosion's easy to use and highly customizable tool for annotating data. Prodigy itself requires a license, but the repos below contain documentation, examples, and editor or notebook integrations.
+
+- [prodigy-recipes](https://github.com/explosion/prodigy-recipes): Sample recipes for Prodigy, along with notebooks and other examples of usage.
+- [vscode-prodigy](https://github.com/explosion/vscode-prodigy): A VS Code extension that lets you run Prodigy inside VS Code.
+- [jupyterlab-prodigy](https://github.com/explosion/jupyterlab-prodigy): An extension for JupyterLab that lets you run Prodigy inside JupyterLab.
+
+## Independent Tools or Projects
+
+These are tools that may be related to or use spaCy, but are functional independent projects in their own right as well.
+
+- [floret](https://github.com/explosion/floret): A modification of fastText to use Bloom Embeddings. Can be used to add vectors with subword features to spaCy, and also works independently in the same manner as fastText.
+- [sense2vec](https://github.com/explosion/sense2vec): A library to make embeddings of noun phrases or words coupled with their part of speech. This library uses spaCy.
+- [spacy-vectors-builder](https://github.com/explosion/spacy-vectors-builder): This is a spaCy project that builds vectors using floret and a lot of input text. It handles downloading the input data as well as the actual building of vectors.
+- [holmes-extractor](https://github.com/explosion/holmes-extractor): Information extraction from English and German texts based on predicate logic. Uses spaCy.
+- [healthsea](https://github.com/explosion/healthsea): Healthsea is a project to extract information from comments about health supplements. Structurally, it's a self-contained, large spaCy project.
+- [spacy-pkuseg](https://github.com/explosion/spacy-pkuseg): A fork of the pkuseg Chinese tokenizer. Used for Chinese support in spaCy, but also works independently.
+- [ml-datasets](https://github.com/explosion/ml-datasets): This repo includes loaders for several standard machine learning datasets, like MNIST or WikiNER, and has historically been used in spaCy example code and documentation.
+
+## Documentation and Informational Repos
+
+These repos are used to support the spaCy docs or otherwise present information about spaCy or other Explosion projects.
+
+- [projects](https://github.com/explosion/projects): The projects repo is used to show detailed examples of spaCy usage. Individual projects can be checked out using the spaCy command line tool, rather than checking out the projects repo directly.
+- [spacy-course](https://github.com/explosion/spacy-course): Home to the interactive spaCy course for learning about how to use the library and some basic NLP principles.
+- [spacy-io-binder](https://github.com/explosion/spacy-io-binder): Home to the notebooks used for interactive examples in the documentation.
+
+## Organizational / Meta
+
+These repos are used for organizing data around spaCy, but are not something an end user would need to install as part of using the library.
+
+- [spacy-models](https://github.com/explosion/spacy-models): This repo contains metadata (but not training data) for all the spaCy models. This includes information about where their training data came from, version compatability, and performance information. It also includes tests for the model packages, and the built models are hosted as releases of this repo.
+- [wheelwright](https://github.com/explosion/wheelwright): A tool for automating our PyPI builds and releases.
+- [ec2buildwheel](https://github.com/explosion/ec2buildwheel): A small project that allows you to build Python packages in the manner of cibuildwheel, but on any EC2 image. Used by wheelwright.
+
+## Other
+
+Repos that don't fit in any of the above categories.
+
+- [blis](https://github.com/explosion/blis): A fork of the official BLIS library. The main branch is not updated, but work continues in various branches. This is used for cython-blis.
+- [tokenizations](https://github.com/explosion/tokenizations): A library originally by Yohei Tamura to align strings with tolerance to some variations in features like case and diacritics, used for aligning tokens and wordpieces. Adopted and maintained by Explosion, but usually spacy-alignments is used instead.
+- [conll-2012](https://github.com/explosion/conll-2012): A repo to hold some slightly cleaned up versions of the official scripts for the CoNLL 2012 shared task involving coreference resolution. Used in the coref project.
+- [fastapi-explosion-extras](https://github.com/explosion/fastapi-explosion-extras): Some small tweaks to FastAPI used at Explosion.
+

licenses/3rd_party_licenses.txt

@@ -127,3 +127,34 @@ distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
+
+
+polyleven
+---------
+
+* Files: spacy/matcher/polyleven.c
+
+MIT License
+
+Copyright (c) 2021 Fujimoto Seiji <fujimoto@ceptord.net>
+Copyright (c) 2021 Max Bachmann <kontakt@maxbachmann.de>
+Copyright (c) 2022 Nick Mazuk
+Copyright (c) 2022 Michael Weiss <code@mweiss.ch>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

pyproject.toml

@@ -6,7 +6,6 @@ requires = [
     "preshed>=3.0.2,<3.1.0",
     "murmurhash>=0.28.0,<1.1.0",
     "thinc>=8.1.0,<8.2.0",
-    "pathy",
     "numpy>=1.15.0",
 ]
 build-backend = "setuptools.build_meta"

requirements.txt

@@ -1,5 +1,5 @@
 # Our libraries
-spacy-legacy>=3.0.9,<3.1.0
+spacy-legacy>=3.0.10,<3.1.0
 spacy-loggers>=1.0.0,<2.0.0
 cymem>=2.0.2,<2.1.0
 preshed>=3.0.2,<3.1.0
@@ -15,7 +15,7 @@ pathy>=0.3.5
 numpy>=1.15.0
 requests>=2.13.0,<3.0.0
 tqdm>=4.38.0,<5.0.0
-pydantic>=1.7.4,!=1.8,!=1.8.1,<1.10.0
+pydantic>=1.7.4,!=1.8,!=1.8.1,<1.11.0
 jinja2
 langcodes>=3.2.0,<4.0.0
 # Official Python utilities
@@ -28,10 +28,12 @@ cython>=0.25,<3.0
 pytest>=5.2.0,!=7.1.0
 pytest-timeout>=1.3.0,<2.0.0
 mock>=2.0.0,<3.0.0
-flake8>=3.8.0,<3.10.0
+flake8>=3.8.0,<6.0.0
 hypothesis>=3.27.0,<7.0.0
-mypy>=0.910,<0.970; platform_machine!='aarch64'
+mypy>=0.980,<0.990; platform_machine != "aarch64" and python_version >= "3.7"
 types-dataclasses>=0.1.3; python_version < "3.7"
 types-mock>=0.1.1
+types-setuptools>=57.0.0
 types-requests
+types-setuptools>=57.0.0
 black>=22.0,<23.0

setup.cfg

@@ -41,7 +41,7 @@ setup_requires =
     thinc>=8.1.0,<8.2.0
 install_requires =
     # Our libraries
-    spacy-legacy>=3.0.9,<3.1.0
+    spacy-legacy>=3.0.10,<3.1.0
     spacy-loggers>=1.0.0,<2.0.0
     murmurhash>=0.28.0,<1.1.0
     cymem>=2.0.2,<2.1.0
@@ -50,13 +50,13 @@ install_requires =
     wasabi>=0.9.1,<1.1.0
     srsly>=2.4.3,<3.0.0
     catalogue>=2.0.6,<2.1.0
+    # Third-party dependencies
     typer>=0.3.0,<0.5.0
     pathy>=0.3.5
-    # Third-party dependencies
     tqdm>=4.38.0,<5.0.0
     numpy>=1.15.0
     requests>=2.13.0,<3.0.0
-    pydantic>=1.7.4,!=1.8,!=1.8.1,<1.10.0
+    pydantic>=1.7.4,!=1.8,!=1.8.1,<1.11.0
     jinja2
     # Official Python utilities
     setuptools
@@ -76,37 +76,41 @@ transformers =
 ray =
     spacy_ray>=0.1.0,<1.0.0
 cuda =
-    cupy>=5.0.0b4,<11.0.0
+    cupy>=5.0.0b4,<12.0.0
 cuda80 =
-    cupy-cuda80>=5.0.0b4,<11.0.0
+    cupy-cuda80>=5.0.0b4,<12.0.0
 cuda90 =
-    cupy-cuda90>=5.0.0b4,<11.0.0
+    cupy-cuda90>=5.0.0b4,<12.0.0
 cuda91 =
-    cupy-cuda91>=5.0.0b4,<11.0.0
+    cupy-cuda91>=5.0.0b4,<12.0.0
 cuda92 =
-    cupy-cuda92>=5.0.0b4,<11.0.0
+    cupy-cuda92>=5.0.0b4,<12.0.0
 cuda100 =
-    cupy-cuda100>=5.0.0b4,<11.0.0
+    cupy-cuda100>=5.0.0b4,<12.0.0
 cuda101 =
-    cupy-cuda101>=5.0.0b4,<11.0.0
+    cupy-cuda101>=5.0.0b4,<12.0.0
 cuda102 =
-    cupy-cuda102>=5.0.0b4,<11.0.0
+    cupy-cuda102>=5.0.0b4,<12.0.0
 cuda110 =
-    cupy-cuda110>=5.0.0b4,<11.0.0
+    cupy-cuda110>=5.0.0b4,<12.0.0
 cuda111 =
-    cupy-cuda111>=5.0.0b4,<11.0.0
+    cupy-cuda111>=5.0.0b4,<12.0.0
 cuda112 =
-    cupy-cuda112>=5.0.0b4,<11.0.0
+    cupy-cuda112>=5.0.0b4,<12.0.0
 cuda113 =
-    cupy-cuda113>=5.0.0b4,<11.0.0
+    cupy-cuda113>=5.0.0b4,<12.0.0
 cuda114 =
-    cupy-cuda114>=5.0.0b4,<11.0.0
+    cupy-cuda114>=5.0.0b4,<12.0.0
 cuda115 =
-    cupy-cuda115>=5.0.0b4,<11.0.0
+    cupy-cuda115>=5.0.0b4,<12.0.0
 cuda116 =
-    cupy-cuda116>=5.0.0b4,<11.0.0
+    cupy-cuda116>=5.0.0b4,<12.0.0
 cuda117 =
-    cupy-cuda117>=5.0.0b4,<11.0.0
+    cupy-cuda117>=5.0.0b4,<12.0.0
+cuda11x =
+    cupy-cuda11x>=11.0.0,<12.0.0
+cuda-autodetect =
+    cupy-wheel>=11.0.0,<12.0.0
 apple =
     thinc-apple-ops>=0.1.0.dev0,<1.0.0
 # Language tokenizers with external dependencies

setup.py

@@ -30,7 +30,9 @@ MOD_NAMES = [
     "spacy.lexeme",
     "spacy.vocab",
     "spacy.attrs",
-    "spacy.kb",
+    "spacy.kb.candidate",
+    "spacy.kb.kb",
+    "spacy.kb.kb_in_memory",
     "spacy.ml.parser_model",
     "spacy.morphology",
     "spacy.pipeline.dep_parser",
@@ -205,6 +207,17 @@ def setup_package():
         get_python_inc(plat_specific=True),
     ]
     ext_modules = []
+    ext_modules.append(
+        Extension(
+            "spacy.matcher.levenshtein",
+            [
+                "spacy/matcher/levenshtein.pyx",
+                "spacy/matcher/polyleven.c",
+            ],
+            language="c",
+            include_dirs=include_dirs,
+        )
+    )
     for name in MOD_NAMES:
         mod_path = name.replace(".", "/") + ".pyx"
         ext = Extension(

spacy/__init__.py

@@ -31,21 +31,21 @@ def load(
     name: Union[str, Path],
     *,
     vocab: Union[Vocab, bool] = True,
-    disable: Iterable[str] = util.SimpleFrozenList(),
+    disable: Union[str, Iterable[str]] = util._DEFAULT_EMPTY_PIPES,
-    enable: Iterable[str] = util.SimpleFrozenList(),
+    enable: Union[str, Iterable[str]] = util._DEFAULT_EMPTY_PIPES,
-    exclude: Iterable[str] = util.SimpleFrozenList(),
+    exclude: Union[str, Iterable[str]] = util._DEFAULT_EMPTY_PIPES,
     config: Union[Dict[str, Any], Config] = util.SimpleFrozenDict(),
 ) -> Language:
     """Load a spaCy model from an installed package or a local path.
 
     name (str): Package name or model path.
     vocab (Vocab): A Vocab object. If True, a vocab is created.
-    disable (Iterable[str]): Names of pipeline components to disable. Disabled
+    disable (Union[str, Iterable[str]]): Name(s) of pipeline component(s) to disable. Disabled
         pipes will be loaded but they won't be run unless you explicitly
         enable them by calling nlp.enable_pipe.
-    enable (Iterable[str]): Names of pipeline components to enable. All other
+    enable (Union[str, Iterable[str]]): Name(s) of pipeline component(s) to enable. All other
         pipes will be disabled (but can be enabled later using nlp.enable_pipe).
-    exclude (Iterable[str]): Names of pipeline components to exclude. Excluded
+    exclude (Union[str, Iterable[str]]): Name(s) of pipeline component(s) to exclude. Excluded
         components won't be loaded.
     config (Dict[str, Any] / Config): Config overrides as nested dict or dict
         keyed by section values in dot notation.

spacy/about.py

@@ -1,6 +1,6 @@
 # fmt: off
 __title__ = "spacy"
-__version__ = "3.4.1"
+__version__ = "3.4.2"
 __download_url__ = "https://github.com/explosion/spacy-models/releases/download"
 __compatibility__ = "https://raw.githubusercontent.com/explosion/spacy-models/master/compatibility.json"
 __projects__ = "https://github.com/explosion/projects"

spacy/cli/_util.py

@@ -573,3 +573,12 @@ def setup_gpu(use_gpu: int, silent=None) -> None:
         local_msg.info("Using CPU")
         if gpu_is_available():
             local_msg.info("To switch to GPU 0, use the option: --gpu-id 0")
+
+
+def _format_number(number: Union[int, float], ndigits: int = 2) -> str:
+    """Formats a number (float or int) rounding to `ndigits`, without truncating trailing 0s,
+    as happens with `round(number, ndigits)`"""
+    if isinstance(number, float):
+        return f"{number:.{ndigits}f}"
+    else:
+        return str(number)

spacy/cli/debug_data.py

@@ -9,7 +9,7 @@ import typer
 import math
 
 from ._util import app, Arg, Opt, show_validation_error, parse_config_overrides
-from ._util import import_code, debug_cli
+from ._util import import_code, debug_cli, _format_number
 from ..training import Example, remove_bilu_prefix
 from ..training.initialize import get_sourced_components
 from ..schemas import ConfigSchemaTraining
@@ -989,7 +989,8 @@ def _get_kl_divergence(p: Counter, q: Counter) -> float:
 def _format_span_row(span_data: List[Dict], labels: List[str]) -> List[Any]:
     """Compile into one list for easier reporting"""
     d = {
-        label: [label] + list(round(d[label], 2) for d in span_data) for label in labels
+        label: [label] + list(_format_number(d[label]) for d in span_data)
+        for label in labels
     }
     return list(d.values())
 
@@ -1004,6 +1005,10 @@ def _get_span_characteristics(
         label: _gmean(l)
         for label, l in compiled_gold["spans_length"][spans_key].items()
     }
+    spans_per_type = {
+        label: len(spans)
+        for label, spans in compiled_gold["spans_per_type"][spans_key].items()
+    }
     min_lengths = [min(l) for l in compiled_gold["spans_length"][spans_key].values()]
     max_lengths = [max(l) for l in compiled_gold["spans_length"][spans_key].values()]
 
@@ -1031,6 +1036,7 @@ def _get_span_characteristics(
     return {
         "sd": span_distinctiveness,
         "bd": sb_distinctiveness,
+        "spans_per_type": spans_per_type,
         "lengths": span_length,
         "min_length": min(min_lengths),
         "max_length": max(max_lengths),
@@ -1045,12 +1051,15 @@ def _get_span_characteristics(
 
 def _print_span_characteristics(span_characteristics: Dict[str, Any]):
     """Print all span characteristics into a table"""
-    headers = ("Span Type", "Length", "SD", "BD")
+    headers = ("Span Type", "Length", "SD", "BD", "N")
+    # Wasabi has this at 30 by default, but we might have some long labels
+    max_col = max(30, max(len(label) for label in span_characteristics["labels"]))
     # Prepare table data with all span characteristics
     table_data = [
         span_characteristics["lengths"],
         span_characteristics["sd"],
         span_characteristics["bd"],
+        span_characteristics["spans_per_type"],
     ]
     table = _format_span_row(
         span_data=table_data, labels=span_characteristics["labels"]
@@ -1061,8 +1070,18 @@ def _print_span_characteristics(span_characteristics: Dict[str, Any]):
         span_characteristics["avg_sd"],
         span_characteristics["avg_bd"],
     ]
-    footer = ["Wgt. Average"] + [str(round(f, 2)) for f in footer_data]
+
-    msg.table(table, footer=footer, header=headers, divider=True)
+    footer = (
+        ["Wgt. Average"] + ["{:.2f}".format(round(f, 2)) for f in footer_data] + ["-"]
+    )
+    msg.table(
+        table,
+        footer=footer,
+        header=headers,
+        divider=True,
+        aligns=["l"] + ["r"] * (len(footer_data) + 1),
+        max_col=max_col,
+    )
 
 
 def _get_spans_length_freq_dist(

spacy/cli/download.py

@@ -7,6 +7,7 @@ import typer
 from ._util import app, Arg, Opt, WHEEL_SUFFIX, SDIST_SUFFIX
 from .. import about
 from ..util import is_package, get_minor_version, run_command
+from ..util import is_prerelease_version
 from ..errors import OLD_MODEL_SHORTCUTS
 
 
@@ -19,7 +20,7 @@ def download_cli(
     ctx: typer.Context,
     model: str = Arg(..., help="Name of pipeline package to download"),
     direct: bool = Opt(False, "--direct", "-d", "-D", help="Force direct download of name + version"),
-    sdist: bool = Opt(False, "--sdist", "-S", help="Download sdist (.tar.gz) archive instead of pre-built binary wheel")
+    sdist: bool = Opt(False, "--sdist", "-S", help="Download sdist (.tar.gz) archive instead of pre-built binary wheel"),
     # fmt: on
 ):
     """
@@ -35,7 +36,12 @@ def download_cli(
     download(model, direct, sdist, *ctx.args)
 
 
-def download(model: str, direct: bool = False, sdist: bool = False, *pip_args) -> None:
+def download(
+    model: str,
+    direct: bool = False,
+    sdist: bool = False,
+    *pip_args,
+) -> None:
     if (
         not (is_package("spacy") or is_package("spacy-nightly"))
         and "--no-deps" not in pip_args
@@ -49,13 +55,10 @@ def download(model: str, direct: bool = False, sdist: bool = False, *pip_args) -
             "dependencies, you'll have to install them manually."
         )
         pip_args = pip_args + ("--no-deps",)
-    suffix = SDIST_SUFFIX if sdist else WHEEL_SUFFIX
-    dl_tpl = "{m}-{v}/{m}-{v}{s}#egg={m}=={v}"
     if direct:
         components = model.split("-")
         model_name = "".join(components[:-1])
         version = components[-1]
-        download_model(dl_tpl.format(m=model_name, v=version, s=suffix), pip_args)
     else:
         model_name = model
         if model in OLD_MODEL_SHORTCUTS:
@@ -66,14 +69,30 @@ def download(model: str, direct: bool = False, sdist: bool = False, *pip_args) -
             model_name = OLD_MODEL_SHORTCUTS[model]
         compatibility = get_compatibility()
         version = get_version(model_name, compatibility)
-        download_model(dl_tpl.format(m=model_name, v=version, s=suffix), pip_args)
+
+    filename = get_model_filename(model_name, version, sdist)
+
+    download_model(filename, pip_args)
     msg.good(
         "Download and installation successful",
         f"You can now load the package via spacy.load('{model_name}')",
     )
 
 
+def get_model_filename(model_name: str, version: str, sdist: bool = False) -> str:
+    dl_tpl = "{m}-{v}/{m}-{v}{s}"
+    egg_tpl = "#egg={m}=={v}"
+    suffix = SDIST_SUFFIX if sdist else WHEEL_SUFFIX
+    filename = dl_tpl.format(m=model_name, v=version, s=suffix)
+    if sdist:
+        filename += egg_tpl.format(m=model_name, v=version)
+    return filename
+
+
 def get_compatibility() -> dict:
+    if is_prerelease_version(about.__version__):
+        version: Optional[str] = about.__version__
+    else:
         version = get_minor_version(about.__version__)
     r = requests.get(about.__compatibility__)
     if r.status_code != 200:
@@ -101,6 +120,11 @@ def get_version(model: str, comp: dict) -> str:
     return comp[model][0]
 
 
+def get_latest_version(model: str) -> str:
+    comp = get_compatibility()
+    return get_version(model, comp)
+
+
 def download_model(
     filename: str, user_pip_args: Optional[Sequence[str]] = None
 ) -> None:

spacy/cli/find_threshold.py

@@ -109,7 +109,7 @@ def find_threshold(
     except KeyError as err:
         wasabi.msg.fail(title=str(err), exits=1)
     if not hasattr(pipe, "scorer"):
-        raise AttributeError(Errors.E1045)
+        raise AttributeError(Errors.E1048)
 
     if not silent:
         wasabi.msg.info(

spacy/cli/info.py

@@ -1,10 +1,13 @@
 from typing import Optional, Dict, Any, Union, List
 import platform
+import pkg_resources
+import json
 from pathlib import Path
 from wasabi import Printer, MarkdownRenderer
 import srsly
 
 from ._util import app, Arg, Opt, string_to_list
+from .download import get_model_filename, get_latest_version
 from .. import util
 from .. import about
 
@@ -16,6 +19,7 @@ def info_cli(
     markdown: bool = Opt(False, "--markdown", "-md", help="Generate Markdown for GitHub issues"),
     silent: bool = Opt(False, "--silent", "-s", "-S", help="Don't print anything (just return)"),
     exclude: str = Opt("labels", "--exclude", "-e", help="Comma-separated keys to exclude from the print-out"),
+    url: bool = Opt(False, "--url", "-u", help="Print the URL to download the most recent compatible version of the pipeline"),
     # fmt: on
 ):
     """
@@ -23,10 +27,19 @@ def info_cli(
     print its meta information. Flag --markdown prints details in Markdown for easy
     copy-pasting to GitHub issues.
 
+    Flag --url prints only the download URL of the most recent compatible
+    version of the pipeline.
+
     DOCS: https://spacy.io/api/cli#info
     """
     exclude = string_to_list(exclude)
-    info(model, markdown=markdown, silent=silent, exclude=exclude)
+    info(
+        model,
+        markdown=markdown,
+        silent=silent,
+        exclude=exclude,
+        url=url,
+    )
 
 
 def info(
@@ -35,11 +48,20 @@ def info(
     markdown: bool = False,
     silent: bool = True,
     exclude: Optional[List[str]] = None,
+    url: bool = False,
 ) -> Union[str, dict]:
     msg = Printer(no_print=silent, pretty=not silent)
     if not exclude:
         exclude = []
-    if model:
+    if url:
+        if model is not None:
+            title = f"Download info for pipeline '{model}'"
+            data = info_model_url(model)
+            print(data["download_url"])
+            return data
+        else:
+            msg.fail("--url option requires a pipeline name", exits=1)
+    elif model:
         title = f"Info about pipeline '{model}'"
         data = info_model(model, silent=silent)
     else:
@@ -99,11 +121,44 @@ def info_model(model: str, *, silent: bool = True) -> Dict[str, Any]:
         meta["source"] = str(model_path.resolve())
     else:
         meta["source"] = str(model_path)
+    download_url = info_installed_model_url(model)
+    if download_url:
+        meta["download_url"] = download_url
     return {
         k: v for k, v in meta.items() if k not in ("accuracy", "performance", "speed")
     }
 
 
+def info_installed_model_url(model: str) -> Optional[str]:
+    """Given a pipeline name, get the download URL if available, otherwise
+    return None.
+
+    This is only available for pipelines installed as modules that have
+    dist-info available.
+    """
+    try:
+        dist = pkg_resources.get_distribution(model)
+        data = json.loads(dist.get_metadata("direct_url.json"))
+        return data["url"]
+    except pkg_resources.DistributionNotFound:
+        # no such package
+        return None
+    except Exception:
+        # something else, like no file or invalid JSON
+        return None
+
+
+def info_model_url(model: str) -> Dict[str, Any]:
+    """Return the download URL for the latest version of a pipeline."""
+    version = get_latest_version(model)
+
+    filename = get_model_filename(model, version)
+    download_url = about.__download_url__ + "/" + filename
+    release_tpl = "https://github.com/explosion/spacy-models/releases/tag/{m}-{v}"
+    release_url = release_tpl.format(m=model, v=version)
+    return {"download_url": download_url, "release_url": release_url}
+
+
 def get_markdown(
     data: Dict[str, Any],
     title: Optional[str] = None,

spacy/cli/package.py

@@ -299,8 +299,8 @@ def get_meta(
     }
     nlp = util.load_model_from_path(Path(model_path))
     meta.update(nlp.meta)
-    meta.update(existing_meta)
     meta["spacy_version"] = util.get_minor_version_range(about.__version__)
+    meta.update(existing_meta)
     meta["vectors"] = {
         "width": nlp.vocab.vectors_length,
         "vectors": len(nlp.vocab.vectors),

spacy/cli/project/dvc.py

@@ -25,6 +25,7 @@ def project_update_dvc_cli(
     project_dir: Path = Arg(Path.cwd(), help="Location of project directory. Defaults to current working directory.", exists=True, file_okay=False),
     workflow: Optional[str] = Arg(None, help=f"Name of workflow defined in {PROJECT_FILE}. Defaults to first workflow if not set."),
     verbose: bool = Opt(False, "--verbose", "-V", help="Print more info"),
+    quiet: bool = Opt(False, "--quiet", "-q", help="Print less info"),
     force: bool = Opt(False, "--force", "-F", help="Force update DVC config"),
     # fmt: on
 ):
@@ -36,7 +37,7 @@ def project_update_dvc_cli(
 
     DOCS: https://spacy.io/api/cli#project-dvc
     """
-    project_update_dvc(project_dir, workflow, verbose=verbose, force=force)
+    project_update_dvc(project_dir, workflow, verbose=verbose, quiet=quiet, force=force)
 
 
 def project_update_dvc(
@@ -44,6 +45,7 @@ def project_update_dvc(
     workflow: Optional[str] = None,
     *,
     verbose: bool = False,
+    quiet: bool = False,
     force: bool = False,
 ) -> None:
     """Update the auto-generated Data Version Control (DVC) config file. A DVC
@@ -54,11 +56,12 @@ def project_update_dvc(
     workflow (Optional[str]): Optional name of workflow defined in project.yml.
         If not set, the first workflow will be used.
     verbose (bool): Print more info.
+    quiet (bool): Print less info.
     force (bool): Force update DVC config.
     """
     config = load_project_config(project_dir)
     updated = update_dvc_config(
-        project_dir, config, workflow, verbose=verbose, force=force
+        project_dir, config, workflow, verbose=verbose, quiet=quiet, force=force
     )
     help_msg = "To execute the workflow with DVC, run: dvc repro"
     if updated:
@@ -72,7 +75,7 @@ def update_dvc_config(
     config: Dict[str, Any],
     workflow: Optional[str] = None,
     verbose: bool = False,
-    silent: bool = False,
+    quiet: bool = False,
     force: bool = False,
 ) -> bool:
     """Re-run the DVC commands in dry mode and update dvc.yaml file in the
@@ -83,7 +86,7 @@ def update_dvc_config(
     path (Path): The path to the project directory.
     config (Dict[str, Any]): The loaded project.yml.
     verbose (bool): Whether to print additional info (via DVC).
-    silent (bool): Don't output anything (via DVC).
+    quiet (bool): Don't output anything (via DVC).
     force (bool): Force update, even if hashes match.
     RETURNS (bool): Whether the DVC config file was updated.
     """
@@ -105,6 +108,14 @@ def update_dvc_config(
         dvc_config_path.unlink()
     dvc_commands = []
     config_commands = {cmd["name"]: cmd for cmd in config.get("commands", [])}
+
+    # some flags that apply to every command
+    flags = []
+    if verbose:
+        flags.append("--verbose")
+    if quiet:
+        flags.append("--quiet")
+
     for name in workflows[workflow]:
         command = config_commands[name]
         deps = command.get("deps", [])
@@ -118,14 +129,26 @@ def update_dvc_config(
         deps_cmd = [c for cl in [["-d", p] for p in deps] for c in cl]
         outputs_cmd = [c for cl in [["-o", p] for p in outputs] for c in cl]
         outputs_nc_cmd = [c for cl in [["-O", p] for p in outputs_no_cache] for c in cl]
-        dvc_cmd = ["run", "-n", name, "-w", str(path), "--no-exec"]
+
+        dvc_cmd = ["run", *flags, "-n", name, "-w", str(path), "--no-exec"]
         if command.get("no_skip"):
             dvc_cmd.append("--always-changed")
         full_cmd = [*dvc_cmd, *deps_cmd, *outputs_cmd, *outputs_nc_cmd, *project_cmd]
         dvc_commands.append(join_command(full_cmd))
+
+    if not dvc_commands:
+        # If we don't check for this, then there will be an error when reading the
+        # config, since DVC wouldn't create it.
+        msg.fail(
+            "No usable commands for DVC found. This can happen if none of your "
+            "commands have dependencies or outputs.",
+            exits=1,
+        )
+
     with working_dir(path):
-        dvc_flags = {"--verbose": verbose, "--quiet": silent}
+        for c in dvc_commands:
-        run_dvc_commands(dvc_commands, flags=dvc_flags)
+            dvc_command = "dvc " + c
+            run_command(dvc_command)
     with dvc_config_path.open("r+", encoding="utf8") as f:
         content = f.read()
         f.seek(0, 0)
@@ -133,26 +156,6 @@ def update_dvc_config(
     return True
 
 
-def run_dvc_commands(
-    commands: Iterable[str] = SimpleFrozenList(), flags: Dict[str, bool] = {}
-) -> None:
-    """Run a sequence of DVC commands in a subprocess, in order.
-
-    commands (List[str]): The string commands without the leading "dvc".
-    flags (Dict[str, bool]): Conditional flags to be added to command. Makes it
-        easier to pass flags like --quiet that depend on a variable or
-        command-line setting while avoiding lots of nested conditionals.
-    """
-    for c in commands:
-        command = split_command(c)
-        dvc_command = ["dvc", *command]
-        # Add the flags if they are set to True
-        for flag, is_active in flags.items():
-            if is_active:
-                dvc_command.append(flag)
-        run_command(dvc_command)
-
-
 def check_workflows(workflows: List[str], workflow: Optional[str] = None) -> None:
     """Validate workflows provided in project.yml and check that a given
     workflow can be used to generate a DVC config.

spacy/cli/project/run.py

@@ -1,5 +1,8 @@
-from typing import Optional, List, Dict, Sequence, Any, Iterable
+from typing import Optional, List, Dict, Sequence, Any, Iterable, Tuple
+import os.path
 from pathlib import Path
+
+import pkg_resources
 from wasabi import msg
 from wasabi.util import locale_escape
 import sys
@@ -71,6 +74,12 @@ def project_run(
     commands = {cmd["name"]: cmd for cmd in config.get("commands", [])}
     workflows = config.get("workflows", {})
     validate_subcommand(list(commands.keys()), list(workflows.keys()), subcommand)
+
+    req_path = project_dir / "requirements.txt"
+    if config.get("check_requirements", True) and os.path.exists(req_path):
+        with req_path.open() as requirements_file:
+            _check_requirements([req.replace("\n", "") for req in requirements_file])
+
     if subcommand in workflows:
         msg.info(f"Running workflow '{subcommand}'")
         for cmd in workflows[subcommand]:
@@ -195,6 +204,8 @@ def validate_subcommand(
         msg.fail(f"No commands or workflows defined in {PROJECT_FILE}", exits=1)
     if subcommand not in commands and subcommand not in workflows:
         help_msg = []
+        if subcommand in ["assets", "asset"]:
+            help_msg.append("Did you mean to run: python -m spacy project assets?")
         if commands:
             help_msg.append(f"Available commands: {', '.join(commands)}")
         if workflows:
@@ -308,3 +319,32 @@ def get_fileinfo(project_dir: Path, paths: List[str]) -> List[Dict[str, Optional
         md5 = get_checksum(file_path) if file_path.exists() else None
         data.append({"path": path, "md5": md5})
     return data
+
+
+def _check_requirements(requirements: List[str]) -> Tuple[bool, bool]:
+    """Checks whether requirements are installed and free of version conflicts.
+    requirements (List[str]): List of requirements.
+    RETURNS (Tuple[bool, bool]): Whether (1) any packages couldn't be imported, (2) any packages with version conflicts
+        exist.
+    """
+
+    failed_pkgs_msgs: List[str] = []
+    conflicting_pkgs_msgs: List[str] = []
+
+    for req in requirements:
+        try:
+            pkg_resources.require(req)
+        except pkg_resources.DistributionNotFound as dnf:
+            failed_pkgs_msgs.append(dnf.report())
+        except pkg_resources.VersionConflict as vc:
+            conflicting_pkgs_msgs.append(vc.report())
+
+    if len(failed_pkgs_msgs) or len(conflicting_pkgs_msgs):
+        msg.warn(
+            title="Missing requirements or requirement conflicts detected. Make sure your Python environment is set up "
+            "correctly and you installed all requirements specified in your project's requirements.txt: "
+        )
+        for pgk_msg in failed_pkgs_msgs + conflicting_pkgs_msgs:
+            msg.text(pgk_msg)
+
+    return len(failed_pkgs_msgs) > 0, len(conflicting_pkgs_msgs) > 0

spacy/cli/templates/quickstart_training.jinja

@@ -271,13 +271,8 @@ factory = "tok2vec"
 [components.tok2vec.model.embed]
 @architectures = "spacy.MultiHashEmbed.v2"
 width = ${components.tok2vec.model.encode.width}
-{% if has_letters -%}
 attrs = ["NORM", "PREFIX", "SUFFIX", "SHAPE"]
-rows = [5000, 2500, 2500, 2500]
+rows = [5000, 1000, 2500, 2500]
-{% else -%}
-attrs = ["ORTH", "SHAPE"]
-rows = [5000, 2500]
-{% endif -%}
 include_static_vectors = {{ "true" if optimize == "accuracy" else "false" }}
 
 [components.tok2vec.model.encode]

spacy/cli/templates/quickstart_training_recommendations.yml

@@ -271,4 +271,3 @@ zh:
     accuracy:
       name: bert-base-chinese
       size_factor: 3
-  has_letters: false

spacy/displacy/__init__.py

@@ -123,7 +123,8 @@ def app(environ, start_response):
 def parse_deps(orig_doc: Doc, options: Dict[str, Any] = {}) -> Dict[str, Any]:
     """Generate dependency parse in {'words': [], 'arcs': []} format.
 
-    doc (Doc): Document do parse.
+    orig_doc (Doc): Document to parse.
+    options (Dict[str, Any]): Dependency parse specific visualisation options.
     RETURNS (dict): Generated dependency parse keyed by words and arcs.
     """
     doc = Doc(orig_doc.vocab).from_bytes(
@@ -209,7 +210,7 @@ def parse_ents(doc: Doc, options: Dict[str, Any] = {}) -> Dict[str, Any]:
 
 
 def parse_spans(doc: Doc, options: Dict[str, Any] = {}) -> Dict[str, Any]:
-    """Generate spans in [{start: i, end: i, label: 'label'}] format.
+    """Generate spans in [{start_token: i, end_token: i, label: 'label'}] format.
 
     doc (Doc): Document to parse.
     options (Dict[str, any]): Span-specific visualisation options.

spacy/errors.py

@@ -16,8 +16,8 @@ def setup_default_warnings():
     filter_warning("ignore", error_msg="numpy.dtype size changed")  # noqa
     filter_warning("ignore", error_msg="numpy.ufunc size changed")  # noqa
 
-    # warn about entity_ruler & matcher having no patterns only once
+    # warn about entity_ruler, span_ruler & matcher having no patterns only once
-    for pipe in ["matcher", "entity_ruler"]:
+    for pipe in ["matcher", "entity_ruler", "span_ruler"]:
         filter_warning("once", error_msg=Warnings.W036.format(name=pipe))
 
     # warn once about lemmatizer without required POS
@@ -212,6 +212,8 @@ class Warnings(metaclass=ErrorsWithCodes):
     W121 = ("Attempting to trace non-existent method '{method}' in pipe '{pipe}'")
     W122 = ("Couldn't trace method '{method}' in pipe '{pipe}'. This can happen if the pipe class "
             "is a Cython extension type.")
+    W123 = ("Argument {arg} with value {arg_value} is used instead of {config_value} as specified in the config. Be "
+            "aware that this might affect other components in your pipeline.")
 
 
 class Errors(metaclass=ErrorsWithCodes):
@@ -230,8 +232,9 @@ class Errors(metaclass=ErrorsWithCodes):
             "initialized component.")
     E004 = ("Can't set up pipeline component: a factory for '{name}' already "
             "exists. Existing factory: {func}. New factory: {new_func}")
-    E005 = ("Pipeline component '{name}' returned None. If you're using a "
+    E005 = ("Pipeline component '{name}' returned {returned_type} instead of a "
-            "custom component, maybe you forgot to return the processed Doc?")
+            "Doc. If you're using a custom component, maybe you forgot to "
+            "return the processed Doc?")
     E006 = ("Invalid constraints for adding pipeline component. You can only "
             "set one of the following: before (component name or index), "
             "after (component name or index), first (True) or last (True). "
@@ -389,7 +392,7 @@ class Errors(metaclass=ErrorsWithCodes):
             "consider using doc.spans instead.")
     E106 = ("Can't find `doc._.{attr}` attribute specified in the underscore "
             "settings: {opts}")
-    E107 = ("Value of `doc._.{attr}` is not JSON-serializable: {value}")
+    E107 = ("Value of custom attribute `{attr}` is not JSON-serializable: {value}")
     E109 = ("Component '{name}' could not be run. Did you forget to "
             "call `initialize()`?")
     E110 = ("Invalid displaCy render wrapper. Expected callable, got: {obj}")
@@ -535,11 +538,14 @@ class Errors(metaclass=ErrorsWithCodes):
     E198 = ("Unable to return {n} most similar vectors for the current vectors "
             "table, which contains {n_rows} vectors.")
     E199 = ("Unable to merge 0-length span at `doc[{start}:{end}]`.")
-    E200 = ("Can't yet set {attr} from Span. Vote for this feature on the "
+    E200 = ("Can't set {attr} from Span.")
-            "issue tracker: http://github.com/explosion/spaCy/issues")
     E202 = ("Unsupported {name} mode '{mode}'. Supported modes: {modes}.")
+    E203 = ("If the {name} embedding layer is not updated "
+            "during training, make sure to include it in 'annotating components'")
 
     # New errors added in v3.x
+    E853 = ("Unsupported component factory name '{name}'. The character '.' is "
+            "not permitted in factory names.")
     E854 = ("Unable to set doc.ents. Check that the 'ents_filter' does not "
             "permit overlapping spans.")
     E855 = ("Invalid {obj}: {obj} is not from the same doc.")
@@ -705,11 +711,11 @@ class Errors(metaclass=ErrorsWithCodes):
             "need to modify the pipeline, use the built-in methods like "
             "`nlp.add_pipe`, `nlp.remove_pipe`, `nlp.disable_pipe` or "
             "`nlp.enable_pipe` instead.")
-    E927 = ("Can't write to frozen list Maybe you're trying to modify a computed "
+    E927 = ("Can't write to frozen list. Maybe you're trying to modify a computed "
             "property or default function argument?")
-    E928 = ("A KnowledgeBase can only be serialized to/from from a directory, "
+    E928 = ("An InMemoryLookupKB can only be serialized to/from from a directory, "
             "but the provided argument {loc} points to a file.")
-    E929 = ("Couldn't read KnowledgeBase from {loc}. The path does not seem to exist.")
+    E929 = ("Couldn't read InMemoryLookupKB from {loc}. The path does not seem to exist.")
     E930 = ("Received invalid get_examples callback in `{method}`. "
             "Expected function that returns an iterable of Example objects but "
             "got: {obj}")
@@ -935,12 +941,19 @@ class Errors(metaclass=ErrorsWithCodes):
     E1040 = ("Doc.from_json requires all tokens to have the same attributes. "
              "Some tokens do not contain annotation for: {partial_attrs}")
     E1041 = ("Expected a string, Doc, or bytes as input, but got: {type}")
-    E1042 = ("Function was called with `{arg1}`={arg1_values} and "
+    E1042 = ("`enable={enable}` and `disable={disable}` are inconsistent with each other.\nIf you only passed "
-             "`{arg2}`={arg2_values} but these arguments are conflicting.")
+             "one of `enable` or `disable`, the other argument is specified in your pipeline's configuration.\nIn that "
+             "case pass an empty list for the previously not specified argument to avoid this error.")
     E1043 = ("Expected None or a value in range [{range_start}, {range_end}] for entity linker threshold, but got "
              "{value}.")
-    E1044 = ("`find_threshold()` only supports components of type `TrainablePipe`.")
+    E1044 = ("Expected `candidates_batch_size` to be >= 1, but got: {value}")
-    E1045 = ("`find_threshold()` only supports components with a `scorer` attribute.")
+    E1045 = ("Encountered {parent} subclass without `{parent}.{method}` "
+             "method in '{name}'. If you want to use this method, make "
+             "sure it's overwritten on the subclass.")
+    E1046 = ("{cls_name} is an abstract class and cannot be instantiated. If you are looking for spaCy's default "
+             "knowledge base, use `InMemoryLookupKB`.")
+    E1047 = ("`find_threshold()` only supports components of type `TrainablePipe`.")
+    E1048 = ("`find_threshold()` only supports components with a `scorer` attribute.")
 
 
 # Deprecated model shortcuts, only used in errors and warnings

spacy/kb/__init__.py

@@ -0,0 +1,3 @@
+from .kb import KnowledgeBase
+from .kb_in_memory import InMemoryLookupKB
+from .candidate import Candidate, get_candidates, get_candidates_batch

spacy/kb/candidate.pxd

@@ -0,0 +1,12 @@
+from .kb cimport KnowledgeBase
+from libcpp.vector cimport vector
+from ..typedefs cimport hash_t
+
+# Object used by the Entity Linker that summarizes one entity-alias candidate combination.
+cdef class Candidate:
+    cdef readonly KnowledgeBase kb
+    cdef hash_t entity_hash
+    cdef float entity_freq
+    cdef vector[float] entity_vector
+    cdef hash_t alias_hash
+    cdef float prior_prob

spacy/kb/candidate.pyx

@@ -0,0 +1,74 @@
+# cython: infer_types=True, profile=True
+
+from typing import Iterable
+from .kb cimport KnowledgeBase
+from ..tokens import Span
+
+cdef class Candidate:
+    """A `Candidate` object refers to a textual mention (`alias`) that may or may not be resolved
+    to a specific `entity` from a Knowledge Base. This will be used as input for the entity linking
+    algorithm which will disambiguate the various candidates to the correct one.
+    Each candidate (alias, entity) pair is assigned a certain prior probability.
+
+    DOCS: https://spacy.io/api/kb/#candidate-init
+    """
+
+    def __init__(self, KnowledgeBase kb, entity_hash, entity_freq, entity_vector, alias_hash, prior_prob):
+        self.kb = kb
+        self.entity_hash = entity_hash
+        self.entity_freq = entity_freq
+        self.entity_vector = entity_vector
+        self.alias_hash = alias_hash
+        self.prior_prob = prior_prob
+
+    @property
+    def entity(self) -> int:
+        """RETURNS (uint64): hash of the entity's KB ID/name"""
+        return self.entity_hash
+
+    @property
+    def entity_(self) -> str:
+        """RETURNS (str): ID/name of this entity in the KB"""
+        return self.kb.vocab.strings[self.entity_hash]
+
+    @property
+    def alias(self) -> int:
+        """RETURNS (uint64): hash of the alias"""
+        return self.alias_hash
+
+    @property
+    def alias_(self) -> str:
+        """RETURNS (str): ID of the original alias"""
+        return self.kb.vocab.strings[self.alias_hash]
+
+    @property
+    def entity_freq(self) -> float:
+        return self.entity_freq
+
+    @property
+    def entity_vector(self) -> Iterable[float]:
+        return self.entity_vector
+
+    @property
+    def prior_prob(self) -> float:
+        return self.prior_prob
+
+
+def get_candidates(kb: KnowledgeBase, mention: Span) -> Iterable[Candidate]:
+    """
+    Return candidate entities for a given mention and fetching appropriate entries from the index.
+    kb (KnowledgeBase): Knowledge base to query.
+    mention (Span): Entity mention for which to identify candidates.
+    RETURNS (Iterable[Candidate]): Identified candidates.
+    """
+    return kb.get_candidates(mention)
+
+
+def get_candidates_batch(kb: KnowledgeBase, mentions: Iterable[Span]) -> Iterable[Iterable[Candidate]]:
+    """
+    Return candidate entities for the given mentions and fetching appropriate entries from the index.
+    kb (KnowledgeBase): Knowledge base to query.
+    mention (Iterable[Span]): Entity mentions for which to identify candidates.
+    RETURNS (Iterable[Iterable[Candidate]]): Identified candidates.
+    """
+    return kb.get_candidates_batch(mentions)

spacy/kb/kb.pxd

@@ -0,0 +1,10 @@
+"""Knowledge-base for entity or concept linking."""
+
+from cymem.cymem cimport Pool
+from libc.stdint cimport int64_t
+from ..vocab cimport Vocab
+
+cdef class KnowledgeBase:
+    cdef Pool mem
+    cdef readonly Vocab vocab
+    cdef readonly int64_t entity_vector_length

spacy/kb/kb.pyx

@@ -0,0 +1,108 @@
+# cython: infer_types=True, profile=True
+
+from pathlib import Path
+from typing import Iterable, Tuple, Union
+from cymem.cymem cimport Pool
+
+from .candidate import Candidate
+from ..tokens import Span
+from ..util import SimpleFrozenList
+from ..errors import Errors
+
+
+cdef class KnowledgeBase:
+    """A `KnowledgeBase` instance stores unique identifiers for entities and their textual aliases,
+    to support entity linking of named entities to real-world concepts.
+    This is an abstract class and requires its operations to be implemented.
+
+    DOCS: https://spacy.io/api/kb
+    """
+
+    def __init__(self, vocab: Vocab, entity_vector_length: int):
+        """Create a KnowledgeBase."""
+        # Make sure abstract KB is not instantiated.
+        if self.__class__ == KnowledgeBase:
+            raise TypeError(
+                Errors.E1046.format(cls_name=self.__class__.__name__)
+            )
+
+        self.vocab = vocab
+        self.entity_vector_length = entity_vector_length
+        self.mem = Pool()
+
+    def get_candidates_batch(self, mentions: Iterable[Span]) -> Iterable[Iterable[Candidate]]:
+        """
+        Return candidate entities for specified texts. Each candidate defines the entity, the original alias,
+        and the prior probability of that alias resolving to that entity.
+        If no candidate is found for a given text, an empty list is returned.
+        mentions (Iterable[Span]): Mentions for which to get candidates.
+        RETURNS (Iterable[Iterable[Candidate]]): Identified candidates.
+        """
+        return [self.get_candidates(span) for span in mentions]
+
+    def get_candidates(self, mention: Span) -> Iterable[Candidate]:
+        """
+        Return candidate entities for specified text. Each candidate defines the entity, the original alias,
+        and the prior probability of that alias resolving to that entity.
+        If the no candidate is found for a given text, an empty list is returned.
+        mention (Span): Mention for which to get candidates.
+        RETURNS (Iterable[Candidate]): Identified candidates.
+        """
+        raise NotImplementedError(
+            Errors.E1045.format(parent="KnowledgeBase", method="get_candidates", name=self.__name__)
+        )
+
+    def get_vectors(self, entities: Iterable[str]) -> Iterable[Iterable[float]]:
+        """
+        Return vectors for entities.
+        entity (str): Entity name/ID.
+        RETURNS (Iterable[Iterable[float]]): Vectors for specified entities.
+        """
+        return [self.get_vector(entity) for entity in entities]
+
+    def get_vector(self, str entity) -> Iterable[float]:
+        """
+        Return vector for entity.
+        entity (str): Entity name/ID.
+        RETURNS (Iterable[float]): Vector for specified entity.
+        """
+        raise NotImplementedError(
+            Errors.E1045.format(parent="KnowledgeBase", method="get_vector", name=self.__name__)
+        )
+
+    def to_bytes(self, **kwargs) -> bytes:
+        """Serialize the current state to a binary string.
+        RETURNS (bytes): Current state as binary string.
+        """
+        raise NotImplementedError(
+            Errors.E1045.format(parent="KnowledgeBase", method="to_bytes", name=self.__name__)
+        )
+
+    def from_bytes(self, bytes_data: bytes, *, exclude: Tuple[str] = tuple()):
+        """Load state from a binary string.
+        bytes_data (bytes): KB state.
+        exclude (Tuple[str]): Properties to exclude when restoring KB.
+        """
+        raise NotImplementedError(
+            Errors.E1045.format(parent="KnowledgeBase", method="from_bytes", name=self.__name__)
+        )
+
+    def to_disk(self, path: Union[str, Path], exclude: Iterable[str] = SimpleFrozenList()) -> None:
+        """
+        Write KnowledgeBase content to disk.
+        path (Union[str, Path]): Target file path.
+        exclude (Iterable[str]): List of components to exclude.
+        """
+        raise NotImplementedError(
+            Errors.E1045.format(parent="KnowledgeBase", method="to_disk", name=self.__name__)
+        )
+
+    def from_disk(self, path: Union[str, Path], exclude: Iterable[str] = SimpleFrozenList()) -> None:
+        """
+        Load KnowledgeBase content from disk.
+        path (Union[str, Path]): Target file path.
+        exclude (Iterable[str]): List of components to exclude.
+        """
+        raise NotImplementedError(
+            Errors.E1045.format(parent="KnowledgeBase", method="from_disk", name=self.__name__)
+        )

spacy/kb/kb_in_memory.pxd

@@ -1,14 +1,12 @@
 """Knowledge-base for entity or concept linking."""
-from cymem.cymem cimport Pool
 from preshed.maps cimport PreshMap
 from libcpp.vector cimport vector
 from libc.stdint cimport int32_t, int64_t
 from libc.stdio cimport FILE
 
-from .vocab cimport Vocab
+from ..typedefs cimport hash_t
-from .typedefs cimport hash_t
+from ..structs cimport KBEntryC, AliasC
-from .structs cimport KBEntryC, AliasC
+from .kb cimport KnowledgeBase
-
 
 ctypedef vector[KBEntryC] entry_vec
 ctypedef vector[AliasC] alias_vec
@@ -16,21 +14,7 @@ ctypedef vector[float] float_vec
 ctypedef vector[float_vec] float_matrix
 
 
-# Object used by the Entity Linker that summarizes one entity-alias candidate combination.
+cdef class InMemoryLookupKB(KnowledgeBase):
-cdef class Candidate:
-    cdef readonly KnowledgeBase kb
-    cdef hash_t entity_hash
-    cdef float entity_freq
-    cdef vector[float] entity_vector
-    cdef hash_t alias_hash
-    cdef float prior_prob
-
-
-cdef class KnowledgeBase:
-    cdef Pool mem
-    cdef readonly Vocab vocab
-    cdef int64_t entity_vector_length
-
     # This maps 64bit keys (hash of unique entity string)
     # to 64bit values (position of the _KBEntryC struct in the _entries vector).
     # The PreshMap is pretty space efficient, as it uses open addressing. So

spacy/kb/kb_in_memory.pyx

@@ -1,8 +1,7 @@
 # cython: infer_types=True, profile=True
-from typing import Iterator, Iterable, Callable, Dict, Any
+from typing import Iterable, Callable, Dict, Any, Union
 
 import srsly
-from cymem.cymem cimport Pool
 from preshed.maps cimport PreshMap
 from cpython.exc cimport PyErr_SetFromErrno
 from libc.stdio cimport fopen, fclose, fread, fwrite, feof, fseek
@@ -12,85 +11,28 @@ from libcpp.vector cimport vector
 from pathlib import Path
 import warnings
 
-from .typedefs cimport hash_t
+from ..tokens import Span
-from .errors import Errors, Warnings
+from ..typedefs cimport hash_t
-from . import util
+from ..errors import Errors, Warnings
-from .util import SimpleFrozenList, ensure_path
+from .. import util
-
+from ..util import SimpleFrozenList, ensure_path
-cdef class Candidate:
+from ..vocab cimport Vocab
-    """A `Candidate` object refers to a textual mention (`alias`) that may or may not be resolved
+from .kb cimport KnowledgeBase
-    to a specific `entity` from a Knowledge Base. This will be used as input for the entity linking
+from .candidate import Candidate as Candidate
-    algorithm which will disambiguate the various candidates to the correct one.
-    Each candidate (alias, entity) pair is assigned to a certain prior probability.
-
-    DOCS: https://spacy.io/api/kb/#candidate_init
-    """
-
-    def __init__(self, KnowledgeBase kb, entity_hash, entity_freq, entity_vector, alias_hash, prior_prob):
-        self.kb = kb
-        self.entity_hash = entity_hash
-        self.entity_freq = entity_freq
-        self.entity_vector = entity_vector
-        self.alias_hash = alias_hash
-        self.prior_prob = prior_prob
-
-    @property
-    def entity(self):
-        """RETURNS (uint64): hash of the entity's KB ID/name"""
-        return self.entity_hash
-
-    @property
-    def entity_(self):
-        """RETURNS (str): ID/name of this entity in the KB"""
-        return self.kb.vocab.strings[self.entity_hash]
-
-    @property
-    def alias(self):
-        """RETURNS (uint64): hash of the alias"""
-        return self.alias_hash
-
-    @property
-    def alias_(self):
-        """RETURNS (str): ID of the original alias"""
-        return self.kb.vocab.strings[self.alias_hash]
-
-    @property
-    def entity_freq(self):
-        return self.entity_freq
-
-    @property
-    def entity_vector(self):
-        return self.entity_vector
-
-    @property
-    def prior_prob(self):
-        return self.prior_prob
 
 
-def get_candidates(KnowledgeBase kb, span) -> Iterator[Candidate]:
+cdef class InMemoryLookupKB(KnowledgeBase):
-    """
+    """An `InMemoryLookupKB` instance stores unique identifiers for entities and their textual aliases,
-    Return candidate entities for a given span by using the text of the span as the alias
-    and fetching appropriate entries from the index.
-    This particular function is optimized to work with the built-in KB functionality,
-    but any other custom candidate generation method can be used in combination with the KB as well.
-    """
-    return kb.get_alias_candidates(span.text)
-
-
-cdef class KnowledgeBase:
-    """A `KnowledgeBase` instance stores unique identifiers for entities and their textual aliases,
     to support entity linking of named entities to real-world concepts.
 
-    DOCS: https://spacy.io/api/kb
+    DOCS: https://spacy.io/api/kb_in_memory
     """
 
     def __init__(self, Vocab vocab, entity_vector_length):
-        """Create a KnowledgeBase."""
+        """Create an InMemoryLookupKB."""
-        self.mem = Pool()
+        super().__init__(vocab, entity_vector_length)
-        self.entity_vector_length = entity_vector_length
         self._entry_index = PreshMap()
         self._alias_index = PreshMap()
-        self.vocab = vocab
         self._create_empty_vectors(dummy_hash=self.vocab.strings[""])
 
     def _initialize_entities(self, int64_t nr_entities):
@@ -104,11 +46,6 @@ cdef class KnowledgeBase:
         self._alias_index = PreshMap(nr_aliases + 1)
         self._aliases_table = alias_vec(nr_aliases + 1)
 
-    @property
-    def entity_vector_length(self):
-        """RETURNS (uint64): length of the entity vectors"""
-        return self.entity_vector_length
-
     def __len__(self):
         return self.get_size_entities()
 
@@ -286,7 +223,10 @@ cdef class KnowledgeBase:
             alias_entry.probs = probs
             self._aliases_table[alias_index] = alias_entry
 
-    def get_alias_candidates(self, str alias) -> Iterator[Candidate]:
+    def get_candidates(self, mention: Span) -> Iterable[Candidate]:
+        return self.get_alias_candidates(mention.text)  # type: ignore
+
+    def get_alias_candidates(self, str alias) -> Iterable[Candidate]:
         """
         Return candidate entities for an alias. Each candidate defines the entity, the original alias,
         and the prior probability of that alias resolving to that entity.

spacy/lang/ca/lemmatizer.py

@@ -72,10 +72,10 @@ class CatalanLemmatizer(Lemmatizer):
                         oov_forms.append(form)
         if not forms:
             forms.extend(oov_forms)
-        if not forms and string in lookup_table.keys():
+
-            forms.append(self.lookup_lemmatize(token)[0])
+        # use lookups, and fall back to the token itself
         if not forms:
-            forms.append(string)
+            forms.append(lookup_table.get(string, [string])[0])
         forms = list(dict.fromkeys(forms))
         self.cache[cache_key] = forms
         return forms

spacy/lang/char_classes.py

@@ -280,7 +280,7 @@ _currency = (
 _punct = (
     r"… …… , : ; \! \? ¿ ؟ ¡ \( \) \[ \] \{ \} < > _ # \* & 。 ？ ！ ， 、 ； ： ～ · । ، ۔ ؛ ٪"
 )
-_quotes = r'\' " ” “ ` ‘ ´ ’ ‚ , „ » « 「 」 『 』 （ ） 〔 〕 【 】 《 》 〈 〉'
+_quotes = r'\' " ” “ ` ‘ ´ ’ ‚ , „ » « 「 」 『 』 （ ） 〔 〕 【 】 《 》 〈 〉 〈 〉  ⟦ ⟧'
 _hyphens = "- – — -- --- —— ~"
 
 # Various symbols like dingbats, but also emoji

spacy/lang/fr/lemmatizer.py

@@ -53,11 +53,16 @@ class FrenchLemmatizer(Lemmatizer):
         rules = rules_table.get(univ_pos, [])
         string = string.lower()
         forms = []
+        # first try lookup in table based on upos
         if string in index:
             forms.append(string)
             self.cache[cache_key] = forms
             return forms
+
+        # then add anything in the exceptions table
         forms.extend(exceptions.get(string, []))
+
+        # if nothing found yet, use the rules
         oov_forms = []
         if not forms:
             for old, new in rules:
@@ -69,12 +74,14 @@ class FrenchLemmatizer(Lemmatizer):
                         forms.append(form)
                     else:
                         oov_forms.append(form)
+
+        # if still nothing, add the oov forms from rules
         if not forms:
             forms.extend(oov_forms)
-        if not forms and string in lookup_table.keys():
+
-            forms.append(self.lookup_lemmatize(token)[0])
+        # use lookups, which fall back to the token itself
         if not forms:
-            forms.append(string)
+            forms.append(lookup_table.get(string, [string])[0])
         forms = list(dict.fromkeys(forms))
         self.cache[cache_key] = forms
         return forms

spacy/lang/grc/__init__.py

@@ -1,11 +1,15 @@
 from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS
 from .stop_words import STOP_WORDS
 from .lex_attrs import LEX_ATTRS
+from .punctuation import TOKENIZER_PREFIXES, TOKENIZER_SUFFIXES, TOKENIZER_INFIXES
 from ...language import Language, BaseDefaults
 
 
 class AncientGreekDefaults(BaseDefaults):
     tokenizer_exceptions = TOKENIZER_EXCEPTIONS
+    prefixes = TOKENIZER_PREFIXES
+    suffixes = TOKENIZER_SUFFIXES
+    infixes = TOKENIZER_INFIXES
     lex_attr_getters = LEX_ATTRS
     stop_words = STOP_WORDS
 

spacy/lang/grc/punctuation.py

@@ -0,0 +1,46 @@
+from ..char_classes import LIST_PUNCT, LIST_ELLIPSES, LIST_QUOTES, LIST_CURRENCY
+from ..char_classes import LIST_ICONS, ALPHA_LOWER, ALPHA_UPPER, ALPHA, HYPHENS
+from ..char_classes import CONCAT_QUOTES
+
+_prefixes = (
+    [
+        "†",
+        "⸏",
+    ]
+    + LIST_PUNCT
+    + LIST_ELLIPSES
+    + LIST_QUOTES
+    + LIST_CURRENCY
+    + LIST_ICONS
+)
+
+_suffixes = (
+    LIST_PUNCT
+    + LIST_ELLIPSES
+    + LIST_QUOTES
+    + LIST_ICONS
+    + [
+        "†",
+        "⸎",
+        r"(?<=[\u1F00-\u1FFF\u0370-\u03FF])[\-\.⸏]",
+    ]
+)
+
+_infixes = (
+    LIST_ELLIPSES
+    + LIST_ICONS
+    + [
+        r"(?<=[0-9])[+\-\*^](?=[0-9-])",
+        r"(?<=[{al}{q}])\.(?=[{au}{q}])".format(
+            al=ALPHA_LOWER, au=ALPHA_UPPER, q=CONCAT_QUOTES
+        ),
+        r"(?<=[{a}]),(?=[{a}])".format(a=ALPHA),
+        r"(?<=[{a}0-9])(?:{h})(?=[{a}])".format(a=ALPHA, h=HYPHENS),
+        r"(?<=[{a}0-9])[:<>=/](?=[{a}])".format(a=ALPHA),
+        r"(?<=[\u1F00-\u1FFF\u0370-\u03FF])—",
+    ]
+)
+
+TOKENIZER_PREFIXES = _prefixes
+TOKENIZER_SUFFIXES = _suffixes
+TOKENIZER_INFIXES = _infixes

spacy/lang/ko/punctuation.py

@@ -3,7 +3,7 @@ from ..punctuation import TOKENIZER_INFIXES as BASE_TOKENIZER_INFIXES
 
 
 _infixes = (
-    ["·", "ㆍ", "\(", "\)"]
+    ["·", "ㆍ", r"\(", r"\)"]
     + [r"(?<=[0-9])~(?=[0-9-])"]
     + LIST_QUOTES
     + BASE_TOKENIZER_INFIXES

spacy/lang/la/__init__.py

@@ -0,0 +1,18 @@
+from ...language import Language, BaseDefaults
+from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS
+from .stop_words import STOP_WORDS
+from .lex_attrs import LEX_ATTRS
+
+
+class LatinDefaults(BaseDefaults):
+    tokenizer_exceptions = TOKENIZER_EXCEPTIONS
+    stop_words = STOP_WORDS
+    lex_attr_getters = LEX_ATTRS
+
+
+class Latin(Language):
+    lang = "la"
+    Defaults = LatinDefaults
+
+
+__all__ = ["Latin"]

spacy/lang/la/lex_attrs.py

@@ -0,0 +1,34 @@
+from ...attrs import LIKE_NUM
+import re
+
+# cf. Goyvaerts/Levithan 2009; case-insensitive, allow 4
+roman_numerals_compile = re.compile(
+    r"(?i)^(?=[MDCLXVI])M*(C[MD]|D?C{0,4})(X[CL]|L?X{0,4})(I[XV]|V?I{0,4})$"
+)
+
+_num_words = set(
+    """
+unus una unum duo duae tres tria quattuor quinque sex septem octo novem decem
+""".split()
+)
+
+_ordinal_words = set(
+    """
+primus prima primum secundus secunda secundum tertius tertia tertium
+""".split()
+)
+
+
+def like_num(text):
+    if text.isdigit():
+        return True
+    if roman_numerals_compile.match(text):
+        return True
+    if text.lower() in _num_words:
+        return True
+    if text.lower() in _ordinal_words:
+        return True
+    return False
+
+
+LEX_ATTRS = {LIKE_NUM: like_num}

spacy/lang/la/stop_words.py

@@ -0,0 +1,37 @@
+# Corrected Perseus list, cf. https://wiki.digitalclassicist.org/Stopwords_for_Greek_and_Latin
+
+STOP_WORDS = set(
+    """
+ab ac ad adhuc aliqui aliquis an ante apud at atque aut autem 
+
+cum cur 
+
+de deinde dum 
+
+ego enim ergo es est et etiam etsi ex 
+
+fio 
+
+haud hic 
+
+iam idem igitur ille in infra inter interim ipse is ita 
+
+magis modo mox 
+
+nam ne nec necque neque nisi non nos 
+
+o ob 
+
+per possum post pro 
+
+quae quam quare qui quia quicumque quidem quilibet quis quisnam quisquam quisque quisquis quo quoniam 
+
+sed si sic sive sub sui sum super suus 
+
+tam tamen trans tu tum 
+
+ubi uel uero
+
+vel vero
+""".split()
+)

spacy/lang/la/tokenizer_exceptions.py

@@ -0,0 +1,76 @@
+from ..tokenizer_exceptions import BASE_EXCEPTIONS
+from ...symbols import ORTH
+from ...util import update_exc
+
+
+## TODO: Look into systematically handling u/v
+_exc = {
+    "mecum": [{ORTH: "me"}, {ORTH: "cum"}],
+    "tecum": [{ORTH: "te"}, {ORTH: "cum"}],
+    "nobiscum": [{ORTH: "nobis"}, {ORTH: "cum"}],
+    "vobiscum": [{ORTH: "vobis"}, {ORTH: "cum"}],
+    "uobiscum": [{ORTH: "uobis"}, {ORTH: "cum"}],
+}
+
+for orth in [
+    "A.",
+    "Agr.",
+    "Ap.",
+    "C.",
+    "Cn.",
+    "D.",
+    "F.",
+    "K.",
+    "L.",
+    "M'.",
+    "M.",
+    "Mam.",
+    "N.",
+    "Oct.",
+    "Opet.",
+    "P.",
+    "Paul.",
+    "Post.",
+    "Pro.",
+    "Q.",
+    "S.",
+    "Ser.",
+    "Sert.",
+    "Sex.",
+    "St.",
+    "Sta.",
+    "T.",
+    "Ti.",
+    "V.",
+    "Vol.",
+    "Vop.",
+    "U.",
+    "Uol.",
+    "Uop.",
+    "Ian.",
+    "Febr.",
+    "Mart.",
+    "Apr.",
+    "Mai.",
+    "Iun.",
+    "Iul.",
+    "Aug.",
+    "Sept.",
+    "Oct.",
+    "Nov.",
+    "Nou.",
+    "Dec.",
+    "Non.",
+    "Id.",
+    "A.D.",
+    "Coll.",
+    "Cos.",
+    "Ord.",
+    "Pl.",
+    "S.C.",
+    "Suff.",
+    "Trib.",
+]:
+    _exc[orth] = [{ORTH: orth}]
+
+TOKENIZER_EXCEPTIONS = update_exc(BASE_EXCEPTIONS, _exc)

spacy/lang/lg/__init__.py

@@ -0,0 +1,18 @@
+from .stop_words import STOP_WORDS
+from .lex_attrs import LEX_ATTRS
+from .punctuation import TOKENIZER_INFIXES
+from ...language import Language, BaseDefaults
+
+
+class LugandaDefaults(BaseDefaults):
+    lex_attr_getters = LEX_ATTRS
+    infixes = TOKENIZER_INFIXES
+    stop_words = STOP_WORDS
+
+
+class Luganda(Language):
+    lang = "lg"
+    Defaults = LugandaDefaults
+
+
+__all__ = ["Luganda"]

spacy/lang/lg/examples.py

@@ -0,0 +1,17 @@
+"""
+Example sentences to test spaCy and its language models.
+
+>>> from spacy.lang.lg.examples import sentences
+>>> docs = nlp.pipe(sentences)
+"""
+
+sentences = [
+    "Mpa ebyafaayo ku byalo Nakatu ne Nkajja",
+    "Okuyita Ttembo kitegeeza kugwa ddalu",
+    "Ekifumu kino kyali kya mulimu ki?",
+    "Ekkovu we liyise wayitibwa mukululo",
+    "Akola mulimu ki oguvaamu ssente?",
+    "Emisumaali egikomerera embaawo giyitibwa nninga",
+    "Abooluganda ab’emmamba ababiri",
+    "Ekisaawe ky'ebyenjigiriza kya mugaso nnyo",
+]

spacy/lang/lg/lex_attrs.py

@@ -0,0 +1,95 @@
+from ...attrs import LIKE_NUM
+
+_num_words = [
+    "nnooti",  # Zero
+    "zeero",  # zero
+    "emu",  # one
+    "bbiri",  # two
+    "ssatu",  # three
+    "nnya",  # four
+    "ttaano",  # five
+    "mukaaga",  # six
+    "musanvu",  # seven
+    "munaana",  # eight
+    "mwenda",  # nine
+    "kkumi",  # ten
+    "kkumi n'emu",  # eleven
+    "kkumi na bbiri",  # twelve
+    "kkumi na ssatu",  # thirteen
+    "kkumi na nnya",  # forteen
+    "kkumi na ttaano",  # fifteen
+    "kkumi na mukaaga",  # sixteen
+    "kkumi na musanvu",  # seventeen
+    "kkumi na munaana",  # eighteen
+    "kkumi na mwenda",  # nineteen
+    "amakumi abiri",  # twenty
+    "amakumi asatu",  # thirty
+    "amakumi ana",  # forty
+    "amakumi ataano",  # fifty
+    "nkaaga",  # sixty
+    "nsanvu",  # seventy
+    "kinaana",  # eighty
+    "kyenda",  # ninety
+    "kikumi",  # hundred
+    "lukumi",  # thousand
+    "kakadde",  # million
+    "kawumbi",  # billion
+    "kase",  # trillion
+    "katabalika",  # quadrillion
+    "keesedde",  # gajillion
+    "kafukunya",  # bazillion
+    "ekisooka",  # first
+    "ekyokubiri",  # second
+    "ekyokusatu",  # third
+    "ekyokuna",  # fourth
+    "ekyokutaano",  # fifith
+    "ekyomukaaga",  # sixth
+    "ekyomusanvu",  # seventh
+    "eky'omunaana",  # eighth
+    "ekyomwenda",  # nineth
+    "ekyekkumi",  # tenth
+    "ekyekkumi n'ekimu",  # eleventh
+    "ekyekkumi n'ebibiri",  # twelveth
+    "ekyekkumi n'ebisatu",  # thirteenth
+    "ekyekkumi n'ebina",  # fourteenth
+    "ekyekkumi n'ebitaano",  # fifteenth
+    "ekyekkumi n'omukaaga",  # sixteenth
+    "ekyekkumi n'omusanvu",  # seventeenth
+    "ekyekkumi n'omunaana",  # eigteenth
+    "ekyekkumi n'omwenda",  # nineteenth
+    "ekyamakumi abiri",  # twentieth
+    "ekyamakumi asatu",  # thirtieth
+    "ekyamakumi ana",  # fortieth
+    "ekyamakumi ataano",  # fiftieth
+    "ekyenkaaga",  # sixtieth
+    "ekyensanvu",  # seventieth
+    "ekyekinaana",  # eightieth
+    "ekyekyenda",  # ninetieth
+    "ekyekikumi",  # hundredth
+    "ekyolukumi",  # thousandth
+    "ekyakakadde",  # millionth
+    "ekyakawumbi",  # billionth
+    "ekyakase",  # trillionth
+    "ekyakatabalika",  # quadrillionth
+    "ekyakeesedde",  # gajillionth
+    "ekyakafukunya",  # bazillionth
+]
+
+
+def like_num(text):
+    if text.startswith(("+", "-", "±", "~")):
+        text = text[1:]
+    text = text.replace(",", "").replace(".", "")
+    if text.isdigit():
+        return True
+    if text.count("/") == 1:
+        num, denom = text.split("/")
+        if num.isdigit() and denom.isdigit():
+            return True
+    text_lower = text.lower()
+    if text_lower in _num_words:
+        return True
+    return False
+
+
+LEX_ATTRS = {LIKE_NUM: like_num}

spacy/lang/lg/punctuation.py

@@ -0,0 +1,19 @@
+from ..char_classes import LIST_ELLIPSES, LIST_ICONS, HYPHENS
+from ..char_classes import CONCAT_QUOTES, ALPHA_LOWER, ALPHA_UPPER, ALPHA
+
+_infixes = (
+    LIST_ELLIPSES
+    + LIST_ICONS
+    + [
+        r"(?<=[0-9])[+\-\*^](?=[0-9-])",
+        r"(?<=[{al}{q}])\.(?=[{au}{q}])".format(
+            al=ALPHA_LOWER, au=ALPHA_UPPER, q=CONCAT_QUOTES
+        ),
+        r"(?<=[{a}]),(?=[{a}])".format(a=ALPHA),
+        r"(?<=[{a}0-9])(?:{h})(?=[{a}])".format(a=ALPHA, h=HYPHENS),
+        r"(?<=[{a}0-9])[:<>=/](?=[{a}])".format(a=ALPHA),
+    ]
+)
+
+
+TOKENIZER_INFIXES = _infixes

spacy/lang/lg/stop_words.py

@@ -0,0 +1,19 @@
+STOP_WORDS = set(
+    """
+abadde abalala abamu abangi abava ajja ali alina ani anti ateekeddwa atewamu
+atya awamu aweebwa ayinza ba baali babadde babalina bajja
+bajjanewankubade bali balina bandi bangi bano bateekeddwa baweebwa bayina bebombi beera bibye
+bimu bingi bino bo bokka bonna buli bulijjo bulungi bwabwe bwaffe bwayo bwe bwonna bya byabwe
+byaffe byebimu byonna ddaa ddala ddi e ebimu ebiri ebweruobulungi ebyo edda ejja ekirala ekyo
+endala engeri ennyo era erimu erina ffe ffenna ga gujja gumu gunno guno gwa gwe kaseera kati
+kennyini ki kiki kikino kikye kikyo kino kirungi kki ku kubangabyombi kubangaolwokuba kudda
+kuva kuwa kwegamba kyaffe kye kyekimuoyo kyekyo kyonna leero liryo lwa lwaki lyabwezaabwe
+lyaffe lyange mbadde mingi mpozzi mu mulinaoyina munda mwegyabwe nolwekyo nabadde nabo nandiyagadde
+nandiye nanti naye ne nedda neera nga nnyingi nnyini nnyinza nnyo nti nyinza nze oba ojja okudda
+okugenda okuggyako okutuusa okuva okuwa oli olina oluvannyuma olwekyobuva omuli ono osobola otya
+oyina oyo seetaaga si sinakindi singa talina tayina tebaali tebaalina tebayina terina tetulina
+tetuteekeddwa tewali teyalina teyayina tolina tu tuyina tulina tuyina twafuna twetaaga wa wabula
+wabweru wadde waggulunnina wakati waliwobangi waliyo wandi wange wano wansi weebwa yabadde yaffe
+ye yenna yennyini yina yonna ziba zijja zonna
+""".split()
+)

spacy/lang/nl/syntax_iterators.py

@@ -40,6 +40,7 @@ def noun_chunks(doclike: Union[Doc, Span]) -> Iterator[Tuple[int, int, int]]:
     span_label = doc.vocab.strings.add("NP")
 
     # Only NOUNS and PRONOUNS matter
+    end_span = -1
     for i, word in enumerate(filter(lambda x: x.pos in [PRON, NOUN], doclike)):
         # For NOUNS
         # Pick children from syntactic parse (only those with certain dependencies)
@@ -58,6 +59,7 @@ def noun_chunks(doclike: Union[Doc, Span]) -> Iterator[Tuple[int, int, int]]:
             children_i = [c.i for c in children] + [word.i]
 
             start_span = min(children_i)
+            if start_span >= end_span:
                 end_span = max(children_i) + 1
                 yield start_span, end_span, span_label
 
@@ -65,6 +67,7 @@ def noun_chunks(doclike: Union[Doc, Span]) -> Iterator[Tuple[int, int, int]]:
         elif word.pos == PRON:
             if word.dep in pronoun_deps:
                 start_span = word.i
+                if start_span >= end_span:
                     end_span = word.i + 1
                     yield start_span, end_span, span_label
 

spacy/lang/ru/__init__.py

@@ -28,7 +28,7 @@ class Russian(Language):
     assigns=["token.lemma"],
     default_config={
         "model": None,
-        "mode": "pymorphy2",
+        "mode": "pymorphy3",
         "overwrite": False,
         "scorer": {"@scorers": "spacy.lemmatizer_scorer.v1"},
     },

spacy/lang/ru/lemmatizer.py

@@ -19,11 +19,11 @@ class RussianLemmatizer(Lemmatizer):
         model: Optional[Model],
         name: str = "lemmatizer",
         *,
-        mode: str = "pymorphy2",
+        mode: str = "pymorphy3",
         overwrite: bool = False,
         scorer: Optional[Callable] = lemmatizer_score,
     ) -> None:
-        if mode == "pymorphy2":
+        if mode in {"pymorphy2", "pymorphy2_lookup"}:
             try:
                 from pymorphy2 import MorphAnalyzer
             except ImportError:
@@ -33,6 +33,16 @@ class RussianLemmatizer(Lemmatizer):
                 ) from None
             if getattr(self, "_morph", None) is None:
                 self._morph = MorphAnalyzer()
+        elif mode == "pymorphy3":
+            try:
+                from pymorphy3 import MorphAnalyzer
+            except ImportError:
+                raise ImportError(
+                    "The Russian lemmatizer mode 'pymorphy3' requires the "
+                    "pymorphy3 library. Install it with: pip install pymorphy3"
+                ) from None
+            if getattr(self, "_morph", None) is None:
+                self._morph = MorphAnalyzer()
         super().__init__(
             vocab, model, name, mode=mode, overwrite=overwrite, scorer=scorer
         )
@@ -104,6 +114,9 @@ class RussianLemmatizer(Lemmatizer):
             return [analyses[0].normal_form]
         return [string]
 
+    def pymorphy3_lemmatize(self, token: Token) -> List[str]:
+        return self.pymorphy2_lemmatize(token)
+
 
 def oc2ud(oc_tag: str) -> Tuple[str, Dict[str, str]]:
     gram_map = {

spacy/lang/sl/__init__.py

@@ -1,9 +1,17 @@
+from .lex_attrs import LEX_ATTRS
+from .punctuation import TOKENIZER_INFIXES, TOKENIZER_SUFFIXES, TOKENIZER_PREFIXES
 from .stop_words import STOP_WORDS
+from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS
 from ...language import Language, BaseDefaults
 
 
 class SlovenianDefaults(BaseDefaults):
     stop_words = STOP_WORDS
+    tokenizer_exceptions = TOKENIZER_EXCEPTIONS
+    prefixes = TOKENIZER_PREFIXES
+    infixes = TOKENIZER_INFIXES
+    suffixes = TOKENIZER_SUFFIXES
+    lex_attr_getters = LEX_ATTRS
 
 
 class Slovenian(Language):

spacy/lang/sl/lex_attrs.py

@@ -0,0 +1,145 @@
+from ...attrs import LIKE_NUM
+from ...attrs import IS_CURRENCY
+import unicodedata
+
+
+_num_words = set(
+    """
+	nula ničla nič ena dva tri štiri pet šest sedem osem
+	devet deset enajst dvanajst trinajst štirinajst petnajst
+	šestnajst sedemnajst osemnajst devetnajst dvajset trideset štirideset
+	petdeset šestdest sedemdeset osemdeset devedeset sto tisoč
+	milijon bilijon trilijon kvadrilijon nešteto
+	
+	en eden enega enemu ennem enim enih enima enimi ene eni eno
+	dveh dvema dvem dvoje trije treh trem tremi troje štirje štirih štirim štirimi
+	petih petim petimi šestih šestim šestimi sedmih sedmim sedmimi osmih osmim osmimi
+	devetih devetim devetimi desetih desetim desetimi enajstih enajstim enajstimi
+	dvanajstih dvanajstim dvanajstimi trinajstih trinajstim trinajstimi
+	šestnajstih šestnajstim šestnajstimi petnajstih petnajstim petnajstimi
+	sedemnajstih sedemnajstim sedemnajstimi osemnajstih osemnajstim osemnajstimi
+	devetnajstih devetnajstim devetnajstimi dvajsetih dvajsetim dvajsetimi  
+	""".split()
+)
+
+_ordinal_words = set(
+    """
+	prvi drugi tretji četrti peti šesti sedmi osmi
+	deveti deseti enajsti dvanajsti trinajsti štirinajsti
+	petnajsti šestnajsti sedemnajsti osemnajsti devetnajsti
+	dvajseti trideseti štirideseti petdeseti šestdeseti sedemdeseti
+	osemdeseti devetdeseti stoti tisoči milijonti bilijonti
+	trilijonti kvadrilijonti nešteti
+	
+	prva druga tretja četrta peta šesta sedma osma
+	deveta deseta enajsta dvanajsta trinajsta štirnajsta
+	petnajsta šestnajsta sedemnajsta osemnajsta devetnajsta
+	dvajseta trideseta štirideseta petdeseta šestdeseta sedemdeseta
+	osemdeseta devetdeseta stota tisoča milijonta bilijonta
+	trilijonta kvadrilijonta nešteta
+	
+	prvo drugo tretje četrto peto šestro sedmo osmo
+	deveto deseto enajsto dvanajsto trinajsto štirnajsto
+	petnajsto šestnajsto sedemnajsto osemnajsto devetnajsto
+	dvajseto trideseto štirideseto petdeseto šestdeseto sedemdeseto
+	osemdeseto devetdeseto stoto tisočo milijonto bilijonto
+	trilijonto kvadrilijonto nešteto
+	
+	prvega drugega tretjega četrtega petega šestega sedmega osmega 
+	devega desetega enajstega dvanajstega trinajstega štirnajstega
+	petnajstega šestnajstega sedemnajstega osemnajstega devetnajstega
+	dvajsetega tridesetega štiridesetega petdesetega šestdesetega sedemdesetega
+	osemdesetega devetdesetega stotega tisočega milijontega bilijontega
+	trilijontega kvadrilijontega neštetega
+	
+	prvemu drugemu tretjemu četrtemu petemu šestemu sedmemu osmemu devetemu desetemu 
+	enajstemu dvanajstemu trinajstemu štirnajstemu petnajstemu šestnajstemu sedemnajstemu
+	osemnajstemu devetnajstemu dvajsetemu tridesetemu štiridesetemu petdesetemu šestdesetemu
+	sedemdesetemu osemdesetemu devetdesetemu stotemu tisočemu milijontemu bilijontemu
+	trilijontemu kvadrilijontemu neštetemu
+	
+	prvem drugem tretjem četrtem petem šestem sedmem osmem devetem desetem
+	enajstem dvanajstem trinajstem štirnajstem petnajstem šestnajstem sedemnajstem
+	osemnajstem devetnajstem dvajsetem tridesetem štiridesetem petdesetem šestdesetem
+	sedemdesetem osemdesetem devetdesetem stotem tisočem milijontem bilijontem
+	trilijontem kvadrilijontem neštetem
+	
+	prvim drugim tretjim četrtim petim šestim sedtim osmim devetim desetim
+	enajstim dvanajstim trinajstim štirnajstim petnajstim šestnajstim sedemnajstim
+	osemnajstim devetnajstim dvajsetim tridesetim štiridesetim petdesetim šestdesetim
+	sedemdesetim osemdesetim devetdesetim stotim tisočim milijontim bilijontim
+	trilijontim kvadrilijontim neštetim
+	    
+	prvih drugih tretjih četrthih petih šestih sedmih osmih deveth desetih
+	enajstih dvanajstih trinajstih štirnajstih petnajstih šestnajstih sedemnajstih
+	osemnajstih devetnajstih dvajsetih tridesetih štiridesetih petdesetih šestdesetih
+	sedemdesetih osemdesetih devetdesetih stotih tisočih milijontih bilijontih
+	trilijontih kvadrilijontih nešteth
+	
+	prvima drugima tretjima četrtima petima šestima sedmima osmima devetima desetima
+	enajstima dvanajstima trinajstima štirnajstima petnajstima šestnajstima sedemnajstima
+	osemnajstima devetnajstima dvajsetima tridesetima štiridesetima petdesetima šestdesetima
+	sedemdesetima osemdesetima devetdesetima stotima tisočima milijontima bilijontima
+	trilijontima kvadrilijontima neštetima
+	
+	prve druge četrte pete šeste sedme osme devete desete
+	enajste dvanajste trinajste štirnajste petnajste šestnajste sedemnajste
+	osemnajste devetnajste dvajsete tridesete štiridesete petdesete šestdesete
+	sedemdesete osemdesete devetdesete stote tisoče milijonte bilijonte 
+	trilijonte kvadrilijonte neštete
+	
+	prvimi drugimi tretjimi četrtimi petimi šestimi sedtimi osmimi devetimi desetimi
+	enajstimi dvanajstimi trinajstimi štirnajstimi petnajstimi šestnajstimi sedemnajstimi
+	osemnajstimi devetnajstimi dvajsetimi tridesetimi štiridesetimi petdesetimi šestdesetimi
+	sedemdesetimi osemdesetimi devetdesetimi stotimi tisočimi milijontimi bilijontimi
+	trilijontimi kvadrilijontimi neštetimi
+	""".split()
+)
+
+_currency_words = set(
+    """
+	evro evra evru evrom evrov evroma evrih evrom evre evri evr eur
+	cent centa centu cenom centov centoma centih centom cente centi
+	dolar dolarja dolarji dolarju dolarjem dolarjev dolarjema dolarjih dolarje usd
+	tolar tolarja tolarji tolarju tolarjem tolarjev tolarjema tolarjih tolarje tol
+	dinar dinarja dinarji dinarju dinarjem dinarjev dinarjema dinarjih dinarje din
+	funt funta funti funtu funtom funtov funtoma funtih funte gpb
+	forint forinta forinti forintu forintom forintov forintoma forintih forinte
+	zlot zlota zloti zlotu zlotom zlotov zlotoma zlotih zlote 
+	rupij rupija rupiji rupiju rupijem rupijev rupijema rupijih rupije
+	jen jena jeni jenu jenom jenov jenoma jenih jene
+	kuna kuni kune kuno kun kunama kunah kunam kunami
+	marka marki marke markama markah markami 
+	""".split()
+)
+
+
+def like_num(text):
+    if text.startswith(("+", "-", "±", "~")):
+        text = text[1:]
+    text = text.replace(",", "").replace(".", "")
+    if text.isdigit():
+        return True
+    if text.count("/") == 1:
+        num, denom = text.split("/")
+        if num.isdigit() and denom.isdigit():
+            return True
+    text_lower = text.lower()
+    if text_lower in _num_words:
+        return True
+    if text_lower in _ordinal_words:
+        return True
+    return False
+
+
+def is_currency(text):
+    text_lower = text.lower()
+    if text in _currency_words:
+        return True
+    for char in text:
+        if unicodedata.category(char) != "Sc":
+            return False
+    return True
+
+
+LEX_ATTRS = {LIKE_NUM: like_num, IS_CURRENCY: is_currency}

spacy/lang/sl/punctuation.py

@@ -0,0 +1,84 @@
+from ..char_classes import (
+    LIST_ELLIPSES,
+    LIST_ICONS,
+    HYPHENS,
+    LIST_PUNCT,
+    LIST_QUOTES,
+    CURRENCY,
+    UNITS,
+    PUNCT,
+    LIST_CURRENCY,
+    CONCAT_QUOTES,
+)
+from ..char_classes import CONCAT_QUOTES, ALPHA_LOWER, ALPHA_UPPER, ALPHA
+from ..char_classes import merge_chars
+from ..punctuation import TOKENIZER_PREFIXES as BASE_TOKENIZER_PREFIXES
+
+
+INCLUDE_SPECIAL = ["\\+", "\\/", "\\•", "\\¯", "\\=", "\\×"] + HYPHENS.split("|")
+
+_prefixes = INCLUDE_SPECIAL + BASE_TOKENIZER_PREFIXES
+
+_suffixes = (
+    INCLUDE_SPECIAL
+    + LIST_PUNCT
+    + LIST_ELLIPSES
+    + LIST_QUOTES
+    + LIST_ICONS
+    + [
+        r"(?<=°[FfCcKk])\.",
+        r"(?<=[0-9])(?:{c})".format(c=CURRENCY),
+        r"(?<=[0-9])(?:{u})".format(u=UNITS),
+        r"(?<=[{al}{e}{p}(?:{q})])\.".format(
+            al=ALPHA_LOWER, e=r"%²\-\+", q=CONCAT_QUOTES, p=PUNCT
+        ),
+        r"(?<=[{au}][{au}])\.".format(au=ALPHA_UPPER),
+        # split initials like J.K. Rowling
+        r"(?<=[A-Z]\.)(?:[A-Z].)",
+    ]
+)
+
+# a list of all suffixes following a hyphen that are shouldn't split (eg. BTC-jev)
+# source: Obeliks tokenizer - https://github.com/clarinsi/obeliks/blob/master/obeliks/res/TokRulesPart1.txt
+CONCAT_QUOTES = CONCAT_QUOTES.replace("'", "")
+HYPHENS_PERMITTED = (
+    "((a)|(evemu)|(evskega)|(i)|(jevega)|(jevska)|(jevskimi)|(jinemu)|(oma)|(ovim)|"
+    "(ovski)|(e)|(evi)|(evskem)|(ih)|(jevem)|(jevske)|(jevsko)|(jini)|(ov)|(ovima)|"
+    "(ovskih)|(em)|(evih)|(evskemu)|(ja)|(jevemu)|(jevskega)|(ji)|(jinih)|(ova)|"
+    "(ovimi)|(ovskim)|(ema)|(evim)|(evski)|(je)|(jevi)|(jevskem)|(jih)|(jinim)|"
+    "(ove)|(ovo)|(ovskima)|(ev)|(evima)|(evskih)|(jem)|(jevih)|(jevskemu)|(jin)|"
+    "(jinima)|(ovega)|(ovska)|(ovskimi)|(eva)|(evimi)|(evskim)|(jema)|(jevim)|"
+    "(jevski)|(jina)|(jinimi)|(ovem)|(ovske)|(ovsko)|(eve)|(evo)|(evskima)|(jev)|"
+    "(jevima)|(jevskih)|(jine)|(jino)|(ovemu)|(ovskega)|(u)|(evega)|(evska)|"
+    "(evskimi)|(jeva)|(jevimi)|(jevskim)|(jinega)|(ju)|(ovi)|(ovskem)|(evem)|"
+    "(evske)|(evsko)|(jeve)|(jevo)|(jevskima)|(jinem)|(om)|(ovih)|(ovskemu)|"
+    "(ovec)|(ovca)|(ovcu)|(ovcem)|(ovcev)|(ovcema)|(ovcih)|(ovci)|(ovce)|(ovcimi)|"
+    "(evec)|(evca)|(evcu)|(evcem)|(evcev)|(evcema)|(evcih)|(evci)|(evce)|(evcimi)|"
+    "(jevec)|(jevca)|(jevcu)|(jevcem)|(jevcev)|(jevcema)|(jevcih)|(jevci)|(jevce)|"
+    "(jevcimi)|(ovka)|(ovke)|(ovki)|(ovko)|(ovk)|(ovkama)|(ovkah)|(ovkam)|(ovkami)|"
+    "(evka)|(evke)|(evki)|(evko)|(evk)|(evkama)|(evkah)|(evkam)|(evkami)|(jevka)|"
+    "(jevke)|(jevki)|(jevko)|(jevk)|(jevkama)|(jevkah)|(jevkam)|(jevkami)|(timi)|"
+    "(im)|(ima)|(a)|(imi)|(e)|(o)|(ega)|(ti)|(em)|(tih)|(emu)|(tim)|(i)|(tima)|"
+    "(ih)|(ta)|(te)|(to)|(tega)|(tem)|(temu))"
+)
+
+_infixes = (
+    LIST_ELLIPSES
+    + LIST_ICONS
+    + [
+        r"(?<=[0-9])[+\-\*^](?=[0-9-])",
+        r"(?<=[{al}{q}])\.(?=[{au}{q}])".format(
+            al=ALPHA_LOWER, au=ALPHA_UPPER, q=CONCAT_QUOTES
+        ),
+        r"(?<=[{a}]),(?=[{a}])".format(a=ALPHA),
+        r"(?<=[{a}0-9])(?:{h})(?!{hp}$)(?=[{a}])".format(
+            a=ALPHA, h=HYPHENS, hp=HYPHENS_PERMITTED
+        ),
+        r"(?<=[{a}0-9])[:<>=/](?=[{a}])".format(a=ALPHA),
+    ]
+)
+
+
+TOKENIZER_PREFIXES = _prefixes
+TOKENIZER_SUFFIXES = _suffixes
+TOKENIZER_INFIXES = _infixes

spacy/lang/sl/stop_words.py

@@ -1,326 +1,84 @@
 # Source: https://github.com/stopwords-iso/stopwords-sl
-# Removed various words that are not normally considered stop words, such as months.
 
 STOP_WORDS = set(
     """
-a
+a ali 
-ali
+
-b
+b bi bil bila bile bili bilo biti blizu bo bodo bojo bolj bom bomo 
-bi
+boste bova boš brez
-bil
+
-bila
+c cel cela celi celo
-bile
+
-bili
+č če često četrta četrtek četrti četrto čez čigav
-bilo
+
-biti
+d da daleč dan danes datum deset deseta deseti deseto devet
-blizu
+deveta deveti deveto do dober dobra dobri dobro dokler dol dolg
-bo
+dolga dolgi dovolj drug druga drugi drugo dva dve
-bodo
+
-bolj
+e eden en ena ene eni enkrat eno etc.
-bom
+
-bomo
-boste
-bova
-boš
-brez
-c
-cel
-cela
-celi
-celo
-d
-da
-daleč
-dan
-danes
-do
-dober
-dobra
-dobri
-dobro
-dokler
-dol
-dovolj
-e
-eden
-en
-ena
-ene
-eni
-enkrat
-eno
-etc.
 f
-g
+
-g.
+g g. ga ga. gor gospa gospod 
-ga
+
-ga.
+h halo 
-gor
+
-gospa
+i idr. ii iii in iv ix iz
-gospod
+
-h
+j jaz je ji jih jim jo jutri
-halo
+
-i
+k kadarkoli kaj kajti kako kakor kamor kamorkoli kar karkoli
-idr.
+katerikoli kdaj kdo kdorkoli ker ki kje kjer kjerkoli
-ii
+ko koder koderkoli koga komu kot kratek kratka kratke kratki
-iii
+
-in
+l lahka lahke lahki lahko le lep lepa lepe lepi lepo leto
-iv
+
-ix
+m majhen majhna majhni malce malo manj me med medtem mene
-iz
+mesec mi midva midve mnogo moj moja moje mora morajo moram
-j
+moramo morate moraš morem mu
-jaz
+
-je
+n na nad naj najina najino najmanj naju največ nam narobe
-ji
+nas nato nazaj naš naša naše ne nedavno nedelja nek neka
-jih
+nekaj nekatere nekateri nekatero nekdo neke nekega neki
-jim
+nekje neko nekoga nekoč ni nikamor nikdar nikjer nikoli
-jo
+nič nje njega njegov njegova njegovo njej njemu njen
-k
+njena njeno nji njih njihov njihova njihovo njiju njim
-kadarkoli
+njo njun njuna njuno no nocoj npr.
-kaj
+
-kajti
+o ob oba obe oboje od odprt odprta odprti okoli on
-kako
+onadva one oni onidve osem osma osmi osmo oz.
-kakor
+
-kamor
+p pa pet peta petek peti peto po pod pogosto poleg poln
-kamorkoli
+polna polni polno ponavadi ponedeljek ponovno potem
-kar
+povsod pozdravljen pozdravljeni prav prava prave pravi
-karkoli
+pravo prazen prazna prazno prbl. precej pred prej preko
-katerikoli
+pri pribl. približno primer pripravljen pripravljena
-kdaj
+pripravljeni proti prva prvi prvo
-kdo
+
-kdorkoli
+r ravno redko res reč
-ker
+
-ki
+s saj sam sama same sami samo se sebe sebi sedaj sedem
-kje
+sedma sedmi sedmo sem seveda si sicer skoraj skozi slab sm
-kjer
+so sobota spet sreda srednja srednji sta ste stran stvar sva
-kjerkoli
+
-ko
+š šest šesta šesti šesto štiri 
-koderkoli
+
-koga
+t ta tak taka take taki tako takoj tam te tebe tebi tega
-komu
+težak težka težki težko ti tista tiste tisti tisto tj.
-kot
+tja to toda torek tretja tretje tretji tri tu tudi tukaj
-l
+tvoj tvoja tvoje
-le
+
-lep
-lepa
-lepe
-lepi
-lepo
-m
-manj
-me
-med
-medtem
-mene
-mi
-midva
-midve
-mnogo
-moj
-moja
-moje
-mora
-morajo
-moram
-moramo
-morate
-moraš
-morem
-mu
-n
-na
-nad
-naj
-najina
-najino
-najmanj
-naju
-največ
-nam
-nas
-nato
-nazaj
-naš
-naša
-naše
-ne
-nedavno
-nek
-neka
-nekaj
-nekatere
-nekateri
-nekatero
-nekdo
-neke
-nekega
-neki
-nekje
-neko
-nekoga
-nekoč
-ni
-nikamor
-nikdar
-nikjer
-nikoli
-nič
-nje
-njega
-njegov
-njegova
-njegovo
-njej
-njemu
-njen
-njena
-njeno
-nji
-njih
-njihov
-njihova
-njihovo
-njiju
-njim
-njo
-njun
-njuna
-njuno
-no
-nocoj
-npr.
-o
-ob
-oba
-obe
-oboje
-od
-okoli
-on
-onadva
-one
-oni
-onidve
-oz.
-p
-pa
-po
-pod
-pogosto
-poleg
-ponavadi
-ponovno
-potem
-povsod
-prbl.
-precej
-pred
-prej
-preko
-pri
-pribl.
-približno
-proti
-r
-redko
-res
-s
-saj
-sam
-sama
-same
-sami
-samo
-se
-sebe
-sebi
-sedaj
-sem
-seveda
-si
-sicer
-skoraj
-skozi
-smo
-so
-spet
-sta
-ste
-sva
-t
-ta
-tak
-taka
-take
-taki
-tako
-takoj
-tam
-te
-tebe
-tebi
-tega
-ti
-tista
-tiste
-tisti
-tisto
-tj.
-tja
-to
-toda
-tu
-tudi
-tukaj
-tvoj
-tvoja
-tvoje
 u
-v
+
-vaju
+v vaju vam vas vaš vaša vaše ve vedno velik velika veliki
-vam
+veliko vendar ves več vi vidva vii viii visok visoka visoke
-vas
+visoki vsa vsaj vsak vsaka vsakdo vsake vsaki vsakomur vse
-vaš
+vsega vsi vso včasih včeraj 
-vaša
+
-vaše
-ve
-vedno
-vendar
-ves
-več
-vi
-vidva
-vii
-viii
-vsa
-vsaj
-vsak
-vsaka
-vsakdo
-vsake
-vsaki
-vsakomur
-vse
-vsega
-vsi
-vso
-včasih
 x 
-z
+
-za
+z za zadaj zadnji zakaj zaprta zaprti zaprto zdaj zelo zunaj
-zadaj
+
-zadnji
+ž že
-zakaj
-zdaj
-zelo
-zunaj
-č
-če
-često
-čez
-čigav
-š
-ž
-že
 """.split()
 )

spacy/lang/sl/tokenizer_exceptions.py

@@ -0,0 +1,272 @@
+from typing import Dict, List
+from ..tokenizer_exceptions import BASE_EXCEPTIONS
+from ...symbols import ORTH, NORM
+from ...util import update_exc
+
+_exc: Dict[str, List[Dict]] = {}
+
+_other_exc = {
+    "t.i.": [{ORTH: "t.", NORM: "tako"}, {ORTH: "i.", NORM: "imenovano"}],
+    "t.j.": [{ORTH: "t.", NORM: "to"}, {ORTH: "j.", NORM: "je"}],
+    "T.j.": [{ORTH: "T.", NORM: "to"}, {ORTH: "j.", NORM: "je"}],
+    "d.o.o.": [
+        {ORTH: "d.", NORM: "družba"},
+        {ORTH: "o.", NORM: "omejeno"},
+        {ORTH: "o.", NORM: "odgovornostjo"},
+    ],
+    "D.O.O.": [
+        {ORTH: "D.", NORM: "družba"},
+        {ORTH: "O.", NORM: "omejeno"},
+        {ORTH: "O.", NORM: "odgovornostjo"},
+    ],
+    "d.n.o.": [
+        {ORTH: "d.", NORM: "družba"},
+        {ORTH: "n.", NORM: "neomejeno"},
+        {ORTH: "o.", NORM: "odgovornostjo"},
+    ],
+    "D.N.O.": [
+        {ORTH: "D.", NORM: "družba"},
+        {ORTH: "N.", NORM: "neomejeno"},
+        {ORTH: "O.", NORM: "odgovornostjo"},
+    ],
+    "d.d.": [{ORTH: "d.", NORM: "delniška"}, {ORTH: "d.", NORM: "družba"}],
+    "D.D.": [{ORTH: "D.", NORM: "delniška"}, {ORTH: "D.", NORM: "družba"}],
+    "s.p.": [{ORTH: "s.", NORM: "samostojni"}, {ORTH: "p.", NORM: "podjetnik"}],
+    "S.P.": [{ORTH: "S.", NORM: "samostojni"}, {ORTH: "P.", NORM: "podjetnik"}],
+    "l.r.": [{ORTH: "l.", NORM: "lastno"}, {ORTH: "r.", NORM: "ročno"}],
+    "le-te": [{ORTH: "le"}, {ORTH: "-"}, {ORTH: "te"}],
+    "Le-te": [{ORTH: "Le"}, {ORTH: "-"}, {ORTH: "te"}],
+    "le-ti": [{ORTH: "le"}, {ORTH: "-"}, {ORTH: "ti"}],
+    "Le-ti": [{ORTH: "Le"}, {ORTH: "-"}, {ORTH: "ti"}],
+    "le-to": [{ORTH: "le"}, {ORTH: "-"}, {ORTH: "to"}],
+    "Le-to": [{ORTH: "Le"}, {ORTH: "-"}, {ORTH: "to"}],
+    "le-ta": [{ORTH: "le"}, {ORTH: "-"}, {ORTH: "ta"}],
+    "Le-ta": [{ORTH: "Le"}, {ORTH: "-"}, {ORTH: "ta"}],
+    "le-tega": [{ORTH: "le"}, {ORTH: "-"}, {ORTH: "tega"}],
+    "Le-tega": [{ORTH: "Le"}, {ORTH: "-"}, {ORTH: "tega"}],
+}
+
+_exc.update(_other_exc)
+
+
+for exc_data in [
+    {ORTH: "adm.", NORM: "administracija"},
+    {ORTH: "aer.", NORM: "aeronavtika"},
+    {ORTH: "agr.", NORM: "agronomija"},
+    {ORTH: "amer.", NORM: "ameriško"},
+    {ORTH: "anat.", NORM: "anatomija"},
+    {ORTH: "angl.", NORM: "angleški"},
+    {ORTH: "ant.", NORM: "antonim"},
+    {ORTH: "antr.", NORM: "antropologija"},
+    {ORTH: "apr.", NORM: "april"},
+    {ORTH: "arab.", NORM: "arabsko"},
+    {ORTH: "arheol.", NORM: "arheologija"},
+    {ORTH: "arhit.", NORM: "arhitektura"},
+    {ORTH: "avg.", NORM: "avgust"},
+    {ORTH: "avstr.", NORM: "avstrijsko"},
+    {ORTH: "avt.", NORM: "avtomobilizem"},
+    {ORTH: "bibl.", NORM: "biblijsko"},
+    {ORTH: "biokem.", NORM: "biokemija"},
+    {ORTH: "biol.", NORM: "biologija"},
+    {ORTH: "bolg.", NORM: "bolgarski"},
+    {ORTH: "bot.", NORM: "botanika"},
+    {ORTH: "cit.", NORM: "citat"},
+    {ORTH: "daj.", NORM: "dajalnik"},
+    {ORTH: "del.", NORM: "deležnik"},
+    {ORTH: "ed.", NORM: "ednina"},
+    {ORTH: "etn.", NORM: "etnografija"},
+    {ORTH: "farm.", NORM: "farmacija"},
+    {ORTH: "filat.", NORM: "filatelija"},
+    {ORTH: "filoz.", NORM: "filozofija"},
+    {ORTH: "fin.", NORM: "finančništvo"},
+    {ORTH: "fiz.", NORM: "fizika"},
+    {ORTH: "fot.", NORM: "fotografija"},
+    {ORTH: "fr.", NORM: "francoski"},
+    {ORTH: "friz.", NORM: "frizerstvo"},
+    {ORTH: "gastr.", NORM: "gastronomija"},
+    {ORTH: "geogr.", NORM: "geografija"},
+    {ORTH: "geol.", NORM: "geologija"},
+    {ORTH: "geom.", NORM: "geometrija"},
+    {ORTH: "germ.", NORM: "germanski"},
+    {ORTH: "gl.", NORM: "glej"},
+    {ORTH: "glag.", NORM: "glagolski"},
+    {ORTH: "glasb.", NORM: "glasba"},
+    {ORTH: "gled.", NORM: "gledališče"},
+    {ORTH: "gost.", NORM: "gostinstvo"},
+    {ORTH: "gozd.", NORM: "gozdarstvo"},
+    {ORTH: "gr.", NORM: "grški"},
+    {ORTH: "grad.", NORM: "gradbeništvo"},
+    {ORTH: "hebr.", NORM: "hebrejsko"},
+    {ORTH: "hrv.", NORM: "hrvaško"},
+    {ORTH: "ide.", NORM: "indoevropsko"},
+    {ORTH: "igr.", NORM: "igre"},
+    {ORTH: "im.", NORM: "imenovalnik"},
+    {ORTH: "iron.", NORM: "ironično"},
+    {ORTH: "it.", NORM: "italijanski"},
+    {ORTH: "itd.", NORM: "in tako dalje"},
+    {ORTH: "itn.", NORM: "in tako naprej"},
+    {ORTH: "ipd.", NORM: "in podobno"},
+    {ORTH: "jap.", NORM: "japonsko"},
+    {ORTH: "jul.", NORM: "julij"},
+    {ORTH: "jun.", NORM: "junij"},
+    {ORTH: "kit.", NORM: "kitajsko"},
+    {ORTH: "knj.", NORM: "knjižno"},
+    {ORTH: "knjiž.", NORM: "knjižno"},
+    {ORTH: "kor.", NORM: "koreografija"},
+    {ORTH: "lat.", NORM: "latinski"},
+    {ORTH: "les.", NORM: "lesna stroka"},
+    {ORTH: "lingv.", NORM: "lingvistika"},
+    {ORTH: "lit.", NORM: "literarni"},
+    {ORTH: "ljubk.", NORM: "ljubkovalno"},
+    {ORTH: "lov.", NORM: "lovstvo"},
+    {ORTH: "m.", NORM: "moški"},
+    {ORTH: "mak.", NORM: "makedonski"},
+    {ORTH: "mar.", NORM: "marec"},
+    {ORTH: "mat.", NORM: "matematika"},
+    {ORTH: "med.", NORM: "medicina"},
+    {ORTH: "meh.", NORM: "mehiško"},
+    {ORTH: "mest.", NORM: "mestnik"},
+    {ORTH: "mdr.", NORM: "med drugim"},
+    {ORTH: "min.", NORM: "mineralogija"},
+    {ORTH: "mitol.", NORM: "mitologija"},
+    {ORTH: "mn.", NORM: "množina"},
+    {ORTH: "mont.", NORM: "montanistika"},
+    {ORTH: "muz.", NORM: "muzikologija"},
+    {ORTH: "nam.", NORM: "namenilnik"},
+    {ORTH: "nar.", NORM: "narečno"},
+    {ORTH: "nav.", NORM: "navadno"},
+    {ORTH: "nedol.", NORM: "nedoločnik"},
+    {ORTH: "nedov.", NORM: "nedovršni"},
+    {ORTH: "neprav.", NORM: "nepravilno"},
+    {ORTH: "nepreh.", NORM: "neprehodno"},
+    {ORTH: "neskl.", NORM: "nesklonljiv(o)"},
+    {ORTH: "nestrok.", NORM: "nestrokovno"},
+    {ORTH: "num.", NORM: "numizmatika"},
+    {ORTH: "npr.", NORM: "na primer"},
+    {ORTH: "obrt.", NORM: "obrtništvo"},
+    {ORTH: "okt.", NORM: "oktober"},
+    {ORTH: "or.", NORM: "orodnik"},
+    {ORTH: "os.", NORM: "oseba"},
+    {ORTH: "otr.", NORM: "otroško"},
+    {ORTH: "oz.", NORM: "oziroma"},
+    {ORTH: "pal.", NORM: "paleontologija"},
+    {ORTH: "papir.", NORM: "papirništvo"},
+    {ORTH: "ped.", NORM: "pedagogika"},
+    {ORTH: "pisar.", NORM: "pisarniško"},
+    {ORTH: "pog.", NORM: "pogovorno"},
+    {ORTH: "polit.", NORM: "politika"},
+    {ORTH: "polj.", NORM: "poljsko"},
+    {ORTH: "poljud.", NORM: "poljudno"},
+    {ORTH: "preg.", NORM: "pregovor"},
+    {ORTH: "preh.", NORM: "prehodno"},
+    {ORTH: "pren.", NORM: "preneseno"},
+    {ORTH: "prid.", NORM: "pridevnik"},
+    {ORTH: "prim.", NORM: "primerjaj"},
+    {ORTH: "prisl.", NORM: "prislov"},
+    {ORTH: "psih.", NORM: "psihologija"},
+    {ORTH: "psiht.", NORM: "psihiatrija"},
+    {ORTH: "rad.", NORM: "radiotehnika"},
+    {ORTH: "rač.", NORM: "računalništvo"},
+    {ORTH: "rib.", NORM: "ribištvo"},
+    {ORTH: "rod.", NORM: "rodilnik"},
+    {ORTH: "rus.", NORM: "rusko"},
+    {ORTH: "s.", NORM: "srednji"},
+    {ORTH: "sam.", NORM: "samostalniški"},
+    {ORTH: "sed.", NORM: "sedanjik"},
+    {ORTH: "sep.", NORM: "september"},
+    {ORTH: "slabš.", NORM: "slabšalno"},
+    {ORTH: "slovan.", NORM: "slovansko"},
+    {ORTH: "slovaš.", NORM: "slovaško"},
+    {ORTH: "srb.", NORM: "srbsko"},
+    {ORTH: "star.", NORM: "starinsko"},
+    {ORTH: "stil.", NORM: "stilno"},
+    {ORTH: "sv.", NORM: "svet(i)"},
+    {ORTH: "teh.", NORM: "tehnika"},
+    {ORTH: "tisk.", NORM: "tiskarstvo"},
+    {ORTH: "tj.", NORM: "to je"},
+    {ORTH: "tož.", NORM: "tožilnik"},
+    {ORTH: "trg.", NORM: "trgovina"},
+    {ORTH: "ukr.", NORM: "ukrajinski"},
+    {ORTH: "um.", NORM: "umetnost"},
+    {ORTH: "vel.", NORM: "velelnik"},
+    {ORTH: "vet.", NORM: "veterina"},
+    {ORTH: "vez.", NORM: "veznik"},
+    {ORTH: "vn.", NORM: "visokonemško"},
+    {ORTH: "voj.", NORM: "vojska"},
+    {ORTH: "vrtn.", NORM: "vrtnarstvo"},
+    {ORTH: "vulg.", NORM: "vulgarno"},
+    {ORTH: "vznes.", NORM: "vzneseno"},
+    {ORTH: "zal.", NORM: "založništvo"},
+    {ORTH: "zastar.", NORM: "zastarelo"},
+    {ORTH: "zgod.", NORM: "zgodovina"},
+    {ORTH: "zool.", NORM: "zoologija"},
+    {ORTH: "čeb.", NORM: "čebelarstvo"},
+    {ORTH: "češ.", NORM: "češki"},
+    {ORTH: "člov.", NORM: "človeškost"},
+    {ORTH: "šah.", NORM: "šahovski"},
+    {ORTH: "šalj.", NORM: "šaljivo"},
+    {ORTH: "šp.", NORM: "španski"},
+    {ORTH: "špan.", NORM: "špansko"},
+    {ORTH: "šport.", NORM: "športni"},
+    {ORTH: "štev.", NORM: "števnik"},
+    {ORTH: "šved.", NORM: "švedsko"},
+    {ORTH: "švic.", NORM: "švicarsko"},
+    {ORTH: "ž.", NORM: "ženski"},
+    {ORTH: "žarg.", NORM: "žargonsko"},
+    {ORTH: "žel.", NORM: "železnica"},
+    {ORTH: "živ.", NORM: "živost"},
+]:
+    _exc[exc_data[ORTH]] = [exc_data]
+
+
+abbrv = """
+Co. Ch. DIPL. DR. Dr. Ev. Inc. Jr. Kr. Mag. M. MR. Mr. Mt. Murr. Npr. OZ. 
+Opr. Osn. Prim. Roj. ST. Sim. Sp. Sred. St. Sv. Škofl. Tel. UR. Zb. 
+a. aa. ab. abc. abit. abl. abs. abt. acc. accel. add. adj. adv. aet. afr. akad. al. alban. all. alleg. 
+alp. alt. alter. alžir. am. an. andr. ang. anh. anon. ans. antrop. apoc. app. approx. apt. ar. arc. arch. 
+arh. arr. as. asist. assist. assoc. asst. astr. attn. aug. avstral. az. b. bab. bal. bbl. bd. belg. bioinf. 
+biomed. bk. bl. bn. borg. bp. br. braz. brit. bros. broš. bt. bu. c. ca. cal. can. cand. cantab. cap. capt.
+cat. cath. cc. cca. cd. cdr. cdre. cent. cerkv. cert. cf. cfr. ch. chap. chem. chr. chs. cic. circ. civ. cl.
+cm. cmd. cnr. co. cod. col. coll. colo. com. comp. con. conc. cond. conn. cons. cont. coop. corr. cost. cp.
+cpl. cr. crd. cres. cresc. ct. cu. d. dan. dat. davč. ddr. dec. ded. def. dem. dent. dept. dia. dip. dipl. 
+dir. disp. diss. div. do. doc. dok. dol. doo. dop. dott. dr. dram. druž. družb. drž. dt. duh. dur. dvr. dwt. e.
+ea. ecc. eccl. eccles. econ. edn. egipt. egr. ekon. eksp. el. em. enc. eng. eo. ep. err. esp. esq. est.
+et. etc. etnogr. etnol. ev. evfem. evr. ex. exc. excl. exp. expl. ext. exx. f. fa. facs. fak. faks. fas.
+fasc. fco. fcp. feb. febr. fec. fed. fem. ff. fff. fid. fig. fil. film. fiziol. fiziot. flam. fm. fo. fol. folk.
+frag. fran. franc. fsc. g. ga. gal. gdč. ge. gen. geod. geog. geotehnol. gg. gimn. glas. glav. gnr. go. gor.
+gosp. gp. graf. gram. gren. grš. gs. h. hab. hf. hist. ho. hort. i. ia. ib. ibid. id. idr. idridr. ill. imen.
+imp. impf. impr. in. inc. incl. ind. indus. inf. inform. ing. init. ins. int. inv. inšp. inštr. inž. is. islam.
+ist. ital. iur. iz. izbr. izd. izg. izgr. izr. izv. j. jak. jam. jan. jav. je. jez. jr. jsl. jud. jug.
+jugoslovan. jur. juž. jv. jz. k. kal. kan. kand. kat. kdo. kem. kip. kmet. kol. kom. komp. konf. kont. kost. kov. 
+kp. kpfw. kr. kraj. krat. kub. kult. kv. kval. l. la. lab. lb. ld. let. lib. lik. litt. lj. ljud. ll. loc. log. 
+loč. lt. ma. madž. mag. manag. manjš. masc. mass. mater. max. maxmax. mb. md. mech. medic. medij. medn. 
+mehč. mem. menedž. mes. mess. metal. meteor. meteorol. mex. mi. mikr. mil. minn. mio. misc. miss. mit. mk. 
+mkt. ml. mlad. mlle. mlr. mm. mme. množ. mo. moj. moš. možn. mr. mrd. mrs. ms. msc. msgr. mt. murr. mus. mut. 
+n. na. nad. nadalj. nadom. nagl. nakl. namer. nan. naniz. nasl. nat. navt. nač. ned. nem. nik. nizoz. nm. nn. 
+no. nom. norv. notr. nov. novogr. ns. o. ob. obd. obj. oblač. obl. oblik. obr. obraz. obs. obst. obt. obč. oc. 
+oct. od. odd. odg. odn. odst. odv. oec. off. ok. okla. okr. ont. oo. op. opis. opp. opr. orch. ord. ore. oreg. 
+org. orient. orig. ork. ort. oseb. osn. ot. ozir. ošk. p. pag. par. para. parc. parl. part. past. pat. pdk. 
+pen. perf. pert. perz. pesn. pet. pev. pf. pfc. ph. pharm. phil. pis. pl. po. pod. podr. podaljš. pogl. pogoj. pojm. 
+pok. pokr. pol. poljed. poljub. polu. pom. pomen. pon. ponov. pop. por. port. pos. posl. posn. pov. pp. ppl. pr. 
+praet. prav. pravopis. pravosl. preb. pred. predl. predm. predp. preds. pref. pregib. prel. prem. premen. prep. 
+pres. pret. prev. pribl. prih. pril. primerj. primor. prip. pripor. prir. prist. priv. proc. prof. prog. proiz. 
+prom. pron. prop. prot. protest. prov. ps. pss. pt. publ. pz. q. qld. qu. quad. que. r. racc. rastl. razgl. 
+razl. razv. rd. red. ref. reg. rel. relig. rep. repr. rer. resp. rest. ret. rev. revol. rež. rim. rist. rkp. rm. 
+roj. rom. romun. rp. rr. rt. rud. ruš. ry. sal. samogl. san. sc. scen. sci. scr. sdv. seg. sek. sen. sept. ser. 
+sev. sg. sgt. sh. sig. sigg. sign. sim. sin. sing. sinh. skand. skl. sklad. sklanj. sklep. skr. sl. slik. slov. 
+slovak. slovn. sn. so. sob. soc. sociol. sod. sopomen. sopr. sor. sov. sovj. sp. spec. spl. spr. spreg. sq. sr. 
+sre. sred. sredoz. srh. ss. ssp. st. sta. stan. stanstar. stcsl. ste. stim. stol. stom. str. stroj. strok. stsl. 
+stud. sup. supl. suppl. svet. sz. t. tab. tech. ted. tehn. tehnol. tek. teks. tekst. tel. temp. ten. teol. ter. 
+term. test. th. theol. tim. tip. tisočl. tit. tl. tol. tolmač. tom. tor. tov. tr. trad. traj. trans. tren. 
+trib. tril. trop. trp. trž. ts. tt. tu. tur. turiz. tvor. tvorb. tč. u. ul. umet. un. univ. up. upr. ur. urad. 
+us. ust. utr. v. va. val. var. varn. ven. ver. verb. vest. vezal. vic. vis. viv. viz. viš. vod. vok. vol. vpr. 
+vrst. vrstil. vs. vv. vzd. vzg. vzh. vzor. w. wed. wg. wk. x. y. z. zah. zaim. zak. zap. zasl. zavar. zač. zb. 
+združ. zg. zn. znan. znanstv. zoot. zun. zv. zvd. á. é. ć. č. čas. čet. čl. člen. čustv. đ. ľ. ł. ş. ŠT. š. šir. 
+škofl. škot. šol. št. števil. štud. ů. ű. žen. žival. 
+""".split()
+
+for orth in abbrv:
+    _exc[orth] = [{ORTH: orth}]
+
+
+TOKENIZER_EXCEPTIONS = update_exc(BASE_EXCEPTIONS, _exc)

spacy/lang/uk/__init__.py

@@ -29,7 +29,7 @@ class Ukrainian(Language):
     assigns=["token.lemma"],
     default_config={
         "model": None,
-        "mode": "pymorphy2",
+        "mode": "pymorphy3",
         "overwrite": False,
         "scorer": {"@scorers": "spacy.lemmatizer_scorer.v1"},
     },

spacy/lang/uk/lemmatizer.py

@@ -14,11 +14,11 @@ class UkrainianLemmatizer(RussianLemmatizer):
         model: Optional[Model],
         name: str = "lemmatizer",
         *,
-        mode: str = "pymorphy2",
+        mode: str = "pymorphy3",
         overwrite: bool = False,
         scorer: Optional[Callable] = lemmatizer_score,
     ) -> None:
-        if mode == "pymorphy2":
+        if mode in {"pymorphy2", "pymorphy2_lookup"}:
             try:
                 from pymorphy2 import MorphAnalyzer
             except ImportError:
@@ -29,6 +29,17 @@ class UkrainianLemmatizer(RussianLemmatizer):
                 ) from None
             if getattr(self, "_morph", None) is None:
                 self._morph = MorphAnalyzer(lang="uk")
+        elif mode == "pymorphy3":
+            try:
+                from pymorphy3 import MorphAnalyzer
+            except ImportError:
+                raise ImportError(
+                    "The Ukrainian lemmatizer mode 'pymorphy3' requires the "
+                    "pymorphy3 library and dictionaries. Install them with: "
+                    "pip install pymorphy3 pymorphy3-dicts-uk"
+                ) from None
+            if getattr(self, "_morph", None) is None:
+                self._morph = MorphAnalyzer(lang="uk")
         super().__init__(
             vocab, model, name, mode=mode, overwrite=overwrite, scorer=scorer
         )

spacy/language.py

@@ -1,4 +1,4 @@
-from typing import Iterator, Optional, Any, Dict, Callable, Iterable, Collection
+from typing import Iterator, Optional, Any, Dict, Callable, Iterable
 from typing import Union, Tuple, List, Set, Pattern, Sequence
 from typing import NoReturn, TYPE_CHECKING, TypeVar, cast, overload
 
@@ -10,6 +10,7 @@ from contextlib import contextmanager
 from copy import deepcopy
 from pathlib import Path
 import warnings
+
 from thinc.api import get_current_ops, Config, CupyOps, Optimizer
 import srsly
 import multiprocessing as mp
@@ -24,7 +25,7 @@ from .pipe_analysis import validate_attrs, analyze_pipes, print_pipe_analysis
 from .training import Example, validate_examples
 from .training.initialize import init_vocab, init_tok2vec
 from .scorer import Scorer
-from .util import registry, SimpleFrozenList, _pipe, raise_error
+from .util import registry, SimpleFrozenList, _pipe, raise_error, _DEFAULT_EMPTY_PIPES
 from .util import SimpleFrozenDict, combine_score_weights, CONFIG_SECTION_ORDER
 from .util import warn_if_jupyter_cupy
 from .lang.tokenizer_exceptions import URL_MATCH, BASE_EXCEPTIONS
@@ -465,6 +466,8 @@ class Language:
         """
         if not isinstance(name, str):
             raise ValueError(Errors.E963.format(decorator="factory"))
+        if "." in name:
+            raise ValueError(Errors.E853.format(name=name))
         if not isinstance(default_config, dict):
             err = Errors.E962.format(
                 style="default config", name=name, cfg_type=type(default_config)
@@ -543,8 +546,11 @@ class Language:
 
         DOCS: https://spacy.io/api/language#component
         """
-        if name is not None and not isinstance(name, str):
+        if name is not None:
+            if not isinstance(name, str):
                 raise ValueError(Errors.E963.format(decorator="component"))
+            if "." in name:
+                raise ValueError(Errors.E853.format(name=name))
         component_name = name if name is not None else util.get_object_name(func)
 
         def add_component(component_func: "Pipe") -> Callable:
@@ -1023,8 +1029,8 @@ class Language:
                 raise ValueError(Errors.E109.format(name=name)) from e
             except Exception as e:
                 error_handler(name, proc, [doc], e)
-            if doc is None:
+            if not isinstance(doc, Doc):
-                raise ValueError(Errors.E005.format(name=name))
+                raise ValueError(Errors.E005.format(name=name, returned_type=type(doc)))
         return doc
 
     def disable_pipes(self, *names) -> "DisabledPipes":
@@ -1058,7 +1064,7 @@ class Language:
         """
         if enable is None and disable is None:
             raise ValueError(Errors.E991)
-        if disable is not None and isinstance(disable, str):
+        if isinstance(disable, str):
             disable = [disable]
         if enable is not None:
             if isinstance(enable, str):
@@ -1693,9 +1699,9 @@ class Language:
         config: Union[Dict[str, Any], Config] = {},
         *,
         vocab: Union[Vocab, bool] = True,
-        disable: Iterable[str] = SimpleFrozenList(),
+        disable: Union[str, Iterable[str]] = _DEFAULT_EMPTY_PIPES,
-        enable: Iterable[str] = SimpleFrozenList(),
+        enable: Union[str, Iterable[str]] = _DEFAULT_EMPTY_PIPES,
-        exclude: Iterable[str] = SimpleFrozenList(),
+        exclude: Union[str, Iterable[str]] = _DEFAULT_EMPTY_PIPES,
         meta: Dict[str, Any] = SimpleFrozenDict(),
         auto_fill: bool = True,
         validate: bool = True,
@@ -1706,12 +1712,12 @@ class Language:
 
         config (Dict[str, Any] / Config): The loaded config.
         vocab (Vocab): A Vocab object. If True, a vocab is created.
-        disable (Iterable[str]): Names of pipeline components to disable.
+        disable (Union[str, Iterable[str]]): Name(s) of pipeline component(s) to disable.
             Disabled pipes will be loaded but they won't be run unless you
             explicitly enable them by calling nlp.enable_pipe.
-        enable (Iterable[str]): Names of pipeline components to enable. All other
+        enable (Union[str, Iterable[str]]): Name(s) of pipeline component(s) to enable. All other
             pipes will be disabled (and can be enabled using `nlp.enable_pipe`).
-        exclude (Iterable[str]): Names of pipeline components to exclude.
+        exclude (Union[str, Iterable[str]]): Name(s) of pipeline component(s) to exclude.
             Excluded components won't be loaded.
         meta (Dict[str, Any]): Meta overrides for nlp.meta.
         auto_fill (bool): Automatically fill in missing values in config based
@@ -1866,9 +1872,38 @@ class Language:
             nlp.vocab.from_bytes(vocab_b)
 
         # Resolve disabled/enabled settings.
+        if isinstance(disable, str):
+            disable = [disable]
+        if isinstance(enable, str):
+            enable = [enable]
+        if isinstance(exclude, str):
+            exclude = [exclude]
+
+        def fetch_pipes_status(value: Iterable[str], key: str) -> Iterable[str]:
+            """Fetch value for `enable` or `disable` w.r.t. the specified config and passed arguments passed to
+            .load(). If both arguments and config specified values for this field, the passed arguments take precedence
+            and a warning is printed.
+            value (Iterable[str]): Passed value for `enable` or `disable`.
+            key (str): Key for field in config (either "enabled" or "disabled").
+            RETURN (Iterable[str]):
+            """
+            # We assume that no argument was passed if the value is the specified default value.
+            if id(value) == id(_DEFAULT_EMPTY_PIPES):
+                return config["nlp"].get(key, [])
+            else:
+                if len(config["nlp"].get(key, [])):
+                    warnings.warn(
+                        Warnings.W123.format(
+                            arg=key[:-1],
+                            arg_value=value,
+                            config_value=config["nlp"][key],
+                        )
+                    )
+                return value
+
         disabled_pipes = cls._resolve_component_status(
-            [*config["nlp"]["disabled"], *disable],
+            fetch_pipes_status(disable, "disabled"),
-            [*config["nlp"].get("enabled", []), *enable],
+            fetch_pipes_status(enable, "enabled"),
             config["nlp"]["pipeline"],
         )
         nlp._disabled = set(p for p in disabled_pipes if p not in exclude)
@@ -2026,37 +2061,34 @@ class Language:
 
     @staticmethod
     def _resolve_component_status(
-        disable: Iterable[str], enable: Iterable[str], pipe_names: Collection[str]
+        disable: Union[str, Iterable[str]],
+        enable: Union[str, Iterable[str]],
+        pipe_names: Iterable[str],
     ) -> Tuple[str, ...]:
         """Derives whether (1) `disable` and `enable` values are consistent and (2)
         resolves those to a single set of disabled components. Raises an error in
         case of inconsistency.
 
-        disable (Iterable[str]): Names of components or serialization fields to disable.
+        disable (Union[str, Iterable[str]]): Name(s) of component(s) or serialization fields to disable.
-        enable (Iterable[str]): Names of pipeline components to enable.
+        enable (Union[str, Iterable[str]]): Name(s) of pipeline component(s) to enable.
         pipe_names (Iterable[str]): Names of all pipeline components.
 
         RETURNS (Tuple[str, ...]): Names of components to exclude from pipeline w.r.t.
                                    specified includes and excludes.
         """
 
-        if disable is not None and isinstance(disable, str):
+        if isinstance(disable, str):
             disable = [disable]
         to_disable = disable
 
         if enable:
+            if isinstance(enable, str):
+                enable = [enable]
             to_disable = [
                 pipe_name for pipe_name in pipe_names if pipe_name not in enable
             ]
             if disable and disable != to_disable:
-                raise ValueError(
+                raise ValueError(Errors.E1042.format(enable=enable, disable=disable))
-                    Errors.E1042.format(
-                        arg1="enable",
-                        arg2="disable",
-                        arg1_values=enable,
-                        arg2_values=disable,
-                    )
-                )
 
         return tuple(to_disable)
 

spacy/matcher/__init__.py

@@ -1,5 +1,6 @@
 from .matcher import Matcher
 from .phrasematcher import PhraseMatcher
 from .dependencymatcher import DependencyMatcher
+from .levenshtein import levenshtein
 
-__all__ = ["Matcher", "PhraseMatcher", "DependencyMatcher"]
+__all__ = ["Matcher", "PhraseMatcher", "DependencyMatcher", "levenshtein"]

spacy/matcher/levenshtein.pyx

@@ -0,0 +1,15 @@
+# cython: profile=True, binding=True, infer_types=True
+from cpython.object cimport PyObject
+from libc.stdint cimport int64_t
+
+from typing import Optional
+
+
+cdef extern from "polyleven.c":
+    int64_t polyleven(PyObject *o1, PyObject *o2, int64_t k)
+
+
+cpdef int64_t levenshtein(a: str, b: str, k: Optional[int] = None):
+    if k is None:
+        k = -1
+    return polyleven(<PyObject*>a, <PyObject*>b, k)

spacy/matcher/matcher.pyx

@@ -1,5 +1,5 @@
 # cython: infer_types=True, cython: profile=True
-from typing import List
+from typing import List, Iterable
 
 from libcpp.vector cimport vector
 from libc.stdint cimport int32_t, int8_t
@@ -867,20 +867,27 @@ class _SetPredicate:
 
     def __call__(self, Token token):
         if self.is_extension:
-            value = get_string_id(token._.get(self.attr))
+            value = token._.get(self.attr)
         else:
             value = get_token_attr_for_matcher(token.c, self.attr)
 
-        if self.predicate in ("IS_SUBSET", "IS_SUPERSET", "INTERSECTS"):
+        if self.predicate in ("IN", "NOT_IN"):
+            if isinstance(value, (str, int)):
+                value = get_string_id(value)
+            else:
+                return False
+        elif self.predicate in ("IS_SUBSET", "IS_SUPERSET", "INTERSECTS"):
+            # ensure that all values are enclosed in a set
             if self.attr == MORPH:
                 # break up MORPH into individual Feat=Val values
                 value = set(get_string_id(v) for v in MorphAnalysis.from_id(self.vocab, value))
-            else:
+            elif isinstance(value, (str, int)):
-                # treat a single value as a list
+                value = set((get_string_id(value),))
-                if isinstance(value, (str, int)):
+            elif isinstance(value, Iterable) and all(isinstance(v, (str, int)) for v in value):
-                    value = set([get_string_id(value)])
-                else:
                 value = set(get_string_id(v) for v in value)
+            else:
+                return False
+
         if self.predicate == "IN":
             return value in self.value
         elif self.predicate == "NOT_IN":

spacy/matcher/polyleven.c

@@ -0,0 +1,384 @@
+/*
+ * Adapted from Polyleven (https://ceptord.net/)
+ *
+ * Source: https://github.com/fujimotos/polyleven/blob/c3f95a080626c5652f0151a2e449963288ccae84/polyleven.c
+ *
+ * Copyright (c) 2021 Fujimoto Seiji <fujimoto@ceptord.net>
+ * Copyright (c) 2021 Max Bachmann <kontakt@maxbachmann.de>
+ * Copyright (c) 2022 Nick Mazuk
+ * Copyright (c) 2022 Michael Weiss <code@mweiss.ch>
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include <Python.h>
+#include <stdint.h>
+
+#define MIN(a,b) ((a) < (b) ? (a) : (b))
+#define MAX(a,b) ((a) > (b) ? (a) : (b))
+#define CDIV(a,b) ((a) / (b) + ((a) % (b) > 0))
+#define BIT(i,n) (((i) >> (n)) & 1)
+#define FLIP(i,n) ((i) ^ ((uint64_t) 1 << (n)))
+#define ISASCII(kd) ((kd) == PyUnicode_1BYTE_KIND)
+
+/*
+ * Bare bone of PyUnicode
+ */
+struct strbuf {
+    void *ptr;
+    int kind;
+    int64_t len;
+};
+
+static void strbuf_init(struct strbuf *s, PyObject *o)
+{
+    s->ptr = PyUnicode_DATA(o);
+    s->kind = PyUnicode_KIND(o);
+    s->len = PyUnicode_GET_LENGTH(o);
+}
+
+#define strbuf_read(s, i) PyUnicode_READ((s)->kind, (s)->ptr, (i))
+
+/*
+ * An encoded mbleven model table.
+ *
+ * Each 8-bit integer represents an edit sequence, with using two
+ * bits for a single operation.
+ *
+ *   01 = DELETE, 10 = INSERT, 11 = REPLACE
+ *
+ * For example, 13 is '1101' in binary notation, so it means
+ * DELETE + REPLACE.
+ */
+static const uint8_t MBLEVEN_MATRIX[] = {
+     3,   0,  0,  0,  0,  0,  0,  0,
+     1,   0,  0,  0,  0,  0,  0,  0,
+     15,  9,  6,  0,  0,  0,  0,  0,
+     13,  7,  0,  0,  0,  0,  0,  0,
+     5,   0,  0,  0,  0,  0,  0,  0,
+     63, 39, 45, 57, 54, 30, 27,  0,
+     61, 55, 31, 37, 25, 22,  0,  0,
+     53, 29, 23,  0,  0,  0,  0,  0,
+     21,  0,  0,  0,  0,  0,  0,  0,
+};
+
+#define MBLEVEN_MATRIX_GET(k, d) ((((k) + (k) * (k)) / 2 - 1) + (d)) * 8
+
+static int64_t mbleven_ascii(char *s1, int64_t len1,
+                             char *s2, int64_t len2, int k)
+{
+    int pos;
+    uint8_t m;
+    int64_t i, j, c, r;
+
+    pos = MBLEVEN_MATRIX_GET(k, len1 - len2);
+    r = k + 1;
+
+    while (MBLEVEN_MATRIX[pos]) {
+        m = MBLEVEN_MATRIX[pos++];
+        i = j = c = 0;
+        while (i < len1 && j < len2) {
+            if (s1[i] != s2[j]) {
+                c++;
+                if (!m) break;
+                if (m & 1) i++;
+                if (m & 2) j++;
+                m >>= 2;
+            } else {
+                i++;
+                j++;
+            }
+        }
+        c += (len1 - i) + (len2 - j);
+        r = MIN(r, c);
+        if (r < 2) {
+            return r;
+        }
+    }
+    return r;
+}
+
+static int64_t mbleven(PyObject *o1, PyObject *o2, int64_t k)
+{
+    int pos;
+    uint8_t m;
+    int64_t i, j, c, r;
+    struct strbuf s1, s2;
+
+    strbuf_init(&s1, o1);
+    strbuf_init(&s2, o2);
+
+    if (s1.len < s2.len)
+        return mbleven(o2, o1, k);
+
+    if (k > 3)
+        return -1;
+
+    if (k < s1.len - s2.len)
+        return k + 1;
+
+    if (ISASCII(s1.kind) && ISASCII(s2.kind))
+        return mbleven_ascii(s1.ptr, s1.len, s2.ptr, s2.len, k);
+
+    pos = MBLEVEN_MATRIX_GET(k, s1.len - s2.len);
+    r = k + 1;
+
+    while (MBLEVEN_MATRIX[pos]) {
+        m = MBLEVEN_MATRIX[pos++];
+        i = j = c = 0;
+        while (i < s1.len && j < s2.len) {
+            if (strbuf_read(&s1, i) != strbuf_read(&s2, j)) {
+                c++;
+                if (!m) break;
+                if (m & 1) i++;
+                if (m & 2) j++;
+                m >>= 2;
+            } else {
+                i++;
+                j++;
+            }
+        }
+        c += (s1.len - i) + (s2.len - j);
+        r = MIN(r, c);
+        if (r < 2) {
+            return r;
+        }
+    }
+    return r;
+}
+
+/*
+ * Data structure to store Peq (equality bit-vector).
+ */
+struct blockmap_entry {
+    uint32_t key[128];
+    uint64_t val[128];
+};
+
+struct blockmap {
+    int64_t nr;
+    struct blockmap_entry *list;
+};
+
+#define blockmap_key(c) ((c) | 0x80000000U)
+#define blockmap_hash(c) ((c) % 128)
+
+static int blockmap_init(struct blockmap *map, struct strbuf *s)
+{
+    int64_t i;
+    struct blockmap_entry *be;
+    uint32_t c, k;
+    uint8_t h;
+
+    map->nr = CDIV(s->len, 64);
+    map->list = calloc(1,  map->nr * sizeof(struct blockmap_entry));
+    if (map->list == NULL) {
+        PyErr_NoMemory();
+        return -1;
+    }
+
+    for (i = 0; i < s->len; i++) {
+        be = &(map->list[i / 64]);
+        c = strbuf_read(s, i);
+        h = blockmap_hash(c);
+        k = blockmap_key(c);
+
+        while (be->key[h] && be->key[h] != k)
+            h = blockmap_hash(h + 1);
+        be->key[h] = k;
+        be->val[h] |= (uint64_t) 1 << (i % 64);
+    }
+    return 0;
+}
+
+static void blockmap_clear(struct blockmap *map)
+{
+    if (map->list)
+        free(map->list);
+    map->list = NULL;
+    map->nr = 0;
+}
+
+static uint64_t blockmap_get(struct blockmap *map, int block, uint32_t c)
+{
+    struct blockmap_entry *be;
+    uint8_t h;
+    uint32_t k;
+
+    h = blockmap_hash(c);
+    k = blockmap_key(c);
+
+    be = &(map->list[block]);
+    while (be->key[h] && be->key[h] != k)
+        h = blockmap_hash(h + 1);
+    return be->key[h] == k ? be->val[h] : 0;
+}
+
+/*
+ * Myers' bit-parallel algorithm
+ *
+ * See: G. Myers. "A fast bit-vector algorithm for approximate string
+ *      matching based on dynamic programming." Journal of the ACM, 1999.
+ */
+static int64_t myers1999_block(struct strbuf *s1, struct strbuf *s2,
+                               struct blockmap *map)
+{
+    uint64_t Eq, Xv, Xh, Ph, Mh, Pv, Mv, Last;
+    uint64_t *Mhc, *Phc;
+    int64_t i, b, hsize, vsize, Score;
+    uint8_t Pb, Mb;
+
+    hsize = CDIV(s1->len, 64);
+    vsize = CDIV(s2->len, 64);
+    Score = s2->len;
+
+    Phc = malloc(hsize * 2 * sizeof(uint64_t));
+    if (Phc == NULL) {
+        PyErr_NoMemory();
+        return -1;
+    }
+    Mhc = Phc + hsize;
+    memset(Phc, -1, hsize * sizeof(uint64_t));
+    memset(Mhc, 0, hsize * sizeof(uint64_t));
+    Last = (uint64_t)1 << ((s2->len - 1) % 64);
+
+    for (b = 0; b < vsize; b++) {
+        Mv = 0;
+        Pv = (uint64_t) -1;
+        Score = s2->len;
+
+        for (i = 0; i < s1->len; i++) {
+            Eq = blockmap_get(map, b, strbuf_read(s1, i));
+
+            Pb = BIT(Phc[i / 64], i % 64);
+            Mb = BIT(Mhc[i / 64], i % 64);
+
+            Xv = Eq | Mv;
+            Xh = ((((Eq | Mb) & Pv) + Pv) ^ Pv) | Eq | Mb;
+
+            Ph = Mv | ~ (Xh | Pv);
+            Mh = Pv & Xh;
+
+            if (Ph & Last) Score++;
+            if (Mh & Last) Score--;
+
+            if ((Ph >> 63) ^ Pb)
+                Phc[i / 64] = FLIP(Phc[i / 64], i % 64);
+
+            if ((Mh >> 63) ^ Mb)
+                Mhc[i / 64] = FLIP(Mhc[i / 64], i % 64);
+
+            Ph = (Ph << 1) | Pb;
+            Mh = (Mh << 1) | Mb;
+
+            Pv = Mh | ~ (Xv | Ph);
+            Mv = Ph & Xv;
+        }
+    }
+    free(Phc);
+    return Score;
+}
+
+static int64_t myers1999_simple(uint8_t *s1, int64_t len1, uint8_t *s2, int64_t len2)
+{
+    uint64_t Peq[256];
+    uint64_t Eq, Xv, Xh, Ph, Mh, Pv, Mv, Last;
+    int64_t i;
+    int64_t Score = len2;
+
+    memset(Peq, 0, sizeof(Peq));
+
+    for (i = 0; i < len2; i++)
+        Peq[s2[i]] |= (uint64_t) 1 << i;
+
+    Mv = 0;
+    Pv = (uint64_t) -1;
+    Last = (uint64_t) 1 << (len2 - 1);
+
+    for (i = 0; i < len1; i++) {
+        Eq = Peq[s1[i]];
+
+        Xv = Eq | Mv;
+        Xh = (((Eq & Pv) + Pv) ^ Pv) | Eq;
+
+        Ph = Mv | ~ (Xh | Pv);
+        Mh = Pv & Xh;
+
+        if (Ph & Last) Score++;
+        if (Mh & Last) Score--;
+
+        Ph = (Ph << 1) | 1;
+        Mh = (Mh << 1);
+
+        Pv = Mh | ~ (Xv | Ph);
+        Mv = Ph & Xv;
+    }
+    return Score;
+}
+
+static int64_t myers1999(PyObject *o1, PyObject *o2)
+{
+    struct strbuf s1, s2;
+    struct blockmap map;
+    int64_t ret;
+
+    strbuf_init(&s1, o1);
+    strbuf_init(&s2, o2);
+
+    if (s1.len < s2.len)
+        return myers1999(o2, o1);
+
+    if (ISASCII(s1.kind) && ISASCII(s2.kind) && s2.len < 65)
+        return myers1999_simple(s1.ptr, s1.len, s2.ptr, s2.len);
+
+    if (blockmap_init(&map, &s2))
+        return -1;
+
+    ret = myers1999_block(&s1, &s2, &map);
+    blockmap_clear(&map);
+    return ret;
+}
+
+/*
+ * Interface functions
+ */
+static int64_t polyleven(PyObject *o1, PyObject *o2, int64_t k)
+{
+    int64_t len1, len2;
+
+    len1 = PyUnicode_GET_LENGTH(o1);
+    len2 = PyUnicode_GET_LENGTH(o2);
+
+    if (len1 < len2)
+        return polyleven(o2, o1, k);
+
+    if (k == 0)
+        return PyUnicode_Compare(o1, o2) ? 1 : 0;
+
+    if (0 < k && k < len1 - len2)
+        return k + 1;
+
+    if (len2 == 0)
+        return len1;
+
+    if (0 < k && k < 4)
+        return mbleven(o1, o2, k);
+
+    return myers1999(o1, o2);
+}

spacy/ml/callbacks.py

@@ -89,11 +89,14 @@ def pipes_with_nvtx_range(
                 types.MethodType(nvtx_range_wrapper_for_pipe_method, pipe), func
             )
 
-            # Try to preserve the original function signature.
+            # We need to preserve the original function signature so that
+            # the original parameters are passed to pydantic for validation downstream.
             try:
                 wrapped_func.__signature__ = inspect.signature(func)  # type: ignore
             except:
-                pass
+                # Can fail for Cython methods that do not have bindings.
+                warnings.warn(Warnings.W122.format(method=name, pipe=pipe.name))
+                continue
 
             try:
                 setattr(

spacy/ml/models/entity_linker.py

@@ -1,11 +1,12 @@
 from pathlib import Path
 from typing import Optional, Callable, Iterable, List, Tuple
 from thinc.types import Floats2d
-from thinc.api import chain, clone, list2ragged, reduce_mean, residual
+from thinc.api import chain, list2ragged, reduce_mean, residual
-from thinc.api import Model, Maxout, Linear, noop, tuplify, Ragged
+from thinc.api import Model, Maxout, Linear, tuplify, Ragged
 
 from ...util import registry
-from ...kb import KnowledgeBase, Candidate, get_candidates
+from ...kb import KnowledgeBase, InMemoryLookupKB
+from ...kb import Candidate, get_candidates, get_candidates_batch
 from ...vocab import Vocab
 from ...tokens import Span, Doc
 from ..extract_spans import extract_spans
@@ -78,9 +79,11 @@ def span_maker_forward(model, docs: List[Doc], is_train) -> Tuple[Ragged, Callab
 
 
 @registry.misc("spacy.KBFromFile.v1")
-def load_kb(kb_path: Path) -> Callable[[Vocab], KnowledgeBase]:
+def load_kb(
-    def kb_from_file(vocab):
+    kb_path: Path,
-        kb = KnowledgeBase(vocab, entity_vector_length=1)
+) -> Callable[[Vocab], KnowledgeBase]:
+    def kb_from_file(vocab: Vocab):
+        kb = InMemoryLookupKB(vocab, entity_vector_length=1)
         kb.from_disk(kb_path)
         return kb
 
@@ -88,9 +91,11 @@ def load_kb(kb_path: Path) -> Callable[[Vocab], KnowledgeBase]:
 
 
 @registry.misc("spacy.EmptyKB.v1")
-def empty_kb(entity_vector_length: int) -> Callable[[Vocab], KnowledgeBase]:
+def empty_kb(
-    def empty_kb_factory(vocab):
+    entity_vector_length: int,
-        return KnowledgeBase(vocab=vocab, entity_vector_length=entity_vector_length)
+) -> Callable[[Vocab], KnowledgeBase]:
+    def empty_kb_factory(vocab: Vocab):
+        return InMemoryLookupKB(vocab=vocab, entity_vector_length=entity_vector_length)
 
     return empty_kb_factory
 
@@ -98,3 +103,10 @@ def empty_kb(entity_vector_length: int) -> Callable[[Vocab], KnowledgeBase]:
 @registry.misc("spacy.CandidateGenerator.v1")
 def create_candidates() -> Callable[[KnowledgeBase, Span], Iterable[Candidate]]:
     return get_candidates
+
+
+@registry.misc("spacy.CandidateBatchGenerator.v1")
+def create_candidates_batch() -> Callable[
+    [KnowledgeBase, Iterable[Span]], Iterable[Iterable[Candidate]]
+]:
+    return get_candidates_batch

spacy/pipeline/edit_tree_lemmatizer.py

@@ -1,7 +1,6 @@
 from typing import cast, Any, Callable, Dict, Iterable, List, Optional
-from typing import Sequence, Tuple, Union
+from typing import Tuple
 from collections import Counter
-from copy import deepcopy
 from itertools import islice
 import numpy as np
 
@@ -149,9 +148,7 @@ class EditTreeLemmatizer(TrainablePipe):
         if not any(len(doc) for doc in docs):
             # Handle cases where there are no tokens in any docs.
             n_labels = len(self.cfg["labels"])
-            guesses: List[Ints2d] = [
+            guesses: List[Ints2d] = [self.model.ops.alloc2i(0, n_labels) for _ in docs]
-                self.model.ops.alloc((0, n_labels), dtype="i") for doc in docs
-            ]
             assert len(guesses) == n_docs
             return guesses
         scores = self.model.predict(docs)

spacy/pipeline/entity_linker.py

@@ -53,9 +53,11 @@ DEFAULT_NEL_MODEL = Config().from_str(default_model_config)["model"]
         "incl_context": True,
         "entity_vector_length": 64,
         "get_candidates": {"@misc": "spacy.CandidateGenerator.v1"},
+        "get_candidates_batch": {"@misc": "spacy.CandidateBatchGenerator.v1"},
         "overwrite": True,
         "scorer": {"@scorers": "spacy.entity_linker_scorer.v1"},
         "use_gold_ents": True,
+        "candidates_batch_size": 1,
         "threshold": None,
     },
     default_score_weights={
@@ -75,9 +77,13 @@ def make_entity_linker(
     incl_context: bool,
     entity_vector_length: int,
     get_candidates: Callable[[KnowledgeBase, Span], Iterable[Candidate]],
+    get_candidates_batch: Callable[
+        [KnowledgeBase, Iterable[Span]], Iterable[Iterable[Candidate]]
+    ],
     overwrite: bool,
     scorer: Optional[Callable],
     use_gold_ents: bool,
+    candidates_batch_size: int,
     threshold: Optional[float] = None,
 ):
     """Construct an EntityLinker component.
@@ -90,17 +96,21 @@ def make_entity_linker(
     incl_prior (bool): Whether or not to include prior probabilities from the KB in the model.
     incl_context (bool): Whether or not to include the local context in the model.
     entity_vector_length (int): Size of encoding vectors in the KB.
-    get_candidates (Callable[[KnowledgeBase, "Span"], Iterable[Candidate]]): Function that
+    get_candidates (Callable[[KnowledgeBase, Span], Iterable[Candidate]]): Function that
         produces a list of candidates, given a certain knowledge base and a textual mention.
+    get_candidates_batch (
+        Callable[[KnowledgeBase, Iterable[Span]], Iterable[Iterable[Candidate]]], Iterable[Candidate]]
+        ): Function that produces a list of candidates, given a certain knowledge base and several textual mentions.
     scorer (Optional[Callable]): The scoring method.
     use_gold_ents (bool): Whether to copy entities from gold docs or not. If false, another
         component must provide entity annotations.
+    candidates_batch_size (int): Size of batches for entity candidate generation.
     threshold (Optional[float]): Confidence threshold for entity predictions. If confidence is below the threshold,
         prediction is discarded. If None, predictions are not filtered by any threshold.
     """
 
     if not model.attrs.get("include_span_maker", False):
-        # The only difference in arguments here is that use_gold_ents is not available
+        # The only difference in arguments here is that use_gold_ents and threshold aren't available.
         return EntityLinker_v1(
             nlp.vocab,
             model,
@@ -124,9 +134,11 @@ def make_entity_linker(
         incl_context=incl_context,
         entity_vector_length=entity_vector_length,
         get_candidates=get_candidates,
+        get_candidates_batch=get_candidates_batch,
         overwrite=overwrite,
         scorer=scorer,
         use_gold_ents=use_gold_ents,
+        candidates_batch_size=candidates_batch_size,
         threshold=threshold,
     )
 
@@ -160,9 +172,13 @@ class EntityLinker(TrainablePipe):
         incl_context: bool,
         entity_vector_length: int,
         get_candidates: Callable[[KnowledgeBase, Span], Iterable[Candidate]],
+        get_candidates_batch: Callable[
+            [KnowledgeBase, Iterable[Span]], Iterable[Iterable[Candidate]]
+        ],
         overwrite: bool = BACKWARD_OVERWRITE,
         scorer: Optional[Callable] = entity_linker_score,
         use_gold_ents: bool,
+        candidates_batch_size: int,
         threshold: Optional[float] = None,
     ) -> None:
         """Initialize an entity linker.
@@ -178,10 +194,14 @@ class EntityLinker(TrainablePipe):
         entity_vector_length (int): Size of encoding vectors in the KB.
         get_candidates (Callable[[KnowledgeBase, Span], Iterable[Candidate]]): Function that
             produces a list of candidates, given a certain knowledge base and a textual mention.
-        scorer (Optional[Callable]): The scoring method. Defaults to
+        get_candidates_batch (
-            Scorer.score_links.
+            Callable[[KnowledgeBase, Iterable[Span]], Iterable[Iterable[Candidate]]],
+            Iterable[Candidate]]
+            ): Function that produces a list of candidates, given a certain knowledge base and several textual mentions.
+        scorer (Optional[Callable]): The scoring method. Defaults to Scorer.score_links.
         use_gold_ents (bool): Whether to copy entities from gold docs or not. If false, another
             component must provide entity annotations.
+        candidates_batch_size (int): Size of batches for entity candidate generation.
         threshold (Optional[float]): Confidence threshold for entity predictions. If confidence is below the
             threshold, prediction is discarded. If None, predictions are not filtered by any threshold.
         DOCS: https://spacy.io/api/entitylinker#init
@@ -204,22 +224,27 @@ class EntityLinker(TrainablePipe):
         self.incl_prior = incl_prior
         self.incl_context = incl_context
         self.get_candidates = get_candidates
+        self.get_candidates_batch = get_candidates_batch
         self.cfg: Dict[str, Any] = {"overwrite": overwrite}
         self.distance = CosineDistance(normalize=False)
         # how many neighbour sentences to take into account
-        # create an empty KB by default. If you want to load a predefined one, specify it in 'initialize'.
+        # create an empty KB by default
         self.kb = empty_kb(entity_vector_length)(self.vocab)
         self.scorer = scorer
         self.use_gold_ents = use_gold_ents
+        self.candidates_batch_size = candidates_batch_size
         self.threshold = threshold
 
+        if candidates_batch_size < 1:
+            raise ValueError(Errors.E1044)
+
     def set_kb(self, kb_loader: Callable[[Vocab], KnowledgeBase]):
         """Define the KB of this pipe by providing a function that will
         create it using this object's vocab."""
         if not callable(kb_loader):
             raise ValueError(Errors.E885.format(arg_type=type(kb_loader)))
 
-        self.kb = kb_loader(self.vocab)
+        self.kb = kb_loader(self.vocab)  # type: ignore
 
     def validate_kb(self) -> None:
         # Raise an error if the knowledge base is not initialized.
@@ -241,8 +266,8 @@ class EntityLinker(TrainablePipe):
         get_examples (Callable[[], Iterable[Example]]): Function that
             returns a representative sample of gold-standard Example objects.
         nlp (Language): The current nlp object the component is part of.
-        kb_loader (Callable[[Vocab], KnowledgeBase]): A function that creates a KnowledgeBase from a Vocab instance.
+        kb_loader (Callable[[Vocab], KnowledgeBase]): A function that creates a KnowledgeBase from a Vocab
-            Note that providing this argument, will overwrite all data accumulated in the current KB.
+            instance. Note that providing this argument will overwrite all data accumulated in the current KB.
             Use this only when loading a KB as-such from file.
 
         DOCS: https://spacy.io/api/entitylinker#initialize
@@ -419,15 +444,40 @@ class EntityLinker(TrainablePipe):
             if len(doc) == 0:
                 continue
             sentences = [s for s in doc.sents]
-            # Looping through each entity (TODO: rewrite)
+
-            for ent in doc.ents:
+            # Loop over entities in batches.
+            for ent_idx in range(0, len(doc.ents), self.candidates_batch_size):
+                ent_batch = doc.ents[ent_idx : ent_idx + self.candidates_batch_size]
+
+                # Look up candidate entities.
+                valid_ent_idx = [
+                    idx
+                    for idx in range(len(ent_batch))
+                    if ent_batch[idx].label_ not in self.labels_discard
+                ]
+
+                batch_candidates = list(
+                    self.get_candidates_batch(
+                        self.kb, [ent_batch[idx] for idx in valid_ent_idx]
+                    )
+                    if self.candidates_batch_size > 1
+                    else [
+                        self.get_candidates(self.kb, ent_batch[idx])
+                        for idx in valid_ent_idx
+                    ]
+                )
+
+                # Looping through each entity in batch (TODO: rewrite)
+                for j, ent in enumerate(ent_batch):
                     sent_index = sentences.index(ent.sent)
                     assert sent_index >= 0
 
                     if self.incl_context:
                         # get n_neighbour sentences, clipped to the length of the document
                         start_sentence = max(0, sent_index - self.n_sents)
-                    end_sentence = min(len(sentences) - 1, sent_index + self.n_sents)
+                        end_sentence = min(
+                            len(sentences) - 1, sent_index + self.n_sents
+                        )
                         start_token = sentences[start_sentence].start
                         end_token = sentences[end_sentence].end
                         sent_doc = doc[start_token:end_token].as_doc()
@@ -440,7 +490,7 @@ class EntityLinker(TrainablePipe):
                         # ignoring this entity - setting to NIL
                         final_kb_ids.append(self.NIL)
                     else:
-                    candidates = list(self.get_candidates(self.kb, ent))
+                        candidates = list(batch_candidates[j])
                         if not candidates:
                             # no prediction possible for this entity - setting to NIL
                             final_kb_ids.append(self.NIL)
@@ -476,9 +526,11 @@ class EntityLinker(TrainablePipe):
                                 scores = prior_probs + sims - (prior_probs * sims)
                             final_kb_ids.append(
                                 candidates[scores.argmax().item()].entity_
-                            if self.threshold is None or scores.max() >= self.threshold
+                                if self.threshold is None
+                                or scores.max() >= self.threshold
                                 else EntityLinker.NIL
                             )
+
         if not (len(final_kb_ids) == entity_count):
             err = Errors.E147.format(
                 method="predict", msg="result variables not of equal length"

spacy/pipeline/entityruler.py

@@ -1,6 +1,5 @@
-import warnings
 from typing import Optional, Union, List, Dict, Tuple, Iterable, Any, Callable, Sequence
-from typing import cast
+import warnings
 from collections import defaultdict
 from pathlib import Path
 import srsly
@@ -317,7 +316,7 @@ class EntityRuler(Pipe):
                     phrase_pattern["id"] = ent_id
                 phrase_patterns.append(phrase_pattern)
             for entry in token_patterns + phrase_patterns:  # type: ignore[operator]
-                label = entry["label"]
+                label = entry["label"]  # type: ignore
                 if "id" in entry:
                     ent_label = label
                     label = self._create_label(label, entry["id"])

spacy/pipeline/legacy/entity_linker.py

@@ -68,8 +68,7 @@ class EntityLinker_v1(TrainablePipe):
         entity_vector_length (int): Size of encoding vectors in the KB.
         get_candidates (Callable[[KnowledgeBase, Span], Iterable[Candidate]]): Function that
             produces a list of candidates, given a certain knowledge base and a textual mention.
-        scorer (Optional[Callable]): The scoring method. Defaults to
+        scorer (Optional[Callable]): The scoring method. Defaults to Scorer.score_links.
-            Scorer.score_links.
         DOCS: https://spacy.io/api/entitylinker#init
         """
         self.vocab = vocab
@@ -115,7 +114,7 @@ class EntityLinker_v1(TrainablePipe):
         get_examples (Callable[[], Iterable[Example]]): Function that
             returns a representative sample of gold-standard Example objects.
         nlp (Language): The current nlp object the component is part of.
-        kb_loader (Callable[[Vocab], KnowledgeBase]): A function that creates a KnowledgeBase from a Vocab instance.
+        kb_loader (Callable[[Vocab], KnowledgeBase]): A function that creates an InMemoryLookupKB from a Vocab instance.
             Note that providing this argument, will overwrite all data accumulated in the current KB.
             Use this only when loading a KB as-such from file.
 

spacy/pipeline/pipe.pyx

@@ -1,4 +1,4 @@
-# cython: infer_types=True, profile=True
+# cython: infer_types=True, profile=True, binding=True
 from typing import Optional, Tuple, Iterable, Iterator, Callable, Union, Dict
 import srsly
 import warnings

spacy/pipeline/spancat.py

@@ -26,17 +26,17 @@ scorer = {"@layers": "spacy.LinearLogistic.v1"}
 hidden_size = 128
 
 [model.tok2vec]
-@architectures = "spacy.Tok2Vec.v1"
+@architectures = "spacy.Tok2Vec.v2"
 
 [model.tok2vec.embed]
-@architectures = "spacy.MultiHashEmbed.v1"
+@architectures = "spacy.MultiHashEmbed.v2"
 width = 96
 rows = [5000, 2000, 1000, 1000]
 attrs = ["ORTH", "PREFIX", "SUFFIX", "SHAPE"]
 include_static_vectors = false
 
 [model.tok2vec.encode]
-@architectures = "spacy.MaxoutWindowEncoder.v1"
+@architectures = "spacy.MaxoutWindowEncoder.v2"
 width = ${model.tok2vec.embed.width}
 window_size = 1
 maxout_pieces = 3
@@ -133,6 +133,9 @@ def make_spancat(
     spans_key (str): Key of the doc.spans dict to save the spans under. During
         initialization and training, the component will look for spans on the
         reference document under the same key.
+    scorer (Optional[Callable]): The scoring method. Defaults to
+        Scorer.score_spans for the Doc.spans[spans_key] with overlapping
+        spans allowed.
     threshold (float): Minimum probability to consider a prediction positive.
         Spans with a positive prediction will be saved on the Doc. Defaults to
         0.5.

spacy/pipeline/textcat_multilabel.py

@@ -19,7 +19,7 @@ multi_label_default_config = """
 @architectures = "spacy.TextCatEnsemble.v2"
 
 [model.tok2vec]
-@architectures = "spacy.Tok2Vec.v1"
+@architectures = "spacy.Tok2Vec.v2"
 
 [model.tok2vec.embed]
 @architectures = "spacy.MultiHashEmbed.v2"
@@ -29,7 +29,7 @@ attrs = ["ORTH", "LOWER", "PREFIX", "SUFFIX", "SHAPE", "ID"]
 include_static_vectors = false
 
 [model.tok2vec.encode]
-@architectures = "spacy.MaxoutWindowEncoder.v1"
+@architectures = "spacy.MaxoutWindowEncoder.v2"
 width = ${model.tok2vec.embed.width}
 window_size = 1
 maxout_pieces = 3
@@ -96,8 +96,8 @@ def make_multilabel_textcat(
     model: Model[List[Doc], List[Floats2d]],
     threshold: float,
     scorer: Optional[Callable],
-) -> "TextCategorizer":
+) -> "MultiLabel_TextCategorizer":
-    """Create a TextCategorizer component. The text categorizer predicts categories
+    """Create a MultiLabel_TextCategorizer component. The text categorizer predicts categories
     over a whole document. It can learn one or more labels, and the labels are considered
     to be non-mutually exclusive, which means that there can be zero or more labels
     per doc).
@@ -105,6 +105,7 @@ def make_multilabel_textcat(
     model (Model[List[Doc], List[Floats2d]]): A model instance that predicts
         scores for each category.
     threshold (float): Cutoff to consider a prediction "positive".
+    scorer (Optional[Callable]): The scoring method.
     """
     return MultiLabel_TextCategorizer(
         nlp.vocab, model, name, threshold=threshold, scorer=scorer
@@ -147,6 +148,7 @@ class MultiLabel_TextCategorizer(TextCategorizer):
         name (str): The component instance name, used to add entries to the
             losses during training.
         threshold (float): Cutoff to consider a prediction "positive".
+        scorer (Optional[Callable]): The scoring method.
 
         DOCS: https://spacy.io/api/textcategorizer#init
         """

spacy/pipeline/tok2vec.py

@@ -123,9 +123,6 @@ class Tok2Vec(TrainablePipe):
             width = self.model.get_dim("nO")
             return [self.model.ops.alloc((0, width)) for doc in docs]
         tokvecs = self.model.predict(docs)
-        batch_id = Tok2VecListener.get_batch_id(docs)
-        for listener in self.listeners:
-            listener.receive(batch_id, tokvecs, _empty_backprop)
         return tokvecs
 
     def set_annotations(self, docs: Sequence[Doc], tokvecses) -> None:
@@ -286,6 +283,17 @@ class Tok2VecListener(Model):
 def forward(model: Tok2VecListener, inputs, is_train: bool):
     """Supply the outputs from the upstream Tok2Vec component."""
     if is_train:
+        # This might occur during training when the tok2vec layer is frozen / hasn't been updated.
+        # In that case, it should be set to "annotating" so we can retrieve the embeddings from the doc.
+        if model._batch_id is None:
+            outputs = []
+            for doc in inputs:
+                if doc.tensor.size == 0:
+                    raise ValueError(Errors.E203.format(name="tok2vec"))
+                else:
+                    outputs.append(doc.tensor)
+            return outputs, _empty_backprop
+        else:
             model.verify_inputs(inputs)
             return model._outputs, model._backprop
     else:
@@ -306,7 +314,7 @@ def forward(model: Tok2VecListener, inputs, is_train: bool):
                 outputs.append(model.ops.alloc2f(len(doc), width))
             else:
                 outputs.append(doc.tensor)
-        return outputs, lambda dX: []
+        return outputs, _empty_backprop
 
 
 def _empty_backprop(dX):  # for pickling

spacy/pipeline/trainable_pipe.pyx

@@ -1,4 +1,4 @@
-# cython: infer_types=True, profile=True
+# cython: infer_types=True, profile=True, binding=True
 from typing import Iterable, Iterator, Optional, Dict, Tuple, Callable
 import srsly
 from thinc.api import set_dropout_rate, Model, Optimizer

spacy/schemas.py

@@ -181,12 +181,12 @@ class TokenPatternNumber(BaseModel):
     IS_SUBSET: Optional[List[StrictInt]] = Field(None, alias="is_subset")
     IS_SUPERSET: Optional[List[StrictInt]] = Field(None, alias="is_superset")
     INTERSECTS: Optional[List[StrictInt]] = Field(None, alias="intersects")
-    EQ: Union[StrictInt, StrictFloat] = Field(None, alias="==")
+    EQ: Optional[Union[StrictInt, StrictFloat]] = Field(None, alias="==")
-    NEQ: Union[StrictInt, StrictFloat] = Field(None, alias="!=")
+    NEQ: Optional[Union[StrictInt, StrictFloat]] = Field(None, alias="!=")
-    GEQ: Union[StrictInt, StrictFloat] = Field(None, alias=">=")
+    GEQ: Optional[Union[StrictInt, StrictFloat]] = Field(None, alias=">=")
-    LEQ: Union[StrictInt, StrictFloat] = Field(None, alias="<=")
+    LEQ: Optional[Union[StrictInt, StrictFloat]] = Field(None, alias="<=")
-    GT: Union[StrictInt, StrictFloat] = Field(None, alias=">")
+    GT: Optional[Union[StrictInt, StrictFloat]] = Field(None, alias=">")
-    LT: Union[StrictInt, StrictFloat] = Field(None, alias="<")
+    LT: Optional[Union[StrictInt, StrictFloat]] = Field(None, alias="<")
 
     class Config:
         extra = "forbid"
@@ -207,7 +207,7 @@ class TokenPatternOperatorSimple(str, Enum):
 
 
 class TokenPatternOperatorMinMax(ConstrainedStr):
-    regex = re.compile("^({\d+}|{\d+,\d*}|{\d*,\d+})$")
+    regex = re.compile(r"^({\d+}|{\d+,\d*}|{\d*,\d+})$")
 
 
 TokenPatternOperator = Union[TokenPatternOperatorSimple, TokenPatternOperatorMinMax]
@@ -430,7 +430,7 @@ class ProjectConfigAssetURL(BaseModel):
     # fmt: off
     dest: StrictStr = Field(..., title="Destination of downloaded asset")
     url: Optional[StrictStr] = Field(None, title="URL of asset")
-    checksum: str = Field(None, title="MD5 hash of file", regex=r"([a-fA-F\d]{32})")
+    checksum: Optional[str] = Field(None, title="MD5 hash of file", regex=r"([a-fA-F\d]{32})")
     description: StrictStr = Field("", title="Description of asset")
     # fmt: on
 
@@ -438,7 +438,7 @@ class ProjectConfigAssetURL(BaseModel):
 class ProjectConfigAssetGit(BaseModel):
     # fmt: off
     git: ProjectConfigAssetGitItem = Field(..., title="Git repo information")
-    checksum: str = Field(None, title="MD5 hash of file", regex=r"([a-fA-F\d]{32})")
+    checksum: Optional[str] = Field(None, title="MD5 hash of file", regex=r"([a-fA-F\d]{32})")
     description: Optional[StrictStr] = Field(None, title="Description of asset")
     # fmt: on
 
@@ -508,12 +508,20 @@ class DocJSONSchema(BaseModel):
         None, title="Indices of sentences' start and end indices"
     )
     text: StrictStr = Field(..., title="Document text")
-    spans: Dict[StrictStr, List[Dict[StrictStr, Union[StrictStr, StrictInt]]]] = Field(
+    spans: Optional[
-        None, title="Span information - end/start indices, label, KB ID"
+        Dict[StrictStr, List[Dict[StrictStr, Union[StrictStr, StrictInt]]]]
-    )
+    ] = Field(None, title="Span information - end/start indices, label, KB ID")
     tokens: List[Dict[StrictStr, Union[StrictStr, StrictInt]]] = Field(
         ..., title="Token information - ID, start, annotations"
     )
-    _: Optional[Dict[StrictStr, Any]] = Field(
+    underscore_doc: Optional[Dict[StrictStr, Any]] = Field(
-        None, title="Any custom data stored in the document's _ attribute"
+        None,
+        title="Any custom data stored in the document's _ attribute",
+        alias="_",
+    )
+    underscore_token: Optional[Dict[StrictStr, List[Dict[StrictStr, Any]]]] = Field(
+        None, title="Any custom data stored in the token's _ attribute"
+    )
+    underscore_span: Optional[Dict[StrictStr, List[Dict[StrictStr, Any]]]] = Field(
+        None, title="Any custom data stored in the span's _ attribute"
     )

spacy/tests/conftest.py

@@ -256,11 +256,21 @@ def ko_tokenizer_tokenizer():
     return nlp.tokenizer
 
 
+@pytest.fixture(scope="module")
+def la_tokenizer():
+    return get_lang_class("la")().tokenizer
+
+
 @pytest.fixture(scope="session")
 def lb_tokenizer():
     return get_lang_class("lb")().tokenizer
 
 
+@pytest.fixture(scope="session")
+def lg_tokenizer():
+    return get_lang_class("lg")().tokenizer
+
+
 @pytest.fixture(scope="session")
 def lt_tokenizer():
     return get_lang_class("lt")().tokenizer
@@ -323,16 +333,24 @@ def ro_tokenizer():
 
 @pytest.fixture(scope="session")
 def ru_tokenizer():
-    pytest.importorskip("pymorphy2")
+    pytest.importorskip("pymorphy3")
     return get_lang_class("ru")().tokenizer
 
 
 @pytest.fixture
 def ru_lemmatizer():
-    pytest.importorskip("pymorphy2")
+    pytest.importorskip("pymorphy3")
     return get_lang_class("ru")().add_pipe("lemmatizer")
 
 
+@pytest.fixture
+def ru_lookup_lemmatizer():
+    pytest.importorskip("pymorphy2")
+    return get_lang_class("ru")().add_pipe(
+        "lemmatizer", config={"mode": "pymorphy2_lookup"}
+    )
+
+
 @pytest.fixture(scope="session")
 def sa_tokenizer():
     return get_lang_class("sa")().tokenizer
@@ -401,15 +419,24 @@ def ky_tokenizer():
 
 @pytest.fixture(scope="session")
 def uk_tokenizer():
-    pytest.importorskip("pymorphy2")
+    pytest.importorskip("pymorphy3")
     return get_lang_class("uk")().tokenizer
 
 
 @pytest.fixture
 def uk_lemmatizer():
+    pytest.importorskip("pymorphy3")
+    pytest.importorskip("pymorphy3_dicts_uk")
+    return get_lang_class("uk")().add_pipe("lemmatizer")
+
+
+@pytest.fixture
+def uk_lookup_lemmatizer():
     pytest.importorskip("pymorphy2")
     pytest.importorskip("pymorphy2_dicts_uk")
-    return get_lang_class("uk")().add_pipe("lemmatizer")
+    return get_lang_class("uk")().add_pipe(
+        "lemmatizer", config={"mode": "pymorphy2_lookup"}
+    )
 
 
 @pytest.fixture(scope="session")

spacy/tests/doc/test_doc_api.py

@@ -3,6 +3,7 @@ import weakref
 import numpy
 from numpy.testing import assert_array_equal
 import pytest
+import warnings
 from thinc.api import NumpyOps, get_current_ops
 
 from spacy.attrs import DEP, ENT_IOB, ENT_TYPE, HEAD, IS_ALPHA, MORPH, POS
@@ -81,6 +82,21 @@ def test_issue2396(en_vocab):
     assert (span.get_lca_matrix() == matrix).all()
 
 
+@pytest.mark.issue(11499)
+def test_init_args_unmodified(en_vocab):
+    words = ["A", "sentence"]
+    ents = ["B-TYPE1", ""]
+    sent_starts = [True, False]
+    Doc(
+        vocab=en_vocab,
+        words=words,
+        ents=ents,
+        sent_starts=sent_starts,
+    )
+    assert ents == ["B-TYPE1", ""]
+    assert sent_starts == [True, False]
+
+
 @pytest.mark.parametrize("text", ["-0.23", "+123,456", "±1"])
 @pytest.mark.parametrize("lang_cls", [English, MultiLanguage])
 @pytest.mark.issue(2782)
@@ -529,9 +545,9 @@ def test_doc_from_array_sent_starts(en_vocab):
     # no warning using default attrs
     attrs = doc._get_array_attrs()
     arr = doc.to_array(attrs)
-    with pytest.warns(None) as record:
+    with warnings.catch_warnings():
+        warnings.simplefilter("error")
         new_doc.from_array(attrs, arr)
-        assert len(record) == 0
     # only SENT_START uses SENT_START
     attrs = [SENT_START]
     arr = doc.to_array(attrs)

spacy/tests/doc/test_json_doc_conversion.py

@@ -1,12 +1,15 @@
 import pytest
 import spacy
 from spacy import schemas
-from spacy.tokens import Doc, Span
+from spacy.tokens import Doc, Span, Token
+import srsly
+from .test_underscore import clean_underscore  # noqa: F401
 
 
 @pytest.fixture()
 def doc(en_vocab):
     words = ["c", "d", "e"]
+    spaces = [True, True, True]
     pos = ["VERB", "NOUN", "NOUN"]
     tags = ["VBP", "NN", "NN"]
     heads = [0, 0, 1]
@@ -17,6 +20,7 @@ def doc(en_vocab):
     return Doc(
         en_vocab,
         words=words,
+        spaces=spaces,
         pos=pos,
         tags=tags,
         heads=heads,
@@ -45,6 +49,47 @@ def doc_without_deps(en_vocab):
     )
 
 
+@pytest.fixture()
+def doc_json():
+    return {
+        "text": "c d e ",
+        "ents": [{"start": 2, "end": 3, "label": "ORG"}],
+        "sents": [{"start": 0, "end": 5}],
+        "tokens": [
+            {
+                "id": 0,
+                "start": 0,
+                "end": 1,
+                "tag": "VBP",
+                "pos": "VERB",
+                "morph": "Feat1=A",
+                "dep": "ROOT",
+                "head": 0,
+            },
+            {
+                "id": 1,
+                "start": 2,
+                "end": 3,
+                "tag": "NN",
+                "pos": "NOUN",
+                "morph": "Feat1=B",
+                "dep": "dobj",
+                "head": 0,
+            },
+            {
+                "id": 2,
+                "start": 4,
+                "end": 5,
+                "tag": "NN",
+                "pos": "NOUN",
+                "morph": "Feat1=A|Feat2=D",
+                "dep": "dobj",
+                "head": 1,
+            },
+        ],
+    }
+
+
 def test_doc_to_json(doc):
     json_doc = doc.to_json()
     assert json_doc["text"] == "c d e "
@@ -56,7 +101,8 @@ def test_doc_to_json(doc):
     assert json_doc["ents"][0]["start"] == 2  # character offset!
     assert json_doc["ents"][0]["end"] == 3  # character offset!
     assert json_doc["ents"][0]["label"] == "ORG"
-    assert not schemas.validate(schemas.DocJSONSchema, json_doc)
+    assert len(schemas.validate(schemas.DocJSONSchema, json_doc)) == 0
+    assert srsly.json_loads(srsly.json_dumps(json_doc)) == json_doc
 
 
 def test_doc_to_json_underscore(doc):
@@ -64,11 +110,99 @@ def test_doc_to_json_underscore(doc):
     Doc.set_extension("json_test2", default=False)
     doc._.json_test1 = "hello world"
     doc._.json_test2 = [1, 2, 3]
+
     json_doc = doc.to_json(underscore=["json_test1", "json_test2"])
     assert "_" in json_doc
     assert json_doc["_"]["json_test1"] == "hello world"
     assert json_doc["_"]["json_test2"] == [1, 2, 3]
-    assert not schemas.validate(schemas.DocJSONSchema, json_doc)
+    assert len(schemas.validate(schemas.DocJSONSchema, json_doc)) == 0
+    assert srsly.json_loads(srsly.json_dumps(json_doc)) == json_doc
+
+
+def test_doc_to_json_with_token_span_attributes(doc):
+    Doc.set_extension("json_test1", default=False)
+    Doc.set_extension("json_test2", default=False)
+    Token.set_extension("token_test", default=False)
+    Span.set_extension("span_test", default=False)
+
+    doc._.json_test1 = "hello world"
+    doc._.json_test2 = [1, 2, 3]
+    doc[0:1]._.span_test = "span_attribute"
+    doc[0:2]._.span_test = "span_attribute_2"
+    doc[0]._.token_test = 117
+    doc[1]._.token_test = 118
+    doc.spans["span_group"] = [doc[0:1]]
+    json_doc = doc.to_json(
+        underscore=["json_test1", "json_test2", "token_test", "span_test"]
+    )
+
+    assert "_" in json_doc
+    assert json_doc["_"]["json_test1"] == "hello world"
+    assert json_doc["_"]["json_test2"] == [1, 2, 3]
+    assert "underscore_token" in json_doc
+    assert "underscore_span" in json_doc
+    assert json_doc["underscore_token"]["token_test"][0]["value"] == 117
+    assert json_doc["underscore_token"]["token_test"][1]["value"] == 118
+    assert json_doc["underscore_span"]["span_test"][0]["value"] == "span_attribute"
+    assert json_doc["underscore_span"]["span_test"][1]["value"] == "span_attribute_2"
+    assert len(schemas.validate(schemas.DocJSONSchema, json_doc)) == 0
+    assert srsly.json_loads(srsly.json_dumps(json_doc)) == json_doc
+
+
+def test_doc_to_json_with_custom_user_data(doc):
+    Doc.set_extension("json_test", default=False)
+    Token.set_extension("token_test", default=False)
+    Span.set_extension("span_test", default=False)
+
+    doc._.json_test = "hello world"
+    doc[0:1]._.span_test = "span_attribute"
+    doc[0]._.token_test = 117
+    json_doc = doc.to_json(underscore=["json_test", "token_test", "span_test"])
+    doc.user_data["user_data_test"] = 10
+    doc.user_data[("user_data_test2", True)] = 10
+
+    assert "_" in json_doc
+    assert json_doc["_"]["json_test"] == "hello world"
+    assert "underscore_token" in json_doc
+    assert "underscore_span" in json_doc
+    assert json_doc["underscore_token"]["token_test"][0]["value"] == 117
+    assert json_doc["underscore_span"]["span_test"][0]["value"] == "span_attribute"
+    assert len(schemas.validate(schemas.DocJSONSchema, json_doc)) == 0
+    assert srsly.json_loads(srsly.json_dumps(json_doc)) == json_doc
+
+
+def test_doc_to_json_with_token_span_same_identifier(doc):
+    Doc.set_extension("my_ext", default=False)
+    Token.set_extension("my_ext", default=False)
+    Span.set_extension("my_ext", default=False)
+
+    doc._.my_ext = "hello world"
+    doc[0:1]._.my_ext = "span_attribute"
+    doc[0]._.my_ext = 117
+    json_doc = doc.to_json(underscore=["my_ext"])
+
+    assert "_" in json_doc
+    assert json_doc["_"]["my_ext"] == "hello world"
+    assert "underscore_token" in json_doc
+    assert "underscore_span" in json_doc
+    assert json_doc["underscore_token"]["my_ext"][0]["value"] == 117
+    assert json_doc["underscore_span"]["my_ext"][0]["value"] == "span_attribute"
+    assert len(schemas.validate(schemas.DocJSONSchema, json_doc)) == 0
+    assert srsly.json_loads(srsly.json_dumps(json_doc)) == json_doc
+
+
+def test_doc_to_json_with_token_attributes_missing(doc):
+    Token.set_extension("token_test", default=False)
+    Span.set_extension("span_test", default=False)
+
+    doc[0:1]._.span_test = "span_attribute"
+    doc[0]._.token_test = 117
+    json_doc = doc.to_json(underscore=["span_test"])
+
+    assert "underscore_span" in json_doc
+    assert json_doc["underscore_span"]["span_test"][0]["value"] == "span_attribute"
+    assert "underscore_token" not in json_doc
+    assert len(schemas.validate(schemas.DocJSONSchema, json_doc)) == 0
 
 
 def test_doc_to_json_underscore_error_attr(doc):
@@ -94,11 +228,29 @@ def test_doc_to_json_span(doc):
     assert len(json_doc["spans"]) == 1
     assert len(json_doc["spans"]["test"]) == 2
     assert json_doc["spans"]["test"][0]["start"] == 0
-    assert not schemas.validate(schemas.DocJSONSchema, json_doc)
+    assert len(schemas.validate(schemas.DocJSONSchema, json_doc)) == 0
 
 
 def test_json_to_doc(doc):
-    new_doc = Doc(doc.vocab).from_json(doc.to_json(), validate=True)
+    json_doc = doc.to_json()
+    json_doc = srsly.json_loads(srsly.json_dumps(json_doc))
+    new_doc = Doc(doc.vocab).from_json(json_doc, validate=True)
+    assert new_doc.text == doc.text == "c d e "
+    assert len(new_doc) == len(doc) == 3
+    assert new_doc[0].pos == doc[0].pos
+    assert new_doc[0].tag == doc[0].tag
+    assert new_doc[0].dep == doc[0].dep
+    assert new_doc[0].head.idx == doc[0].head.idx
+    assert new_doc[0].lemma == doc[0].lemma
+    assert len(new_doc.ents) == 1
+    assert new_doc.ents[0].start == 1
+    assert new_doc.ents[0].end == 2
+    assert new_doc.ents[0].label_ == "ORG"
+    assert doc.to_bytes() == new_doc.to_bytes()
+
+
+def test_json_to_doc_compat(doc, doc_json):
+    new_doc = Doc(doc.vocab).from_json(doc_json, validate=True)
     new_tokens = [token for token in new_doc]
     assert new_doc.text == doc.text == "c d e "
     assert len(new_tokens) == len([token for token in doc]) == 3
@@ -114,11 +266,8 @@ def test_json_to_doc(doc):
 
 
 def test_json_to_doc_underscore(doc):
-    if not Doc.has_extension("json_test1"):
     Doc.set_extension("json_test1", default=False)
-    if not Doc.has_extension("json_test2"):
     Doc.set_extension("json_test2", default=False)
-
     doc._.json_test1 = "hello world"
     doc._.json_test2 = [1, 2, 3]
     json_doc = doc.to_json(underscore=["json_test1", "json_test2"])
@@ -126,6 +275,38 @@ def test_json_to_doc_underscore(doc):
     assert all([new_doc.has_extension(f"json_test{i}") for i in range(1, 3)])
     assert new_doc._.json_test1 == "hello world"
     assert new_doc._.json_test2 == [1, 2, 3]
+    assert doc.to_bytes() == new_doc.to_bytes()
+
+
+def test_json_to_doc_with_token_span_attributes(doc):
+    Doc.set_extension("json_test1", default=False)
+    Doc.set_extension("json_test2", default=False)
+    Token.set_extension("token_test", default=False)
+    Span.set_extension("span_test", default=False)
+    doc._.json_test1 = "hello world"
+    doc._.json_test2 = [1, 2, 3]
+    doc[0:1]._.span_test = "span_attribute"
+    doc[0:2]._.span_test = "span_attribute_2"
+    doc[0]._.token_test = 117
+    doc[1]._.token_test = 118
+
+    json_doc = doc.to_json(
+        underscore=["json_test1", "json_test2", "token_test", "span_test"]
+    )
+    json_doc = srsly.json_loads(srsly.json_dumps(json_doc))
+    new_doc = Doc(doc.vocab).from_json(json_doc, validate=True)
+
+    assert all([new_doc.has_extension(f"json_test{i}") for i in range(1, 3)])
+    assert new_doc._.json_test1 == "hello world"
+    assert new_doc._.json_test2 == [1, 2, 3]
+    assert new_doc[0]._.token_test == 117
+    assert new_doc[1]._.token_test == 118
+    assert new_doc[0:1]._.span_test == "span_attribute"
+    assert new_doc[0:2]._.span_test == "span_attribute_2"
+    assert new_doc.user_data == doc.user_data
+    assert new_doc.to_bytes(exclude=["user_data"]) == doc.to_bytes(
+        exclude=["user_data"]
+    )
 
 
 def test_json_to_doc_spans(doc):

spacy/tests/lang/grc/test_tokenizer.py

@@ -0,0 +1,18 @@
+import pytest
+
+
+# fmt: off
+GRC_TOKEN_EXCEPTION_TESTS = [
+    ("τὸ 〈τῆς〉 φιλοσοφίας ἔργον ἔνιοί φασιν ἀπὸ ⟦βαρβάρων⟧ ἄρξαι.", ["τὸ", "〈", "τῆς", "〉", "φιλοσοφίας", "ἔργον", "ἔνιοί", "φασιν", "ἀπὸ", "⟦", "βαρβάρων", "⟧", "ἄρξαι", "."]),
+    ("τὴν δὲ τῶν Αἰγυπτίων φιλοσοφίαν εἶναι τοιαύτην περί τε †θεῶν† καὶ ὑπὲρ δικαιοσύνης.", ["τὴν", "δὲ", "τῶν", "Αἰγυπτίων", "φιλοσοφίαν", "εἶναι", "τοιαύτην", "περί", "τε", "†", "θεῶν", "†", "καὶ", "ὑπὲρ", "δικαιοσύνης", "."]),
+    ("⸏πόσις δ' Ἐρεχθεύς ἐστί μοι σεσωσμένος⸏", ["⸏", "πόσις", "δ'", "Ἐρεχθεύς", "ἐστί", "μοι", "σεσωσμένος", "⸏"]),
+    ("⸏ὔπνον ἴδωμεν⸎", ["⸏", "ὔπνον", "ἴδωμεν", "⸎"]),
+]
+# fmt: on
+
+
+@pytest.mark.parametrize("text,expected_tokens", GRC_TOKEN_EXCEPTION_TESTS)
+def test_grc_tokenizer(grc_tokenizer, text, expected_tokens):
+    tokens = grc_tokenizer(text)
+    token_list = [token.text for token in tokens if not token.is_space]
+    assert expected_tokens == token_list

spacy/tests/lang/la/test_exception.py

@@ -0,0 +1,8 @@
+import pytest
+
+
+def test_la_tokenizer_handles_exc_in_text(la_tokenizer):
+    text = "scio te omnia facturum, ut nobiscum quam primum sis"
+    tokens = la_tokenizer(text)
+    assert len(tokens) == 11
+    assert tokens[6].text == "nobis"

spacy/tests/lang/la/test_text.py

@@ -0,0 +1,35 @@
+import pytest
+from spacy.lang.la.lex_attrs import like_num
+
+
+@pytest.mark.parametrize(
+    "text,match",
+    [
+        ("IIII", True),
+        ("VI", True),
+        ("vi", True),
+        ("IV", True),
+        ("iv", True),
+        ("IX", True),
+        ("ix", True),
+        ("MMXXII", True),
+        ("0", True),
+        ("1", True),
+        ("quattuor", True),
+        ("decem", True),
+        ("tertius", True),
+        ("canis", False),
+        ("MMXX11", False),
+        (",", False),
+    ],
+)
+def test_lex_attrs_like_number(la_tokenizer, text, match):
+    tokens = la_tokenizer(text)
+    assert len(tokens) == 1
+    assert tokens[0].like_num == match
+
+
+@pytest.mark.parametrize("word", ["quinque"])
+def test_la_lex_attrs_capitals(word):
+    assert like_num(word)
+    assert like_num(word.upper())

spacy/tests/lang/lg/test_tokenizer.py

@@ -0,0 +1,15 @@
+import pytest
+
+LG_BASIC_TOKENIZATION_TESTS = [
+    (
+        "Abooluganda ab’emmamba ababiri",
+        ["Abooluganda", "ab’emmamba", "ababiri"],
+    ),
+]
+
+
+@pytest.mark.parametrize("text,expected_tokens", LG_BASIC_TOKENIZATION_TESTS)
+def test_lg_tokenizer_basic(lg_tokenizer, text, expected_tokens):
+    tokens = lg_tokenizer(text)
+    token_list = [token.text for token in tokens if not token.is_space]
+    assert expected_tokens == token_list

spacy/tests/lang/nl/test_noun_chunks.py

@@ -1,5 +1,6 @@
-from spacy.tokens import Doc
 import pytest
+from spacy.tokens import Doc
+from spacy.util import filter_spans
 
 
 @pytest.fixture
@@ -207,3 +208,18 @@ def test_chunking(nl_sample, nl_reference_chunking):
     """
     chunks = [s.text.lower() for s in nl_sample.noun_chunks]
     assert chunks == nl_reference_chunking
+
+
+@pytest.mark.issue(10846)
+def test_no_overlapping_chunks(nl_vocab):
+    # fmt: off
+    doc = Doc(
+        nl_vocab,
+        words=["Dit", "programma", "wordt", "beschouwd", "als", "'s", "werelds", "eerste", "computerprogramma"],
+        deps=["det", "nsubj:pass", "aux:pass", "ROOT", "mark", "det", "fixed", "amod", "xcomp"],
+        heads=[1, 3, 3, 3, 8, 8, 5, 8, 3],
+        pos=["DET", "NOUN", "AUX", "VERB", "SCONJ", "DET", "NOUN", "ADJ", "NOUN"],
+    )
+    # fmt: on
+    chunks = list(doc.noun_chunks)
+    assert filter_spans(chunks) == chunks

spacy/tests/lang/ru/test_lemmatizer.py

@@ -2,6 +2,9 @@ import pytest
 from spacy.tokens import Doc
 
 
+pytestmark = pytest.mark.filterwarnings("ignore::DeprecationWarning")
+
+
 def test_ru_doc_lemmatization(ru_lemmatizer):
     words = ["мама", "мыла", "раму"]
     pos = ["NOUN", "VERB", "NOUN"]
@@ -75,3 +78,17 @@ def test_ru_lemmatizer_punct(ru_lemmatizer):
     assert ru_lemmatizer.pymorphy2_lemmatize(doc[0]) == ['"']
     doc = Doc(ru_lemmatizer.vocab, words=["»"], pos=["PUNCT"])
     assert ru_lemmatizer.pymorphy2_lemmatize(doc[0]) == ['"']
+
+
+def test_ru_doc_lookup_lemmatization(ru_lookup_lemmatizer):
+    words = ["мама", "мыла", "раму"]
+    pos = ["NOUN", "VERB", "NOUN"]
+    morphs = [
+        "Animacy=Anim|Case=Nom|Gender=Fem|Number=Sing",
+        "Aspect=Imp|Gender=Fem|Mood=Ind|Number=Sing|Tense=Past|VerbForm=Fin|Voice=Act",
+        "Animacy=Anim|Case=Acc|Gender=Fem|Number=Sing",
+    ]
+    doc = Doc(ru_lookup_lemmatizer.vocab, words=words, pos=pos, morphs=morphs)
+    doc = ru_lookup_lemmatizer(doc)
+    lemmas = [token.lemma_ for token in doc]
+    assert lemmas == ["мама", "мыла", "раму"]

spacy/tests/lang/sl/test_text.py

@@ -20,7 +20,6 @@ od katerih so te svoboščine odvisne,
     assert len(tokens) == 116
 
 
-@pytest.mark.xfail
 def test_ordinal_number(sl_tokenizer):
     text = "10. decembra 1948"
     tokens = sl_tokenizer(text)

spacy/tests/lang/uk/test_lemmatizer.py

@@ -1,7 +1,19 @@
+import pytest
 from spacy.tokens import Doc
 
 
+pytestmark = pytest.mark.filterwarnings("ignore::DeprecationWarning")
+
+
 def test_uk_lemmatizer(uk_lemmatizer):
     """Check that the default uk lemmatizer runs."""
     doc = Doc(uk_lemmatizer.vocab, words=["a", "b", "c"])
     uk_lemmatizer(doc)
+    assert [token.lemma for token in doc]
+
+
+def test_uk_lookup_lemmatizer(uk_lookup_lemmatizer):
+    """Check that the lookup uk lemmatizer runs."""
+    doc = Doc(uk_lookup_lemmatizer.vocab, words=["a", "b", "c"])
+    uk_lookup_lemmatizer(doc)
+    assert [token.lemma for token in doc]

spacy/tests/matcher/test_levenshtein.py

@@ -0,0 +1,44 @@
+import pytest
+from spacy.matcher import levenshtein
+
+
+# empty string plus 10 random ASCII, 10 random unicode, and 2 random long tests
+# from polyleven
+@pytest.mark.parametrize(
+    "dist,a,b",
+    [
+        (0, "", ""),
+        (4, "bbcb", "caba"),
+        (3, "abcb", "cacc"),
+        (3, "aa", "ccc"),
+        (1, "cca", "ccac"),
+        (1, "aba", "aa"),
+        (4, "bcbb", "abac"),
+        (3, "acbc", "bba"),
+        (3, "cbba", "a"),
+        (2, "bcc", "ba"),
+        (4, "aaa", "ccbb"),
+        (3, "うあい", "いいうい"),
+        (2, "あううい", "うあい"),
+        (3, "いういい", "うううあ"),
+        (2, "うい", "あいあ"),
+        (2, "いあい", "いう"),
+        (1, "いい", "あいい"),
+        (3, "あうあ", "いいああ"),
+        (4, "いあうう", "ううああ"),
+        (3, "いあいい", "ういああ"),
+        (3, "いいああ", "ううあう"),
+        (
+            166,
+            "TCTGGGCACGGATTCGTCAGATTCCATGTCCATATTTGAGGCTCTTGCAGGCAAAATTTGGGCATGTGAACTCCTTATAGTCCCCGTGC",
+            "ATATGGATTGGGGGCATTCAAAGATACGGTTTCCCTTTCTTCAGTTTCGCGCGGCGCACGTCCGGGTGCGAGCCAGTTCGTCTTACTCACATTGTCGACTTCACGAATCGCGCATGATGTGCTTAGCCTGTACTTACGAACGAACTTTCGGTCCAAATACATTCTATCAACACCGAGGTATCCGTGCCACACGCCGAAGCTCGACCGTGTTCGTTGAGAGGTGGAAATGGTAAAAGATGAACATAGTC",
+        ),
+        (
+            111,
+            "GGTTCGGCCGAATTCATAGAGCGTGGTAGTCGACGGTATCCCGCCTGGTAGGGGCCCCTTCTACCTAGCGGAAGTTTGTCAGTACTCTATAACACGAGGGCCTCTCACACCCTAGATCGTCCAGCCACTCGAAGATCGCAGCACCCTTACAGAAAGGCATTAATGTTTCTCCTAGCACTTGTGCAATGGTGAAGGAGTGATG",
+            "CGTAACACTTCGCGCTACTGGGCTGCAACGTCTTGGGCATACATGCAAGATTATCTAATGCAAGCTTGAGCCCCGCTTGCGGAATTTCCCTAATCGGGGTCCCTTCCTGTTACGATAAGGACGCGTGCACT",
+        ),
+    ],
+)
+def test_levenshtein(dist, a, b):
+    assert levenshtein(a, b) == dist

spacy/tests/matcher/test_matcher_api.py

@@ -368,6 +368,16 @@ def test_matcher_intersect_value_operator(en_vocab):
     doc[0]._.ext = ["A", "B"]
     assert len(matcher(doc)) == 1
 
+    # INTERSECTS matches nothing for iterables that aren't all str or int
+    matcher = Matcher(en_vocab)
+    pattern = [{"_": {"ext": {"INTERSECTS": ["Abx", "C"]}}}]
+    matcher.add("M", [pattern])
+    doc = Doc(en_vocab, words=["a", "b", "c"])
+    doc[0]._.ext = [["Abx"], "B"]
+    assert len(matcher(doc)) == 0
+    doc[0]._.ext = ["Abx", "B"]
+    assert len(matcher(doc)) == 1
+
     # INTERSECTS with an empty pattern list matches nothing
     matcher = Matcher(en_vocab)
     pattern = [{"_": {"ext": {"INTERSECTS": []}}}]
@@ -476,14 +486,22 @@ def test_matcher_extension_set_membership(en_vocab):
     assert len(matches) == 0
 
 
-@pytest.mark.xfail(reason="IN predicate must handle sequence values in extensions")
 def test_matcher_extension_in_set_predicate(en_vocab):
     matcher = Matcher(en_vocab)
     Token.set_extension("ext", default=[])
     pattern = [{"_": {"ext": {"IN": ["A", "C"]}}}]
     matcher.add("M", [pattern])
     doc = Doc(en_vocab, words=["a", "b", "c"])
+
+    # The IN predicate expects an exact match between the
+    # extension value and one of the pattern's values.
     doc[0]._.ext = ["A", "B"]
+    assert len(matcher(doc)) == 0
+
+    doc[0]._.ext = ["A"]
+    assert len(matcher(doc)) == 0
+
+    doc[0]._.ext = "A"
     assert len(matcher(doc)) == 1
 
 

spacy/tests/matcher/test_phrase_matcher.py

@@ -1,4 +1,5 @@
 import pytest
+import warnings
 import srsly
 from mock import Mock
 
@@ -344,13 +345,13 @@ def test_phrase_matcher_validation(en_vocab):
         matcher.add("TEST1", [doc1])
     with pytest.warns(UserWarning):
         matcher.add("TEST2", [doc2])
-    with pytest.warns(None) as record:
+    with warnings.catch_warnings():
+        warnings.simplefilter("error")
         matcher.add("TEST3", [doc3])
-        assert not record.list
     matcher = PhraseMatcher(en_vocab, attr="POS", validate=True)
-    with pytest.warns(None) as record:
+    with warnings.catch_warnings():
+        warnings.simplefilter("error")
         matcher.add("TEST4", [doc2])
-        assert not record.list
 
 
 def test_attr_validation(en_vocab):

spacy/tests/package/test_requirements.py

@@ -17,6 +17,7 @@ def test_build_dependencies():
         "types-dataclasses",
         "types-mock",
         "types-requests",
+        "types-setuptools",
     ]
     # ignore language-specific packages that shouldn't be installed by all
     libs_ignore_setup = [

spacy/tests/pipeline/test_entity_linker.py

@@ -6,7 +6,7 @@ from numpy.testing import assert_equal
 from spacy import registry, util
 from spacy.attrs import ENT_KB_ID
 from spacy.compat import pickle
-from spacy.kb import Candidate, KnowledgeBase, get_candidates
+from spacy.kb import Candidate, InMemoryLookupKB, get_candidates, KnowledgeBase
 from spacy.lang.en import English
 from spacy.ml import load_kb
 from spacy.pipeline import EntityLinker
@@ -34,7 +34,7 @@ def assert_almost_equal(a, b):
 def test_issue4674():
     """Test that setting entities with overlapping identifiers does not mess up IO"""
     nlp = English()
-    kb = KnowledgeBase(nlp.vocab, entity_vector_length=3)
+    kb = InMemoryLookupKB(nlp.vocab, entity_vector_length=3)
     vector1 = [0.9, 1.1, 1.01]
     vector2 = [1.8, 2.25, 2.01]
     with pytest.warns(UserWarning):
@@ -51,7 +51,7 @@ def test_issue4674():
             dir_path.mkdir()
         file_path = dir_path / "kb"
         kb.to_disk(str(file_path))
-        kb2 = KnowledgeBase(nlp.vocab, entity_vector_length=3)
+        kb2 = InMemoryLookupKB(nlp.vocab, entity_vector_length=3)
         kb2.from_disk(str(file_path))
     assert kb2.get_size_entities() == 1
 
@@ -59,9 +59,9 @@ def test_issue4674():
 @pytest.mark.issue(6730)
 def test_issue6730(en_vocab):
     """Ensure that the KB does not accept empty strings, but otherwise IO works fine."""
-    from spacy.kb import KnowledgeBase
+    from spacy.kb.kb_in_memory import InMemoryLookupKB
 
-    kb = KnowledgeBase(en_vocab, entity_vector_length=3)
+    kb = InMemoryLookupKB(en_vocab, entity_vector_length=3)
     kb.add_entity(entity="1", freq=148, entity_vector=[1, 2, 3])
 
     with pytest.raises(ValueError):
@@ -127,7 +127,7 @@ def test_issue7065_b():
 
     def create_kb(vocab):
         # create artificial KB
-        mykb = KnowledgeBase(vocab, entity_vector_length=vector_length)
+        mykb = InMemoryLookupKB(vocab, entity_vector_length=vector_length)
         mykb.add_entity(entity="Q270853", freq=12, entity_vector=[9, 1, -7])
         mykb.add_alias(
             alias="No. 8",
@@ -190,7 +190,7 @@ def test_no_entities():
 
     def create_kb(vocab):
         # create artificial KB
-        mykb = KnowledgeBase(vocab, entity_vector_length=vector_length)
+        mykb = InMemoryLookupKB(vocab, entity_vector_length=vector_length)
         mykb.add_entity(entity="Q2146908", freq=12, entity_vector=[6, -4, 3])
         mykb.add_alias("Russ Cochran", ["Q2146908"], [0.9])
         return mykb
@@ -231,7 +231,7 @@ def test_partial_links():
 
     def create_kb(vocab):
         # create artificial KB
-        mykb = KnowledgeBase(vocab, entity_vector_length=vector_length)
+        mykb = InMemoryLookupKB(vocab, entity_vector_length=vector_length)
         mykb.add_entity(entity="Q2146908", freq=12, entity_vector=[6, -4, 3])
         mykb.add_alias("Russ Cochran", ["Q2146908"], [0.9])
         return mykb
@@ -263,7 +263,7 @@ def test_partial_links():
 
 def test_kb_valid_entities(nlp):
     """Test the valid construction of a KB with 3 entities and two aliases"""
-    mykb = KnowledgeBase(nlp.vocab, entity_vector_length=3)
+    mykb = InMemoryLookupKB(nlp.vocab, entity_vector_length=3)
 
     # adding entities
     mykb.add_entity(entity="Q1", freq=19, entity_vector=[8, 4, 3])
@@ -292,7 +292,7 @@ def test_kb_valid_entities(nlp):
 
 def test_kb_invalid_entities(nlp):
     """Test the invalid construction of a KB with an alias linked to a non-existing entity"""
-    mykb = KnowledgeBase(nlp.vocab, entity_vector_length=1)
+    mykb = InMemoryLookupKB(nlp.vocab, entity_vector_length=1)
 
     # adding entities
     mykb.add_entity(entity="Q1", freq=19, entity_vector=[1])
@@ -308,7 +308,7 @@ def test_kb_invalid_entities(nlp):
 
 def test_kb_invalid_probabilities(nlp):
     """Test the invalid construction of a KB with wrong prior probabilities"""
-    mykb = KnowledgeBase(nlp.vocab, entity_vector_length=1)
+    mykb = InMemoryLookupKB(nlp.vocab, entity_vector_length=1)
 
     # adding entities
     mykb.add_entity(entity="Q1", freq=19, entity_vector=[1])
@@ -322,7 +322,7 @@ def test_kb_invalid_probabilities(nlp):
 
 def test_kb_invalid_combination(nlp):
     """Test the invalid construction of a KB with non-matching entity and probability lists"""
-    mykb = KnowledgeBase(nlp.vocab, entity_vector_length=1)
+    mykb = InMemoryLookupKB(nlp.vocab, entity_vector_length=1)
 
     # adding entities
     mykb.add_entity(entity="Q1", freq=19, entity_vector=[1])
@@ -338,7 +338,7 @@ def test_kb_invalid_combination(nlp):
 
 def test_kb_invalid_entity_vector(nlp):
     """Test the invalid construction of a KB with non-matching entity vector lengths"""
-    mykb = KnowledgeBase(nlp.vocab, entity_vector_length=3)
+    mykb = InMemoryLookupKB(nlp.vocab, entity_vector_length=3)
 
     # adding entities
     mykb.add_entity(entity="Q1", freq=19, entity_vector=[1, 2, 3])
@@ -376,7 +376,7 @@ def test_kb_initialize_empty(nlp):
 
 def test_kb_serialize(nlp):
     """Test serialization of the KB"""
-    mykb = KnowledgeBase(nlp.vocab, entity_vector_length=1)
+    mykb = InMemoryLookupKB(nlp.vocab, entity_vector_length=1)
     with make_tempdir() as d:
         # normal read-write behaviour
         mykb.to_disk(d / "kb")
@@ -393,12 +393,12 @@ def test_kb_serialize(nlp):
 @pytest.mark.issue(9137)
 def test_kb_serialize_2(nlp):
     v = [5, 6, 7, 8]
-    kb1 = KnowledgeBase(vocab=nlp.vocab, entity_vector_length=4)
+    kb1 = InMemoryLookupKB(vocab=nlp.vocab, entity_vector_length=4)
     kb1.set_entities(["E1"], [1], [v])
     assert kb1.get_vector("E1") == v
     with make_tempdir() as d:
         kb1.to_disk(d / "kb")
-        kb2 = KnowledgeBase(vocab=nlp.vocab, entity_vector_length=4)
+        kb2 = InMemoryLookupKB(vocab=nlp.vocab, entity_vector_length=4)
         kb2.from_disk(d / "kb")
         assert kb2.get_vector("E1") == v
 
@@ -408,7 +408,7 @@ def test_kb_set_entities(nlp):
     v = [5, 6, 7, 8]
     v1 = [1, 1, 1, 0]
     v2 = [2, 2, 2, 3]
-    kb1 = KnowledgeBase(vocab=nlp.vocab, entity_vector_length=4)
+    kb1 = InMemoryLookupKB(vocab=nlp.vocab, entity_vector_length=4)
     kb1.set_entities(["E0"], [1], [v])
     assert kb1.get_entity_strings() == ["E0"]
     kb1.set_entities(["E1", "E2"], [1, 9], [v1, v2])
@@ -417,7 +417,7 @@ def test_kb_set_entities(nlp):
     assert kb1.get_vector("E2") == v2
     with make_tempdir() as d:
         kb1.to_disk(d / "kb")
-        kb2 = KnowledgeBase(vocab=nlp.vocab, entity_vector_length=4)
+        kb2 = InMemoryLookupKB(vocab=nlp.vocab, entity_vector_length=4)
         kb2.from_disk(d / "kb")
         assert set(kb2.get_entity_strings()) == {"E1", "E2"}
         assert kb2.get_vector("E1") == v1
@@ -428,7 +428,7 @@ def test_kb_serialize_vocab(nlp):
     """Test serialization of the KB and custom strings"""
     entity = "MyFunnyID"
     assert entity not in nlp.vocab.strings
-    mykb = KnowledgeBase(nlp.vocab, entity_vector_length=1)
+    mykb = InMemoryLookupKB(nlp.vocab, entity_vector_length=1)
     assert not mykb.contains_entity(entity)
     mykb.add_entity(entity, freq=342, entity_vector=[3])
     assert mykb.contains_entity(entity)
@@ -436,14 +436,14 @@ def test_kb_serialize_vocab(nlp):
     with make_tempdir() as d:
         # normal read-write behaviour
         mykb.to_disk(d / "kb")
-        mykb_new = KnowledgeBase(Vocab(), entity_vector_length=1)
+        mykb_new = InMemoryLookupKB(Vocab(), entity_vector_length=1)
         mykb_new.from_disk(d / "kb")
         assert entity in mykb_new.vocab.strings
 
 
 def test_candidate_generation(nlp):
     """Test correct candidate generation"""
-    mykb = KnowledgeBase(nlp.vocab, entity_vector_length=1)
+    mykb = InMemoryLookupKB(nlp.vocab, entity_vector_length=1)
     doc = nlp("douglas adam Adam shrubbery")
 
     douglas_ent = doc[0:1]
@@ -481,7 +481,7 @@ def test_el_pipe_configuration(nlp):
     ruler.add_patterns([pattern])
 
     def create_kb(vocab):
-        kb = KnowledgeBase(vocab, entity_vector_length=1)
+        kb = InMemoryLookupKB(vocab, entity_vector_length=1)
         kb.add_entity(entity="Q2", freq=12, entity_vector=[2])
         kb.add_entity(entity="Q3", freq=5, entity_vector=[3])
         kb.add_alias(alias="douglas", entities=["Q2", "Q3"], probabilities=[0.8, 0.1])
@@ -500,10 +500,21 @@ def test_el_pipe_configuration(nlp):
     def get_lowercased_candidates(kb, span):
         return kb.get_alias_candidates(span.text.lower())
 
+    def get_lowercased_candidates_batch(kb, spans):
+        return [get_lowercased_candidates(kb, span) for span in spans]
+
     @registry.misc("spacy.LowercaseCandidateGenerator.v1")
-    def create_candidates() -> Callable[[KnowledgeBase, "Span"], Iterable[Candidate]]:
+    def create_candidates() -> Callable[
+        [InMemoryLookupKB, "Span"], Iterable[Candidate]
+    ]:
         return get_lowercased_candidates
 
+    @registry.misc("spacy.LowercaseCandidateBatchGenerator.v1")
+    def create_candidates_batch() -> Callable[
+        [InMemoryLookupKB, Iterable["Span"]], Iterable[Iterable[Candidate]]
+    ]:
+        return get_lowercased_candidates_batch
+
     # replace the pipe with a new one with with a different candidate generator
     entity_linker = nlp.replace_pipe(
         "entity_linker",
@@ -511,6 +522,9 @@ def test_el_pipe_configuration(nlp):
         config={
             "incl_context": False,
             "get_candidates": {"@misc": "spacy.LowercaseCandidateGenerator.v1"},
+            "get_candidates_batch": {
+                "@misc": "spacy.LowercaseCandidateBatchGenerator.v1"
+            },
         },
     )
     entity_linker.set_kb(create_kb)
@@ -532,7 +546,7 @@ def test_nel_nsents(nlp):
 
 def test_vocab_serialization(nlp):
     """Test that string information is retained across storage"""
-    mykb = KnowledgeBase(nlp.vocab, entity_vector_length=1)
+    mykb = InMemoryLookupKB(nlp.vocab, entity_vector_length=1)
 
     # adding entities
     mykb.add_entity(entity="Q1", freq=27, entity_vector=[1])
@@ -552,7 +566,7 @@ def test_vocab_serialization(nlp):
 
     with make_tempdir() as d:
         mykb.to_disk(d / "kb")
-        kb_new_vocab = KnowledgeBase(Vocab(), entity_vector_length=1)
+        kb_new_vocab = InMemoryLookupKB(Vocab(), entity_vector_length=1)
         kb_new_vocab.from_disk(d / "kb")
 
         candidates = kb_new_vocab.get_alias_candidates("adam")
@@ -568,7 +582,7 @@ def test_vocab_serialization(nlp):
 
 def test_append_alias(nlp):
     """Test that we can append additional alias-entity pairs"""
-    mykb = KnowledgeBase(nlp.vocab, entity_vector_length=1)
+    mykb = InMemoryLookupKB(nlp.vocab, entity_vector_length=1)
 
     # adding entities
     mykb.add_entity(entity="Q1", freq=27, entity_vector=[1])
@@ -599,7 +613,7 @@ def test_append_alias(nlp):
 @pytest.mark.filterwarnings("ignore:\\[W036")
 def test_append_invalid_alias(nlp):
     """Test that append an alias will throw an error if prior probs are exceeding 1"""
-    mykb = KnowledgeBase(nlp.vocab, entity_vector_length=1)
+    mykb = InMemoryLookupKB(nlp.vocab, entity_vector_length=1)
 
     # adding entities
     mykb.add_entity(entity="Q1", freq=27, entity_vector=[1])
@@ -621,7 +635,7 @@ def test_preserving_links_asdoc(nlp):
     vector_length = 1
 
     def create_kb(vocab):
-        mykb = KnowledgeBase(vocab, entity_vector_length=vector_length)
+        mykb = InMemoryLookupKB(vocab, entity_vector_length=vector_length)
         # adding entities
         mykb.add_entity(entity="Q1", freq=19, entity_vector=[1])
         mykb.add_entity(entity="Q2", freq=8, entity_vector=[1])
@@ -723,7 +737,7 @@ def test_overfitting_IO():
         # create artificial KB - assign same prior weight to the two russ cochran's
         # Q2146908 (Russ Cochran): American golfer
         # Q7381115 (Russ Cochran): publisher
-        mykb = KnowledgeBase(vocab, entity_vector_length=vector_length)
+        mykb = InMemoryLookupKB(vocab, entity_vector_length=vector_length)
         mykb.add_entity(entity="Q2146908", freq=12, entity_vector=[6, -4, 3])
         mykb.add_entity(entity="Q7381115", freq=12, entity_vector=[9, 1, -7])
         mykb.add_alias(
@@ -805,7 +819,7 @@ def test_kb_serialization():
         kb_dir = tmp_dir / "kb"
         nlp1 = English()
         assert "Q2146908" not in nlp1.vocab.strings
-        mykb = KnowledgeBase(nlp1.vocab, entity_vector_length=vector_length)
+        mykb = InMemoryLookupKB(nlp1.vocab, entity_vector_length=vector_length)
         mykb.add_entity(entity="Q2146908", freq=12, entity_vector=[6, -4, 3])
         mykb.add_alias(alias="Russ Cochran", entities=["Q2146908"], probabilities=[0.8])
         assert "Q2146908" in nlp1.vocab.strings
@@ -828,7 +842,7 @@ def test_kb_serialization():
 def test_kb_pickle():
     # Test that the KB can be pickled
     nlp = English()
-    kb_1 = KnowledgeBase(nlp.vocab, entity_vector_length=3)
+    kb_1 = InMemoryLookupKB(nlp.vocab, entity_vector_length=3)
     kb_1.add_entity(entity="Q2146908", freq=12, entity_vector=[6, -4, 3])
     assert not kb_1.contains_alias("Russ Cochran")
     kb_1.add_alias(alias="Russ Cochran", entities=["Q2146908"], probabilities=[0.8])
@@ -842,7 +856,7 @@ def test_kb_pickle():
 def test_nel_pickle():
     # Test that a pipeline with an EL component can be pickled
     def create_kb(vocab):
-        kb = KnowledgeBase(vocab, entity_vector_length=3)
+        kb = InMemoryLookupKB(vocab, entity_vector_length=3)
         kb.add_entity(entity="Q2146908", freq=12, entity_vector=[6, -4, 3])
         kb.add_alias(alias="Russ Cochran", entities=["Q2146908"], probabilities=[0.8])
         return kb
@@ -864,7 +878,7 @@ def test_nel_pickle():
 def test_kb_to_bytes():
     # Test that the KB's to_bytes method works correctly
     nlp = English()
-    kb_1 = KnowledgeBase(nlp.vocab, entity_vector_length=3)
+    kb_1 = InMemoryLookupKB(nlp.vocab, entity_vector_length=3)
     kb_1.add_entity(entity="Q2146908", freq=12, entity_vector=[6, -4, 3])
     kb_1.add_entity(entity="Q66", freq=9, entity_vector=[1, 2, 3])
     kb_1.add_alias(alias="Russ Cochran", entities=["Q2146908"], probabilities=[0.8])
@@ -874,7 +888,7 @@ def test_kb_to_bytes():
     )
     assert kb_1.contains_alias("Russ Cochran")
     kb_bytes = kb_1.to_bytes()
-    kb_2 = KnowledgeBase(nlp.vocab, entity_vector_length=3)
+    kb_2 = InMemoryLookupKB(nlp.vocab, entity_vector_length=3)
     assert not kb_2.contains_alias("Russ Cochran")
     kb_2 = kb_2.from_bytes(kb_bytes)
     # check that both KBs are exactly the same
@@ -897,7 +911,7 @@ def test_kb_to_bytes():
 def test_nel_to_bytes():
     # Test that a pipeline with an EL component can be converted to bytes
     def create_kb(vocab):
-        kb = KnowledgeBase(vocab, entity_vector_length=3)
+        kb = InMemoryLookupKB(vocab, entity_vector_length=3)
         kb.add_entity(entity="Q2146908", freq=12, entity_vector=[6, -4, 3])
         kb.add_alias(alias="Russ Cochran", entities=["Q2146908"], probabilities=[0.8])
         return kb
@@ -987,7 +1001,7 @@ def test_legacy_architectures(name, config):
         train_examples.append(Example.from_dict(doc, annotation))
 
     def create_kb(vocab):
-        mykb = KnowledgeBase(vocab, entity_vector_length=vector_length)
+        mykb = InMemoryLookupKB(vocab, entity_vector_length=vector_length)
         mykb.add_entity(entity="Q2146908", freq=12, entity_vector=[6, -4, 3])
         mykb.add_entity(entity="Q7381115", freq=12, entity_vector=[9, 1, -7])
         mykb.add_alias(
@@ -1048,9 +1062,13 @@ def test_no_gold_ents(patterns):
     for eg in train_examples:
         eg.predicted = ruler(eg.predicted)
 
+    # Entity ruler is no longer needed (initialization below wipes out the
+    # patterns and causes warnings)
+    nlp.remove_pipe("entity_ruler")
+
     def create_kb(vocab):
         # create artificial KB
-        mykb = KnowledgeBase(vocab, entity_vector_length=vector_length)
+        mykb = InMemoryLookupKB(vocab, entity_vector_length=vector_length)
         mykb.add_entity(entity="Q613241", freq=12, entity_vector=[6, -4, 3])
         mykb.add_alias("Kirby", ["Q613241"], [0.9])
         # Placeholder
@@ -1100,7 +1118,7 @@ def test_tokenization_mismatch():
 
     def create_kb(vocab):
         # create placeholder KB
-        mykb = KnowledgeBase(vocab, entity_vector_length=vector_length)
+        mykb = InMemoryLookupKB(vocab, entity_vector_length=vector_length)
         mykb.add_entity(entity="Q613241", freq=12, entity_vector=[6, -4, 3])
         mykb.add_alias("Kirby", ["Q613241"], [0.9])
         return mykb
@@ -1117,6 +1135,12 @@ def test_tokenization_mismatch():
     nlp.evaluate(train_examples)
 
 
+def test_abstract_kb_instantiation():
+    """Test whether instantiation of abstract KB base class fails."""
+    with pytest.raises(TypeError):
+        KnowledgeBase(None, 3)
+
+
 # fmt: off
 @pytest.mark.parametrize(
     "meet_threshold,config",
@@ -1147,7 +1171,7 @@ def test_threshold(meet_threshold: bool, config: Dict[str, Any]):
 
     def create_kb(vocab):
         # create artificial KB
-        mykb = KnowledgeBase(vocab, entity_vector_length=3)
+        mykb = InMemoryLookupKB(vocab, entity_vector_length=3)
         mykb.add_entity(entity=entity_id, freq=12, entity_vector=[6, -4, 3])
         mykb.add_alias(
             alias="Mahler",

spacy/tests/pipeline/test_pipe_methods.py

@@ -605,10 +605,35 @@ def test_update_with_annotates():
             assert results[component] == ""
 
 
-def test_load_disable_enable() -> None:
+@pytest.mark.issue(11443)
-    """
+def test_enable_disable_conflict_with_config():
-    Tests spacy.load() with dis-/enabling components.
+    """Test conflict between enable/disable w.r.t. `nlp.disabled` set in the config."""
-    """
+    nlp = English()
+    nlp.add_pipe("tagger")
+    nlp.add_pipe("senter")
+    nlp.add_pipe("sentencizer")
+
+    with make_tempdir() as tmp_dir:
+        nlp.to_disk(tmp_dir)
+        # Expected to fail, as config and arguments conflict.
+        with pytest.raises(ValueError):
+            spacy.load(
+                tmp_dir, enable=["tagger"], config={"nlp": {"disabled": ["senter"]}}
+            )
+        # Expected to succeed without warning due to the lack of a conflicting config option.
+        spacy.load(tmp_dir, enable=["tagger"])
+        # Expected to succeed with a warning, as disable=[] should override the config setting.
+        with pytest.warns(UserWarning):
+            spacy.load(
+                tmp_dir,
+                enable=["tagger"],
+                disable=[],
+                config={"nlp": {"disabled": ["senter"]}},
+            )
+
+
+def test_load_disable_enable():
+    """Tests spacy.load() with dis-/enabling components."""
 
     base_nlp = English()
     for pipe in ("sentencizer", "tagger", "parser"):
@@ -618,6 +643,7 @@ def test_load_disable_enable() -> None:
         base_nlp.to_disk(tmp_dir)
         to_disable = ["parser", "tagger"]
         to_enable = ["tagger", "parser"]
+        single_str = "tagger"
 
         # Setting only `disable`.
         nlp = spacy.load(tmp_dir, disable=to_disable)
@@ -632,6 +658,16 @@ def test_load_disable_enable() -> None:
             ]
         )
 
+        # Loading with a string representing one component
+        nlp = spacy.load(tmp_dir, exclude=single_str)
+        assert single_str not in nlp.component_names
+
+        nlp = spacy.load(tmp_dir, disable=single_str)
+        assert single_str in nlp.component_names
+        assert single_str not in nlp.pipe_names
+        assert nlp._disabled == {single_str}
+        assert nlp.disabled == [single_str]
+
         # Testing consistent enable/disable combination.
         nlp = spacy.load(
             tmp_dir,

spacy/tests/pipeline/test_tok2vec.py

@@ -230,6 +230,87 @@ def test_tok2vec_listener_callback():
     assert get_dX(Y) is not None
 
 
+def test_tok2vec_listener_overfitting():
+    """ Test that a pipeline with a listener properly overfits, even if 'tok2vec' is in the annotating components """
+    orig_config = Config().from_str(cfg_string)
+    nlp = util.load_model_from_config(orig_config, auto_fill=True, validate=True)
+    train_examples = []
+    for t in TRAIN_DATA:
+        train_examples.append(Example.from_dict(nlp.make_doc(t[0]), t[1]))
+    optimizer = nlp.initialize(get_examples=lambda: train_examples)
+
+    for i in range(50):
+        losses = {}
+        nlp.update(train_examples, sgd=optimizer, losses=losses, annotates=["tok2vec"])
+    assert losses["tagger"] < 0.00001
+
+    # test the trained model
+    test_text = "I like blue eggs"
+    doc = nlp(test_text)
+    assert doc[0].tag_ == "N"
+    assert doc[1].tag_ == "V"
+    assert doc[2].tag_ == "J"
+    assert doc[3].tag_ == "N"
+
+    # Also test the results are still the same after IO
+    with make_tempdir() as tmp_dir:
+        nlp.to_disk(tmp_dir)
+        nlp2 = util.load_model_from_path(tmp_dir)
+        doc2 = nlp2(test_text)
+        assert doc2[0].tag_ == "N"
+        assert doc2[1].tag_ == "V"
+        assert doc2[2].tag_ == "J"
+        assert doc2[3].tag_ == "N"
+
+
+def test_tok2vec_frozen_not_annotating():
+    """ Test that a pipeline with a frozen tok2vec raises an error when the tok2vec is not annotating """
+    orig_config = Config().from_str(cfg_string)
+    nlp = util.load_model_from_config(orig_config, auto_fill=True, validate=True)
+    train_examples = []
+    for t in TRAIN_DATA:
+        train_examples.append(Example.from_dict(nlp.make_doc(t[0]), t[1]))
+    optimizer = nlp.initialize(get_examples=lambda: train_examples)
+
+    for i in range(2):
+        losses = {}
+        with pytest.raises(ValueError, match=r"the tok2vec embedding layer is not updated"):
+            nlp.update(train_examples, sgd=optimizer, losses=losses, exclude=["tok2vec"])
+
+
+def test_tok2vec_frozen_overfitting():
+    """ Test that a pipeline with a frozen & annotating tok2vec can still overfit """
+    orig_config = Config().from_str(cfg_string)
+    nlp = util.load_model_from_config(orig_config, auto_fill=True, validate=True)
+    train_examples = []
+    for t in TRAIN_DATA:
+        train_examples.append(Example.from_dict(nlp.make_doc(t[0]), t[1]))
+    optimizer = nlp.initialize(get_examples=lambda: train_examples)
+
+    for i in range(100):
+        losses = {}
+        nlp.update(train_examples, sgd=optimizer, losses=losses, exclude=["tok2vec"], annotates=["tok2vec"])
+    assert losses["tagger"] < 0.0001
+
+    # test the trained model
+    test_text = "I like blue eggs"
+    doc = nlp(test_text)
+    assert doc[0].tag_ == "N"
+    assert doc[1].tag_ == "V"
+    assert doc[2].tag_ == "J"
+    assert doc[3].tag_ == "N"
+
+    # Also test the results are still the same after IO
+    with make_tempdir() as tmp_dir:
+        nlp.to_disk(tmp_dir)
+        nlp2 = util.load_model_from_path(tmp_dir)
+        doc2 = nlp2(test_text)
+        assert doc2[0].tag_ == "N"
+        assert doc2[1].tag_ == "V"
+        assert doc2[2].tag_ == "J"
+        assert doc2[3].tag_ == "N"
+
+
 def test_replace_listeners():
     orig_config = Config().from_str(cfg_string)
     nlp = util.load_model_from_config(orig_config, auto_fill=True, validate=True)

spacy/tests/serialize/test_resource_warning.py

@@ -3,7 +3,7 @@ from unittest import TestCase
 import pytest
 import srsly
 from numpy import zeros
-from spacy.kb import KnowledgeBase, Writer
+from spacy.kb.kb_in_memory import InMemoryLookupKB, Writer
 from spacy.vectors import Vectors
 from spacy.language import Language
 from spacy.pipeline import TrainablePipe
@@ -71,7 +71,7 @@ def entity_linker():
     nlp = Language()
 
     def create_kb(vocab):
-        kb = KnowledgeBase(vocab, entity_vector_length=1)
+        kb = InMemoryLookupKB(vocab, entity_vector_length=1)
         kb.add_entity("test", 0.0, zeros((1, 1), dtype="f"))
         return kb
 
@@ -120,7 +120,7 @@ def test_writer_with_path_py35():
 
 def test_save_and_load_knowledge_base():
     nlp = Language()
-    kb = KnowledgeBase(nlp.vocab, entity_vector_length=1)
+    kb = InMemoryLookupKB(nlp.vocab, entity_vector_length=1)
     with make_tempdir() as d:
         path = d / "kb"
         try:
@@ -129,7 +129,7 @@ def test_save_and_load_knowledge_base():
             pytest.fail(str(e))
 
         try:
-            kb_loaded = KnowledgeBase(nlp.vocab, entity_vector_length=1)
+            kb_loaded = InMemoryLookupKB(nlp.vocab, entity_vector_length=1)
             kb_loaded.from_disk(path)
         except Exception as e:
             pytest.fail(str(e))

spacy/tests/serialize/test_serialize_kb.py

@@ -2,7 +2,7 @@ from typing import Callable
 
 from spacy import util
 from spacy.util import ensure_path, registry, load_model_from_config
-from spacy.kb import KnowledgeBase
+from spacy.kb.kb_in_memory import InMemoryLookupKB
 from spacy.vocab import Vocab
 from thinc.api import Config
 
@@ -22,7 +22,7 @@ def test_serialize_kb_disk(en_vocab):
             dir_path.mkdir()
         file_path = dir_path / "kb"
         kb1.to_disk(str(file_path))
-        kb2 = KnowledgeBase(vocab=en_vocab, entity_vector_length=3)
+        kb2 = InMemoryLookupKB(vocab=en_vocab, entity_vector_length=3)
         kb2.from_disk(str(file_path))
 
     # final assertions
@@ -30,7 +30,7 @@ def test_serialize_kb_disk(en_vocab):
 
 
 def _get_dummy_kb(vocab):
-    kb = KnowledgeBase(vocab, entity_vector_length=3)
+    kb = InMemoryLookupKB(vocab, entity_vector_length=3)
     kb.add_entity(entity="Q53", freq=33, entity_vector=[0, 5, 3])
     kb.add_entity(entity="Q17", freq=2, entity_vector=[7, 1, 0])
     kb.add_entity(entity="Q007", freq=7, entity_vector=[0, 0, 7])
@@ -104,7 +104,7 @@ def test_serialize_subclassed_kb():
     custom_field = 666
     """
 
-    class SubKnowledgeBase(KnowledgeBase):
+    class SubInMemoryLookupKB(InMemoryLookupKB):
         def __init__(self, vocab, entity_vector_length, custom_field):
             super().__init__(vocab, entity_vector_length)
             self.custom_field = custom_field
@@ -112,9 +112,9 @@ def test_serialize_subclassed_kb():
     @registry.misc("spacy.CustomKB.v1")
     def custom_kb(
         entity_vector_length: int, custom_field: int
-    ) -> Callable[[Vocab], KnowledgeBase]:
+    ) -> Callable[[Vocab], InMemoryLookupKB]:
         def custom_kb_factory(vocab):
-            kb = SubKnowledgeBase(
+            kb = SubInMemoryLookupKB(
                 vocab=vocab,
                 entity_vector_length=entity_vector_length,
                 custom_field=custom_field,
@@ -129,7 +129,7 @@ def test_serialize_subclassed_kb():
     nlp.initialize()
 
     entity_linker = nlp.get_pipe("entity_linker")
-    assert type(entity_linker.kb) == SubKnowledgeBase
+    assert type(entity_linker.kb) == SubInMemoryLookupKB
     assert entity_linker.kb.entity_vector_length == 342
     assert entity_linker.kb.custom_field == 666
 
@@ -139,6 +139,6 @@ def test_serialize_subclassed_kb():
         nlp2 = util.load_model_from_path(tmp_dir)
         entity_linker2 = nlp2.get_pipe("entity_linker")
         # After IO, the KB is the standard one
-        assert type(entity_linker2.kb) == KnowledgeBase
+        assert type(entity_linker2.kb) == InMemoryLookupKB
         assert entity_linker2.kb.entity_vector_length == 342
         assert not hasattr(entity_linker2.kb, "custom_field")

spacy/tests/serialize/test_serialize_pipeline.py

@@ -404,10 +404,11 @@ def test_serialize_pipeline_disable_enable():
     assert nlp3.component_names == ["ner", "tagger"]
     with make_tempdir() as d:
         nlp3.to_disk(d)
+        with pytest.warns(UserWarning):
             nlp4 = spacy.load(d, disable=["ner"])
-    assert nlp4.pipe_names == []
+    assert nlp4.pipe_names == ["tagger"]
     assert nlp4.component_names == ["ner", "tagger"]
-    assert nlp4.disabled == ["ner", "tagger"]
+    assert nlp4.disabled == ["ner"]
     with make_tempdir() as d:
         nlp.to_disk(d)
         nlp5 = spacy.load(d, exclude=["tagger"])