Merge branch 'master' into feature/coref

.github/ISSUE_TEMPLATE.md

@@ -1,18 +0,0 @@
-<!--- Please provide a summary in the title and describe your issue here.
-Is this a bug or feature request? If a bug, include all the steps that led to the issue.
-
-If you're looking for help with your code, consider posting a question here:
-
-- GitHub Discussions: https://github.com/explosion/spaCy/discussions
-- Stack Overflow: http://stackoverflow.com/questions/tagged/spacy
--->
-
-## Your Environment
-
-<!-- Include details of your environment. If you're using spaCy 1.7+, you can also type
-`python -m spacy info --markdown` and copy-paste the result here.-->
-
-- Operating System:
-- Python Version Used:
-- spaCy Version Used:
-- Environment Information:

.github/ISSUE_TEMPLATE/01_bugs.md

@@ -1,6 +1,6 @@
 ---
-name: "\U0001F6A8 Bug Report"
-about: Did you come across a bug or unexpected behaviour differing from the docs?
+name: "\U0001F6A8 Submit a Bug Report"
+about: Use this template if you came across a bug or unexpected behaviour differing from the docs.
 
 ---
 

.github/ISSUE_TEMPLATE/02_docs.md

@@ -1,5 +1,5 @@
 ---
-name: "\U0001F4DA Documentation"
+name: "\U0001F4DA Submit a Documentation Report"
 about: Did you spot a mistake in the docs, is anything unclear or do you have a
   suggestion?
 

.github/ISSUE_TEMPLATE/03_other.md

@@ -1,19 +0,0 @@
----
-name: "\U0001F4AC Anything else?"
-about: For feature and project ideas, general usage questions or help with your code, please post on the GitHub Discussions board instead.
----
-
-<!-- Describe your issue here. Please keep in mind that the GitHub issue tracker is mostly intended for reports related to the spaCy code base and source, and for bugs and enhancements. If you're looking for help with your code, consider posting a question here:
-
-- GitHub Discussions: https://github.com/explosion/spaCy/discussions
-- Stack Overflow: http://stackoverflow.com/questions/tagged/spacy
--->
-
-## Your Environment
-
-<!-- Include details of your environment. If you're using spaCy 1.7+, you can also type `python -m spacy info --markdown` and copy-paste the result here.-->
-
-- Operating System:
-- Python Version Used:
-- spaCy Version Used:
-- Environment Information:

.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,14 @@
+blank_issues_enabled: false
+contact_links:
+  - name: 🗯 Discussions Forum
+    url: https://github.com/explosion/spaCy/discussions
+    about: Usage questions, general discussion and anything else that isn't a bug report.
+  - name: 📖 spaCy FAQ & Troubleshooting
+    url: https://github.com/explosion/spaCy/discussions/8226
+    about: Before you post, check out the FAQ for answers to common community questions!
+  - name: 💫 spaCy Usage Guides & API reference
+    url: https://spacy.io/usage
+    about: Everything you need to know about spaCy and how to use it.
+  - name: 🛠 Submit a Pull Request
+    url: https://github.com/explosion/spaCy/pulls
+    about: Did you spot a mistake and know how to fix it? Feel free to submit a PR straight away!

.github/azure-steps.yml

@@ -11,6 +11,10 @@ steps:
       versionSpec: ${{ parameters.python_version }}
       architecture: ${{ parameters.architecture }}
 
+  - bash: |
+      echo "##vso[task.setvariable variable=python_version]${{ parameters.python_version }}"
+    displayName: 'Set variables'
+
   - script: |
       ${{ parameters.prefix }} python -m pip install -U pip setuptools
       ${{ parameters.prefix }} python -m pip install -U -r requirements.txt
@@ -55,3 +59,44 @@ steps:
       ${{ parameters.prefix }} python -m pytest --pyargs spacy -p spacy.tests.enable_gpu
     displayName: "Run GPU tests"
     condition: eq(${{ parameters.gpu }}, true)
+
+  - script: |
+      python -m spacy download ca_core_news_sm
+      python -m spacy download ca_core_news_md
+      python -c "import spacy; nlp=spacy.load('ca_core_news_sm'); doc=nlp('test')"
+    displayName: 'Test download CLI'
+    condition: eq(variables['python_version'], '3.8')
+
+  - script: |
+      python -m spacy convert extra/example_data/ner_example_data/ner-token-per-line-conll2003.json .
+    displayName: 'Test convert CLI'
+    condition: eq(variables['python_version'], '3.8')
+
+  - script: |
+      python -m spacy init config -p ner -l ca ner.cfg
+      python -m spacy debug config ner.cfg --paths.train ner-token-per-line-conll2003.spacy --paths.dev ner-token-per-line-conll2003.spacy
+    displayName: 'Test debug config CLI'
+    condition: eq(variables['python_version'], '3.8')
+
+  - script: |
+      # will have errors due to sparse data, check for summary in output
+      python -m spacy debug data ner.cfg --paths.train ner-token-per-line-conll2003.spacy --paths.dev ner-token-per-line-conll2003.spacy | grep -q Summary
+    displayName: 'Test debug data CLI'
+    condition: eq(variables['python_version'], '3.8')
+
+  - script: |
+      python -m spacy train ner.cfg --paths.train ner-token-per-line-conll2003.spacy --paths.dev ner-token-per-line-conll2003.spacy --training.max_steps 10 --gpu-id -1
+    displayName: 'Test train CLI'
+    condition: eq(variables['python_version'], '3.8')
+
+  - script: |
+      python -c "import spacy; config = spacy.util.load_config('ner.cfg'); config['components']['ner'] = {'source': 'ca_core_news_sm'}; config.to_disk('ner_source_sm.cfg')"
+      PYTHONWARNINGS="error,ignore::DeprecationWarning" python -m spacy assemble ner_source_sm.cfg output_dir
+    displayName: 'Test assemble CLI'
+    condition: eq(variables['python_version'], '3.8')
+
+  - script: |
+      python -c "import spacy; config = spacy.util.load_config('ner.cfg'); config['components']['ner'] = {'source': 'ca_core_news_md'}; config.to_disk('ner_source_md.cfg')"
+      python -m spacy assemble ner_source_md.cfg output_dir 2>&1 | grep -q W113
+    displayName: 'Test assemble CLI vectors warning'
+    condition: eq(variables['python_version'], '3.8')

.github/contributors/ZeeD.md

@@ -0,0 +1,106 @@
+# spaCy contributor agreement
+
+This spaCy Contributor Agreement (**"SCA"**) is based on the
+[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf).
+The SCA applies to any contribution that you make to any product or project
+managed by us (the **"project"**), and sets out the intellectual property rights
+you grant to us in the contributed materials. The term **"us"** shall mean
+[ExplosionAI GmbH](https://explosion.ai/legal). The term
+**"you"** shall mean the person or entity identified below.
+
+If you agree to be bound by these terms, fill in the information requested
+below and include the filled-in version with your first pull request, under the
+folder [`.github/contributors/`](/.github/contributors/). The name of the file
+should be your GitHub username, with the extension `.md`. For example, the user
+example_user would create the file `.github/contributors/example_user.md`.
+
+Read this agreement carefully before signing. These terms and conditions
+constitute a binding legal agreement.
+
+## Contributor Agreement
+
+1. The term "contribution" or "contributed materials" means any source code,
+object code, patch, tool, sample, graphic, specification, manual,
+documentation, or any other material posted or submitted by you to the project.
+
+2. With respect to any worldwide copyrights, or copyright applications and
+registrations, in your contribution:
+
+    * you hereby assign to us joint ownership, and to the extent that such
+    assignment is or becomes invalid, ineffective or unenforceable, you hereby
+    grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge,
+    royalty-free, unrestricted license to exercise all rights under those
+    copyrights. This includes, at our option, the right to sublicense these same
+    rights to third parties through multiple levels of sublicensees or other
+    licensing arrangements;
+
+    * you agree that each of us can do all things in relation to your
+    contribution as if each of us were the sole owners, and if one of us makes
+    a derivative work of your contribution, the one who makes the derivative
+    work (or has it made will be the sole owner of that derivative work;
+
+    * you agree that you will not assert any moral rights in your contribution
+    against us, our licensees or transferees;
+
+    * you agree that we may register a copyright in your contribution and
+    exercise all ownership rights associated with it; and
+
+    * you agree that neither of us has any duty to consult with, obtain the
+    consent of, pay or render an accounting to the other for any use or
+    distribution of your contribution.
+
+3. With respect to any patents you own, or that you can license without payment
+to any third party, you hereby grant to us a perpetual, irrevocable,
+non-exclusive, worldwide, no-charge, royalty-free license to:
+
+    * make, have made, use, sell, offer to sell, import, and otherwise transfer
+    your contribution in whole or in part, alone or in combination with or
+    included in any product, work or materials arising out of the project to
+    which your contribution was submitted, and
+
+    * at our option, to sublicense these same rights to third parties through
+    multiple levels of sublicensees or other licensing arrangements.
+
+4. Except as set out above, you keep all right, title, and interest in your
+contribution. The rights that you grant to us under these terms are effective
+on the date you first submitted a contribution to us, even if your submission
+took place before the date you sign these terms.
+
+5. You covenant, represent, warrant and agree that:
+
+    * Each contribution that you submit is and shall be an original work of
+    authorship and you can legally grant the rights set out in this SCA;
+
+    * to the best of your knowledge, each contribution will not violate any
+    third party's copyrights, trademarks, patents, or other intellectual
+    property rights; and
+
+    * each contribution shall be in compliance with U.S. export control laws and
+    other applicable export and import laws. You agree to notify us if you
+    become aware of any circumstance which would make any of the foregoing
+    representations inaccurate in any respect. We may publicly disclose your
+    participation in the project, including the fact that you have signed the SCA.
+
+6. This SCA is governed by the laws of the State of California and applicable
+U.S. Federal law. Any choice of law rules will not apply.
+
+7. Please place an “x” on one of the applicable statement below. Please do NOT
+mark both statements:
+
+    * [x] I am signing on behalf of myself as an individual and no other person
+    or entity, including my employer, has or will have rights with respect to my
+    contributions.
+
+    * [ ] I am signing on behalf of my employer or a legal entity and I have the
+    actual authority to contractually bind that entity.
+
+## Contributor Details
+
+| Field                          | Entry                |
+|------------------------------- | -------------------- |
+| Name                           | Vito De Tullio       |
+| Company name (if applicable)   |                      |
+| Title or role (if applicable)  |                      |
+| Date                           | 2021-06-01           |
+| GitHub username                | ZeeD                 |
+| Website (optional)             |                      |

.github/contributors/bodak.md

@@ -0,0 +1,106 @@
+# spaCy contributor agreement
+
+This spaCy Contributor Agreement (**"SCA"**) is based on the
+[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf).
+The SCA applies to any contribution that you make to any product or project
+managed by us (the **"project"**), and sets out the intellectual property rights
+you grant to us in the contributed materials. The term **"us"** shall mean
+[ExplosionAI GmbH](https://explosion.ai/legal). The term
+**"you"** shall mean the person or entity identified below.
+
+If you agree to be bound by these terms, fill in the information requested
+below and include the filled-in version with your first pull request, under the
+folder [`.github/contributors/`](/.github/contributors/). The name of the file
+should be your GitHub username, with the extension `.md`. For example, the user
+example_user would create the file `.github/contributors/example_user.md`.
+
+Read this agreement carefully before signing. These terms and conditions
+constitute a binding legal agreement.
+
+## Contributor Agreement
+
+1. The term "contribution" or "contributed materials" means any source code,
+object code, patch, tool, sample, graphic, specification, manual,
+documentation, or any other material posted or submitted by you to the project.
+
+2. With respect to any worldwide copyrights, or copyright applications and
+registrations, in your contribution:
+
+    * you hereby assign to us joint ownership, and to the extent that such
+    assignment is or becomes invalid, ineffective or unenforceable, you hereby
+    grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge,
+    royalty-free, unrestricted license to exercise all rights under those
+    copyrights. This includes, at our option, the right to sublicense these same
+    rights to third parties through multiple levels of sublicensees or other
+    licensing arrangements;
+
+    * you agree that each of us can do all things in relation to your
+    contribution as if each of us were the sole owners, and if one of us makes
+    a derivative work of your contribution, the one who makes the derivative
+    work (or has it made will be the sole owner of that derivative work;
+
+    * you agree that you will not assert any moral rights in your contribution
+    against us, our licensees or transferees;
+
+    * you agree that we may register a copyright in your contribution and
+    exercise all ownership rights associated with it; and
+
+    * you agree that neither of us has any duty to consult with, obtain the
+    consent of, pay or render an accounting to the other for any use or
+    distribution of your contribution.
+
+3. With respect to any patents you own, or that you can license without payment
+to any third party, you hereby grant to us a perpetual, irrevocable,
+non-exclusive, worldwide, no-charge, royalty-free license to:
+
+    * make, have made, use, sell, offer to sell, import, and otherwise transfer
+    your contribution in whole or in part, alone or in combination with or
+    included in any product, work or materials arising out of the project to
+    which your contribution was submitted, and
+
+    * at our option, to sublicense these same rights to third parties through
+    multiple levels of sublicensees or other licensing arrangements.
+
+4. Except as set out above, you keep all right, title, and interest in your
+contribution. The rights that you grant to us under these terms are effective
+on the date you first submitted a contribution to us, even if your submission
+took place before the date you sign these terms.
+
+5. You covenant, represent, warrant and agree that:
+
+    * Each contribution that you submit is and shall be an original work of
+    authorship and you can legally grant the rights set out in this SCA;
+
+    * to the best of your knowledge, each contribution will not violate any
+    third party's copyrights, trademarks, patents, or other intellectual
+    property rights; and
+
+    * each contribution shall be in compliance with U.S. export control laws and
+    other applicable export and import laws. You agree to notify us if you
+    become aware of any circumstance which would make any of the foregoing
+    representations inaccurate in any respect. We may publicly disclose your
+    participation in the project, including the fact that you have signed the SCA.
+
+6. This SCA is governed by the laws of the State of California and applicable
+U.S. Federal law. Any choice of law rules will not apply.
+
+7. Please place an “x” on one of the applicable statement below. Please do NOT
+mark both statements:
+
+    * [x] I am signing on behalf of myself as an individual and no other person
+    or entity, including my employer, has or will have rights with respect to my
+    contributions.
+
+    * [ ] I am signing on behalf of my employer or a legal entity and I have the
+    actual authority to contractually bind that entity.
+
+## Contributor Details
+
+| Field                          | Entry                |
+|------------------------------- | -------------------- |
+| Name                           | Kristian Boda        |
+| Company name (if applicable)   |                      |
+| Title or role (if applicable)  |                      |
+| Date                           | 18.05.2021           |
+| GitHub username                | bodak                |
+| Website (optional)             |                      |

.github/contributors/gtoffoli.md

@@ -0,0 +1,106 @@
+# spaCy contributor agreement
+
+This spaCy Contributor Agreement (**"SCA"**) is based on the
+[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf).
+The SCA applies to any contribution that you make to any product or project
+managed by us (the **"project"**), and sets out the intellectual property rights
+you grant to us in the contributed materials. The term **"us"** shall mean
+[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term
+**"you"** shall mean the person or entity identified below.
+
+If you agree to be bound by these terms, fill in the information requested
+below and include the filled-in version with your first pull request, under the
+folder [`.github/contributors/`](/.github/contributors/). The name of the file
+should be your GitHub username, with the extension `.md`. For example, the user
+example_user would create the file `.github/contributors/example_user.md`.
+
+Read this agreement carefully before signing. These terms and conditions
+constitute a binding legal agreement.
+
+## Contributor Agreement
+
+1. The term "contribution" or "contributed materials" means any source code,
+object code, patch, tool, sample, graphic, specification, manual,
+documentation, or any other material posted or submitted by you to the project.
+
+2. With respect to any worldwide copyrights, or copyright applications and
+registrations, in your contribution:
+
+    * you hereby assign to us joint ownership, and to the extent that such
+    assignment is or becomes invalid, ineffective or unenforceable, you hereby
+    grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge,
+    royalty-free, unrestricted license to exercise all rights under those
+    copyrights. This includes, at our option, the right to sublicense these same
+    rights to third parties through multiple levels of sublicensees or other
+    licensing arrangements;
+
+    * you agree that each of us can do all things in relation to your
+    contribution as if each of us were the sole owners, and if one of us makes
+    a derivative work of your contribution, the one who makes the derivative
+    work (or has it made will be the sole owner of that derivative work;
+
+    * you agree that you will not assert any moral rights in your contribution
+    against us, our licensees or transferees;
+
+    * you agree that we may register a copyright in your contribution and
+    exercise all ownership rights associated with it; and
+
+    * you agree that neither of us has any duty to consult with, obtain the
+    consent of, pay or render an accounting to the other for any use or
+    distribution of your contribution.
+
+3. With respect to any patents you own, or that you can license without payment
+to any third party, you hereby grant to us a perpetual, irrevocable,
+non-exclusive, worldwide, no-charge, royalty-free license to:
+
+    * make, have made, use, sell, offer to sell, import, and otherwise transfer
+    your contribution in whole or in part, alone or in combination with or
+    included in any product, work or materials arising out of the project to
+    which your contribution was submitted, and
+
+    * at our option, to sublicense these same rights to third parties through
+    multiple levels of sublicensees or other licensing arrangements.
+
+4. Except as set out above, you keep all right, title, and interest in your
+contribution. The rights that you grant to us under these terms are effective
+on the date you first submitted a contribution to us, even if your submission
+took place before the date you sign these terms.
+
+5. You covenant, represent, warrant and agree that:
+
+    * Each contribution that you submit is and shall be an original work of
+    authorship and you can legally grant the rights set out in this SCA;
+
+    * to the best of your knowledge, each contribution will not violate any
+    third party's copyrights, trademarks, patents, or other intellectual
+    property rights; and
+
+    * each contribution shall be in compliance with U.S. export control laws and
+    other applicable export and import laws. You agree to notify us if you
+    become aware of any circumstance which would make any of the foregoing
+    representations inaccurate in any respect. We may publicly disclose your 
+    participation in the project, including the fact that you have signed the SCA.
+
+6. This SCA is governed by the laws of the State of California and applicable
+U.S. Federal law. Any choice of law rules will not apply.
+
+7. Please place an “x” on one of the applicable statement below. Please do NOT
+mark both statements:
+
+    * [x] I am signing on behalf of myself as an individual and no other person
+    or entity, including my employer, has or will have rights with respect to my
+    contributions.
+
+    * [ ] I am signing on behalf of my employer or a legal entity and I have the
+    actual authority to contractually bind that entity.
+
+## Contributor Details
+
+| Field                          | Entry                    |
+|------------------------------- | ------------------------ |
+| Name                           | Giovanni Toffoli         |
+| Company name (if applicable)   |                          |
+| Title or role (if applicable)  |                          |
+| Date                           | 2021-05-12               |
+| GitHub username                | gtoffoli                 |
+| Website (optional)             |                          |

.github/contributors/julien-talkair.md

@@ -0,0 +1,106 @@
+# spaCy contributor agreement
+
+This spaCy Contributor Agreement (**"SCA"**) is based on the
+[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf).
+The SCA applies to any contribution that you make to any product or project
+managed by us (the **"project"**), and sets out the intellectual property rights
+you grant to us in the contributed materials. The term **"us"** shall mean
+[ExplosionAI GmbH](https://explosion.ai/legal). The term
+**"you"** shall mean the person or entity identified below.
+
+If you agree to be bound by these terms, fill in the information requested
+below and include the filled-in version with your first pull request, under the
+folder [`.github/contributors/`](/.github/contributors/). The name of the file
+should be your GitHub username, with the extension `.md`. For example, the user
+example_user would create the file `.github/contributors/example_user.md`.
+
+Read this agreement carefully before signing. These terms and conditions
+constitute a binding legal agreement.
+
+## Contributor Agreement
+
+1. The term "contribution" or "contributed materials" means any source code,
+object code, patch, tool, sample, graphic, specification, manual,
+documentation, or any other material posted or submitted by you to the project.
+
+2. With respect to any worldwide copyrights, or copyright applications and
+registrations, in your contribution:
+
+    * you hereby assign to us joint ownership, and to the extent that such
+    assignment is or becomes invalid, ineffective or unenforceable, you hereby
+    grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge,
+    royalty-free, unrestricted license to exercise all rights under those
+    copyrights. This includes, at our option, the right to sublicense these same
+    rights to third parties through multiple levels of sublicensees or other
+    licensing arrangements;
+
+    * you agree that each of us can do all things in relation to your
+    contribution as if each of us were the sole owners, and if one of us makes
+    a derivative work of your contribution, the one who makes the derivative
+    work (or has it made will be the sole owner of that derivative work;
+
+    * you agree that you will not assert any moral rights in your contribution
+    against us, our licensees or transferees;
+
+    * you agree that we may register a copyright in your contribution and
+    exercise all ownership rights associated with it; and
+
+    * you agree that neither of us has any duty to consult with, obtain the
+    consent of, pay or render an accounting to the other for any use or
+    distribution of your contribution.
+
+3. With respect to any patents you own, or that you can license without payment
+to any third party, you hereby grant to us a perpetual, irrevocable,
+non-exclusive, worldwide, no-charge, royalty-free license to:
+
+    * make, have made, use, sell, offer to sell, import, and otherwise transfer
+    your contribution in whole or in part, alone or in combination with or
+    included in any product, work or materials arising out of the project to
+    which your contribution was submitted, and
+
+    * at our option, to sublicense these same rights to third parties through
+    multiple levels of sublicensees or other licensing arrangements.
+
+4. Except as set out above, you keep all right, title, and interest in your
+contribution. The rights that you grant to us under these terms are effective
+on the date you first submitted a contribution to us, even if your submission
+took place before the date you sign these terms.
+
+5. You covenant, represent, warrant and agree that:
+
+    * Each contribution that you submit is and shall be an original work of
+    authorship and you can legally grant the rights set out in this SCA;
+
+    * to the best of your knowledge, each contribution will not violate any
+    third party's copyrights, trademarks, patents, or other intellectual
+    property rights; and
+
+    * each contribution shall be in compliance with U.S. export control laws and
+    other applicable export and import laws. You agree to notify us if you
+    become aware of any circumstance which would make any of the foregoing
+    representations inaccurate in any respect. We may publicly disclose your
+    participation in the project, including the fact that you have signed the SCA.
+
+6. This SCA is governed by the laws of the State of California and applicable
+U.S. Federal law. Any choice of law rules will not apply.
+
+7. Please place an “x” on one of the applicable statement below. Please do NOT
+mark both statements:
+
+    * [ ] I am signing on behalf of myself as an individual and no other person
+    or entity, including my employer, has or will have rights with respect to my
+    contributions.
+
+    * [x] I am signing on behalf of my employer or a legal entity and I have the
+    actual authority to contractually bind that entity.
+
+## Contributor Details
+
+| Field                          | Entry                |
+|------------------------------- | -------------------- |
+| Name                           |  Julien Rossi                    |
+| Company name (if applicable)   |  TalkAir BV                    |
+| Title or role (if applicable)  |  CTO, Partner                    |
+| Date                           |  June 28 2021                    |
+| GitHub username                |  julien-talkair                    |
+| Website (optional)             |                      |

.github/contributors/narayanacharya6.md

@@ -0,0 +1,106 @@
+# spaCy contributor agreement
+
+This spaCy Contributor Agreement (**"SCA"**) is based on the
+[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf).
+The SCA applies to any contribution that you make to any product or project
+managed by us (the **"project"**), and sets out the intellectual property rights
+you grant to us in the contributed materials. The term **"us"** shall mean
+[ExplosionAI GmbH](https://explosion.ai/legal). The term
+**"you"** shall mean the person or entity identified below.
+
+If you agree to be bound by these terms, fill in the information requested
+below and include the filled-in version with your first pull request, under the
+folder [`.github/contributors/`](/.github/contributors/). The name of the file
+should be your GitHub username, with the extension `.md`. For example, the user
+example_user would create the file `.github/contributors/example_user.md`.
+
+Read this agreement carefully before signing. These terms and conditions
+constitute a binding legal agreement.
+
+## Contributor Agreement
+
+1. The term "contribution" or "contributed materials" means any source code,
+object code, patch, tool, sample, graphic, specification, manual,
+documentation, or any other material posted or submitted by you to the project.
+
+2. With respect to any worldwide copyrights, or copyright applications and
+registrations, in your contribution:
+
+    * you hereby assign to us joint ownership, and to the extent that such
+    assignment is or becomes invalid, ineffective or unenforceable, you hereby
+    grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge,
+    royalty-free, unrestricted license to exercise all rights under those
+    copyrights. This includes, at our option, the right to sublicense these same
+    rights to third parties through multiple levels of sublicensees or other
+    licensing arrangements;
+
+    * you agree that each of us can do all things in relation to your
+    contribution as if each of us were the sole owners, and if one of us makes
+    a derivative work of your contribution, the one who makes the derivative
+    work (or has it made will be the sole owner of that derivative work;
+
+    * you agree that you will not assert any moral rights in your contribution
+    against us, our licensees or transferees;
+
+    * you agree that we may register a copyright in your contribution and
+    exercise all ownership rights associated with it; and
+
+    * you agree that neither of us has any duty to consult with, obtain the
+    consent of, pay or render an accounting to the other for any use or
+    distribution of your contribution.
+
+3. With respect to any patents you own, or that you can license without payment
+to any third party, you hereby grant to us a perpetual, irrevocable,
+non-exclusive, worldwide, no-charge, royalty-free license to:
+
+    * make, have made, use, sell, offer to sell, import, and otherwise transfer
+    your contribution in whole or in part, alone or in combination with or
+    included in any product, work or materials arising out of the project to
+    which your contribution was submitted, and
+
+    * at our option, to sublicense these same rights to third parties through
+    multiple levels of sublicensees or other licensing arrangements.
+
+4. Except as set out above, you keep all right, title, and interest in your
+contribution. The rights that you grant to us under these terms are effective
+on the date you first submitted a contribution to us, even if your submission
+took place before the date you sign these terms.
+
+5. You covenant, represent, warrant and agree that:
+
+    * Each contribution that you submit is and shall be an original work of
+    authorship and you can legally grant the rights set out in this SCA;
+
+    * to the best of your knowledge, each contribution will not violate any
+    third party's copyrights, trademarks, patents, or other intellectual
+    property rights; and
+
+    * each contribution shall be in compliance with U.S. export control laws and
+    other applicable export and import laws. You agree to notify us if you
+    become aware of any circumstance which would make any of the foregoing
+    representations inaccurate in any respect. We may publicly disclose your
+    participation in the project, including the fact that you have signed the SCA.
+
+6. This SCA is governed by the laws of the State of California and applicable
+U.S. Federal law. Any choice of law rules will not apply.
+
+7. Please place an “x” on one of the applicable statement below. Please do NOT
+mark both statements:
+
+    * [x] I am signing on behalf of myself as an individual and no other person
+    or entity, including my employer, has or will have rights with respect to my
+    contributions.
+
+    * [ ] I am signing on behalf of my employer or a legal entity and I have the
+    actual authority to contractually bind that entity.
+
+## Contributor Details
+
+| Field                          | Entry                |
+|------------------------------- | -------------------- |
+| Name                           | Narayan Acharya      |
+| Company name (if applicable)   |                      |
+| Title or role (if applicable)  |                      |
+| Date                           | 29 APR 2021          |
+| GitHub username                | narayanacharya6      |
+| Website (optional)             | narayanacharya.com   |

.github/contributors/xadrianzetx.md

@@ -0,0 +1,106 @@
+# spaCy contributor agreement
+
+This spaCy Contributor Agreement (**"SCA"**) is based on the
+[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf).
+The SCA applies to any contribution that you make to any product or project
+managed by us (the **"project"**), and sets out the intellectual property rights
+you grant to us in the contributed materials. The term **"us"** shall mean
+[ExplosionAI GmbH](https://explosion.ai/legal). The term
+**"you"** shall mean the person or entity identified below.
+
+If you agree to be bound by these terms, fill in the information requested
+below and include the filled-in version with your first pull request, under the
+folder [`.github/contributors/`](/.github/contributors/). The name of the file
+should be your GitHub username, with the extension `.md`. For example, the user
+example_user would create the file `.github/contributors/example_user.md`.
+
+Read this agreement carefully before signing. These terms and conditions
+constitute a binding legal agreement.
+
+## Contributor Agreement
+
+1. The term "contribution" or "contributed materials" means any source code,
+object code, patch, tool, sample, graphic, specification, manual,
+documentation, or any other material posted or submitted by you to the project.
+
+2. With respect to any worldwide copyrights, or copyright applications and
+registrations, in your contribution:
+
+    * you hereby assign to us joint ownership, and to the extent that such
+    assignment is or becomes invalid, ineffective or unenforceable, you hereby
+    grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge,
+    royalty-free, unrestricted license to exercise all rights under those
+    copyrights. This includes, at our option, the right to sublicense these same
+    rights to third parties through multiple levels of sublicensees or other
+    licensing arrangements;
+
+    * you agree that each of us can do all things in relation to your
+    contribution as if each of us were the sole owners, and if one of us makes
+    a derivative work of your contribution, the one who makes the derivative
+    work (or has it made will be the sole owner of that derivative work;
+
+    * you agree that you will not assert any moral rights in your contribution
+    against us, our licensees or transferees;
+
+    * you agree that we may register a copyright in your contribution and
+    exercise all ownership rights associated with it; and
+
+    * you agree that neither of us has any duty to consult with, obtain the
+    consent of, pay or render an accounting to the other for any use or
+    distribution of your contribution.
+
+3. With respect to any patents you own, or that you can license without payment
+to any third party, you hereby grant to us a perpetual, irrevocable,
+non-exclusive, worldwide, no-charge, royalty-free license to:
+
+    * make, have made, use, sell, offer to sell, import, and otherwise transfer
+    your contribution in whole or in part, alone or in combination with or
+    included in any product, work or materials arising out of the project to
+    which your contribution was submitted, and
+
+    * at our option, to sublicense these same rights to third parties through
+    multiple levels of sublicensees or other licensing arrangements.
+
+4. Except as set out above, you keep all right, title, and interest in your
+contribution. The rights that you grant to us under these terms are effective
+on the date you first submitted a contribution to us, even if your submission
+took place before the date you sign these terms.
+
+5. You covenant, represent, warrant and agree that:
+
+    * Each contribution that you submit is and shall be an original work of
+    authorship and you can legally grant the rights set out in this SCA;
+
+    * to the best of your knowledge, each contribution will not violate any
+    third party's copyrights, trademarks, patents, or other intellectual
+    property rights; and
+
+    * each contribution shall be in compliance with U.S. export control laws and
+    other applicable export and import laws. You agree to notify us if you
+    become aware of any circumstance which would make any of the foregoing
+    representations inaccurate in any respect. We may publicly disclose your
+    participation in the project, including the fact that you have signed the SCA.
+
+6. This SCA is governed by the laws of the State of California and applicable
+U.S. Federal law. Any choice of law rules will not apply.
+
+7. Please place an “x” on one of the applicable statement below. Please do NOT
+mark both statements:
+
+    * [x] I am signing on behalf of myself as an individual and no other person
+    or entity, including my employer, has or will have rights with respect to my
+    contributions.
+
+    * [ ] I am signing on behalf of my employer or a legal entity and I have the
+    actual authority to contractually bind that entity.
+
+## Contributor Details
+
+| Field                          | Entry                |
+|------------------------------- | -------------------- |
+| Name                           |Adrian Zuber          |
+| Company name (if applicable)   |                      |
+| Title or role (if applicable)  |                      |
+| Date                           |20-06-2021            |
+| GitHub username                |xadrianzetx           |
+| Website (optional)             |                      |

.github/contributors/yohasebe.md

@@ -0,0 +1,106 @@
+# spaCy contributor agreement
+
+This spaCy Contributor Agreement (**"SCA"**) is based on the
+[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf).
+The SCA applies to any contribution that you make to any product or project
+managed by us (the **"project"**), and sets out the intellectual property rights
+you grant to us in the contributed materials. The term **"us"** shall mean
+[ExplosionAI GmbH](https://explosion.ai/legal). The term
+**"you"** shall mean the person or entity identified below.
+
+If you agree to be bound by these terms, fill in the information requested
+below and include the filled-in version with your first pull request, under the
+folder [`.github/contributors/`](/.github/contributors/). The name of the file
+should be your GitHub username, with the extension `.md`. For example, the user
+example_user would create the file `.github/contributors/example_user.md`.
+
+Read this agreement carefully before signing. These terms and conditions
+constitute a binding legal agreement.
+
+## Contributor Agreement
+
+1. The term "contribution" or "contributed materials" means any source code,
+object code, patch, tool, sample, graphic, specification, manual,
+documentation, or any other material posted or submitted by you to the project.
+
+2. With respect to any worldwide copyrights, or copyright applications and
+registrations, in your contribution:
+
+    * you hereby assign to us joint ownership, and to the extent that such
+    assignment is or becomes invalid, ineffective or unenforceable, you hereby
+    grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge,
+    royalty-free, unrestricted license to exercise all rights under those
+    copyrights. This includes, at our option, the right to sublicense these same
+    rights to third parties through multiple levels of sublicensees or other
+    licensing arrangements;
+
+    * you agree that each of us can do all things in relation to your
+    contribution as if each of us were the sole owners, and if one of us makes
+    a derivative work of your contribution, the one who makes the derivative
+    work (or has it made will be the sole owner of that derivative work;
+
+    * you agree that you will not assert any moral rights in your contribution
+    against us, our licensees or transferees;
+
+    * you agree that we may register a copyright in your contribution and
+    exercise all ownership rights associated with it; and
+
+    * you agree that neither of us has any duty to consult with, obtain the
+    consent of, pay or render an accounting to the other for any use or
+    distribution of your contribution.
+
+3. With respect to any patents you own, or that you can license without payment
+to any third party, you hereby grant to us a perpetual, irrevocable,
+non-exclusive, worldwide, no-charge, royalty-free license to:
+
+    * make, have made, use, sell, offer to sell, import, and otherwise transfer
+    your contribution in whole or in part, alone or in combination with or
+    included in any product, work or materials arising out of the project to
+    which your contribution was submitted, and
+
+    * at our option, to sublicense these same rights to third parties through
+    multiple levels of sublicensees or other licensing arrangements.
+
+4. Except as set out above, you keep all right, title, and interest in your
+contribution. The rights that you grant to us under these terms are effective
+on the date you first submitted a contribution to us, even if your submission
+took place before the date you sign these terms.
+
+5. You covenant, represent, warrant and agree that:
+
+    * Each contribution that you submit is and shall be an original work of
+    authorship and you can legally grant the rights set out in this SCA;
+
+    * to the best of your knowledge, each contribution will not violate any
+    third party's copyrights, trademarks, patents, or other intellectual
+    property rights; and
+
+    * each contribution shall be in compliance with U.S. export control laws and
+    other applicable export and import laws. You agree to notify us if you
+    become aware of any circumstance which would make any of the foregoing
+    representations inaccurate in any respect. We may publicly disclose your
+    participation in the project, including the fact that you have signed the SCA.
+
+6. This SCA is governed by the laws of the State of California and applicable
+U.S. Federal law. Any choice of law rules will not apply.
+
+7. Please place an “x” on one of the applicable statement below. Please do NOT
+mark both statements:
+
+    * [x] I am signing on behalf of myself as an individual and no other person
+    or entity, including my employer, has or will have rights with respect to my
+    contributions.
+
+    * [ ] I am signing on behalf of my employer or a legal entity and I have the
+    actual authority to contractually bind that entity.
+
+## Contributor Details
+
+| Field                          | Entry                |
+|------------------------------- | -------------------- |
+| Name                           | Yoichiro Hasebe      |
+| Company name (if applicable)   |                      |
+| Title or role (if applicable)  |                      |
+| Date                           | July 4th, 2021       |
+| GitHub username                | yohasebe             |
+| Website (optional)             | https://yohasebe.com |

.github/workflows/autoblack.yml

@@ -0,0 +1,44 @@
+# GitHub Action that uses Black to reformat all Python code and submits a PR
+# in regular intervals. Inspired by: https://github.com/cclauss/autoblack
+
+name: autoblack
+on:
+  workflow_dispatch:  # allow manual trigger
+  schedule:
+    - cron: '0 8 * * 5'  # every Friday at 8am UTC
+
+jobs:
+  autoblack:
+    if: github.repository_owner = 'explosion'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+        with:
+            ref: ${{ github.head_ref }}
+      - uses: actions/setup-python@v2
+      - run: pip install black
+      - name: Auto-format code if needed
+        run: black spacy
+      # We can't run black --check here because that returns a non-zero excit
+      # code and makes GitHub think the action failed
+      - name: Check for modified files
+        id: git-check
+        run: echo ::set-output name=modified::$(if git diff-index --quiet HEAD --; then echo "false"; else echo "true"; fi)
+      - name: Create Pull Request
+        if: steps.git-check.outputs.modified == 'true'
+        uses: peter-evans/create-pull-request@v3
+        with:
+            title: Auto-format code with black
+            labels: meta
+            commit-message: Auto-format code with black
+            committer: GitHub <noreply@github.com>
+            author: explosion-bot <explosion-bot@users.noreply.github.com>
+            body: _This PR is auto-generated._
+            branch: autoblack
+            delete-branch: true
+            draft: false
+      - name: Check outputs
+        if: steps.git-check.outputs.modified == 'true'
+        run: |
+          echo "Pull Request Number - ${{ steps.cpr.outputs.pull-request-number }}"
+          echo "Pull Request URL - ${{ steps.cpr.outputs.pull-request-url }}"

.pre-commit-config.yaml

@@ -0,0 +1,12 @@
+repos:
+-   repo: https://github.com/ambv/black
+    rev: 21.6b0
+    hooks:
+    - id: black
+      language_version: python3.7
+-   repo: https://gitlab.com/pycqa/flake8
+    rev: 3.9.2
+    hooks:
+    - id: flake8
+      args:
+        - "--config=setup.cfg"

CONTRIBUTING.md

@@ -2,11 +2,7 @@
 
 # Contribute to spaCy
 
-Thanks for your interest in contributing to spaCy 🎉 The project is maintained
-by **[@honnibal](https://github.com/honnibal)**,
-**[@ines](https://github.com/ines)**, **[@svlandeg](https://github.com/svlandeg)** and
-**[@adrianeboyd](https://github.com/adrianeboyd)**,
-and we'll do our best to help you get started. This page will give you a quick
+Thanks for your interest in contributing to spaCy 🎉 This page will give you a quick
 overview of how things are organized and most importantly, how to get involved.
 
 ## Table of contents
@@ -181,6 +177,15 @@ tools installed.
 **⚠️ Note that formatting and linting is currently only possible for Python
 modules in `.py` files, not Cython modules in `.pyx` and `.pxd` files.**
 
+### Pre-Commit Hooks
+
+After cloning the repo, after installing the packages from `requirements.txt`, enter the repo folder and run `pre-commit install`.
+Each time a `git commit` is initiated, `black` and `flake8` will run automatically on the modified files only.
+
+In case of error, or when `black` modified a file, the modified file needs to be `git add` once again and a new
+`git commit` has to be issued.
+
+
 ### Code formatting
 
 [`black`](https://github.com/ambv/black) is an opinionated Python code

MANIFEST.in

@@ -8,3 +8,4 @@ recursive-exclude spacy/lang *.json
 recursive-include spacy/lang *.json.gz
 recursive-include spacy/cli *.json *.yml
 recursive-include licenses *
+recursive-exclude spacy *.cpp

Makefile

@@ -1,7 +1,7 @@
 SHELL := /bin/bash
 
 ifndef SPACY_EXTRAS
-override SPACY_EXTRAS = spacy-lookups-data==1.0.0 jieba spacy-pkuseg==0.0.28 sudachipy sudachidict_core pymorphy2
+override SPACY_EXTRAS = spacy-lookups-data==1.0.2 jieba spacy-pkuseg==0.0.28 sudachipy sudachidict_core pymorphy2
 endif
 
 ifndef PYVER

README.md

@@ -61,11 +61,11 @@ open-source software, released under the MIT license.
 ## 💬 Where to ask questions
 
 The spaCy project is maintained by **[@honnibal](https://github.com/honnibal)**,
-**[@ines](https://github.com/ines)**, **[@svlandeg](https://github.com/svlandeg)** and
-**[@adrianeboyd](https://github.com/adrianeboyd)**. Please understand that we won't
-be able to provide individual support via email. We also believe that help is
-much more valuable if it's shared publicly, so that more people can benefit from
-it.
+**[@ines](https://github.com/ines)**, **[@svlandeg](https://github.com/svlandeg)**,
+**[@adrianeboyd](https://github.com/adrianeboyd)** and **[@polm](https://github.com/polm)**.
+Please understand that we won't be able to provide individual support via email.
+We also believe that help is much more valuable if it's shared publicly, so that
+more people can benefit from it.
 
 | Type                            | Platforms                               |
 | ------------------------------- | --------------------------------------- |

azure-pipelines.yml

@@ -22,13 +22,13 @@ jobs:
   # defined in .flake8 and overwrites the selected codes.
   - job: "Validate"
     pool:
-      vmImage: "ubuntu-16.04"
+      vmImage: "ubuntu-18.04"
     steps:
       - task: UsePythonVersion@0
         inputs:
           versionSpec: "3.7"
       - script: |
-          pip install flake8==3.5.0
+          pip install flake8==3.9.2
           python -m flake8 spacy --count --select=E901,E999,F821,F822,F823 --show-source --statistics
         displayName: "flake8"
 
@@ -38,7 +38,7 @@ jobs:
       matrix:
         # We're only running one platform per Python version to speed up builds
         Python36Linux:
-          imageName: "ubuntu-16.04"
+          imageName: "ubuntu-18.04"
           python.version: "3.6"
         #        Python36Windows:
         #          imageName: "vs2017-win2016"
@@ -47,7 +47,7 @@ jobs:
         #          imageName: "macos-10.14"
         #          python.version: "3.6"
         #        Python37Linux:
-        #          imageName: "ubuntu-16.04"
+        #          imageName: "ubuntu-18.04"
         #          python.version: "3.7"
         Python37Windows:
           imageName: "vs2017-win2016"
@@ -56,7 +56,7 @@ jobs:
         #          imageName: "macos-10.14"
         #          python.version: "3.7"
         #        Python38Linux:
-        #          imageName: "ubuntu-16.04"
+        #          imageName: "ubuntu-18.04"
         #          python.version: "3.8"
         #        Python38Windows:
         #          imageName: "vs2017-win2016"
@@ -65,7 +65,7 @@ jobs:
           imageName: "macos-10.14"
           python.version: "3.8"
         Python39Linux:
-          imageName: "ubuntu-16.04"
+          imageName: "ubuntu-18.04"
           python.version: "3.9"
         Python39Windows:
           imageName: "vs2017-win2016"

pyproject.toml

@@ -1,11 +1,11 @@
 [build-system]
 requires = [
     "setuptools",
-    "cython>=0.25",
+    "cython>=0.25,<3.0",
     "cymem>=2.0.2,<2.1.0",
     "preshed>=3.0.2,<3.1.0",
     "murmurhash>=0.28.0,<1.1.0",
-    "thinc>=8.0.3,<8.1.0",
+    "thinc>=8.0.7,<8.1.0",
     "blis>=0.4.0,<0.8.0",
     "pathy",
     "numpy>=1.15.0",

requirements.txt

@@ -1,8 +1,8 @@
 # Our libraries
-spacy-legacy>=3.0.5,<3.1.0
+spacy-legacy>=3.0.7,<3.1.0
 cymem>=2.0.2,<2.1.0
 preshed>=3.0.2,<3.1.0
-thinc>=8.0.3,<8.1.0
+thinc>=8.0.7,<8.1.0
 blis>=0.4.0,<0.8.0
 ml_datasets>=0.2.0,<0.3.0
 murmurhash>=0.28.0,<1.1.0
@@ -22,9 +22,10 @@ setuptools
 packaging>=20.0
 typing_extensions>=3.7.4.1,<4.0.0.0; python_version < "3.8"
 # Development dependencies
-cython>=0.25
+pre-commit>=2.13.0
+cython>=0.25,<3.0
 pytest>=5.2.0
 pytest-timeout>=1.3.0,<2.0.0
 mock>=2.0.0,<3.0.0
-flake8>=3.5.0,<3.6.0
+flake8>=3.8.0,<3.10.0
 hypothesis>=3.27.0,<7.0.0

setup.cfg

@@ -22,26 +22,29 @@ classifiers =
     Programming Language :: Python :: 3.8
     Programming Language :: Python :: 3.9
     Topic :: Scientific/Engineering
+project_urls =
+    Release notes = https://github.com/explosion/spaCy/releases
+    Source = https://github.com/explosion/spaCy
 
 [options]
 zip_safe = false
 include_package_data = true
 python_requires = >=3.6
 setup_requires =
-    cython>=0.25
+    cython>=0.25,<3.0
     numpy>=1.15.0
     # We also need our Cython packages here to compile against
     cymem>=2.0.2,<2.1.0
     preshed>=3.0.2,<3.1.0
     murmurhash>=0.28.0,<1.1.0
-    thinc>=8.0.3,<8.1.0
+    thinc>=8.0.7,<8.1.0
 install_requires =
     # Our libraries
-    spacy-legacy>=3.0.5,<3.1.0
+    spacy-legacy>=3.0.7,<3.1.0
     murmurhash>=0.28.0,<1.1.0
     cymem>=2.0.2,<2.1.0
     preshed>=3.0.2,<3.1.0
-    thinc>=8.0.3,<8.1.0
+    thinc>=8.0.7,<8.1.0
     blis>=0.4.0,<0.8.0
     wasabi>=0.8.1,<1.1.0
     srsly>=2.4.1,<3.0.0
@@ -61,11 +64,11 @@ install_requires =
 
 [options.entry_points]
 console_scripts =
-    spacy = spacy.cli:app
+    spacy = spacy.cli:setup_cli
 
 [options.extras_require]
 lookups =
-    spacy_lookups_data>=1.0.0,<1.1.0
+    spacy_lookups_data>=1.0.2,<1.1.0
 transformers =
     spacy_transformers>=1.0.1,<1.1.0
 ray =
@@ -108,7 +111,7 @@ universal = false
 formats = gztar
 
 [flake8]
-ignore = E203, E266, E501, E731, W503, E741
+ignore = E203, E266, E501, E731, W503, E741, F541
 max-line-length = 80
 select = B,C,E,F,W,T4,B9
 exclude =

spacy/__init__.py

@@ -1,10 +1,11 @@
 from typing import Union, Iterable, Dict, Any
 from pathlib import Path
-import warnings
 import sys
 
-warnings.filterwarnings("ignore", message="numpy.dtype size changed")  # noqa
-warnings.filterwarnings("ignore", message="numpy.ufunc size changed")  # noqa
+# set library-specific custom warning handling before doing anything else
+from .errors import setup_default_warnings
+
+setup_default_warnings()
 
 # These are imported as part of the API
 from thinc.api import prefer_gpu, require_gpu, require_cpu  # noqa: F401

spacy/about.py

@@ -1,6 +1,6 @@
 # fmt: off
 __title__ = "spacy"
-__version__ = "3.0.6"
+__version__ = "3.1.0"
 __download_url__ = "https://github.com/explosion/spacy-models/releases/download"
 __compatibility__ = "https://raw.githubusercontent.com/explosion/spacy-models/master/compatibility.json"
 __projects__ = "https://github.com/explosion/projects"

spacy/attrs.pyx

@@ -74,7 +74,6 @@ IDS = {
     "SUFFIX": SUFFIX,
 
     "LENGTH": LENGTH,
-    "CLUSTER": CLUSTER,
     "LEMMA": LEMMA,
     "POS": POS,
     "TAG": TAG,
@@ -85,9 +84,7 @@ IDS = {
     "ENT_KB_ID": ENT_KB_ID,
     "HEAD": HEAD,
     "SENT_START": SENT_START,
-    "SENT_END": SENT_END,
     "SPACY": SPACY,
-    "PROB": PROB,
     "LANG": LANG,
     "MORPH": MORPH,
     "IDX": IDX

spacy/cli/_util.py

@@ -2,7 +2,7 @@ from typing import Dict, Any, Union, List, Optional, Tuple, Iterable, TYPE_CHECK
 import sys
 import shutil
 from pathlib import Path
-from wasabi import msg
+from wasabi import msg, Printer
 import srsly
 import hashlib
 import typer
@@ -504,12 +504,16 @@ def string_to_list(value: str, intify: bool = False) -> Union[List[str], List[in
     return result
 
 
-def setup_gpu(use_gpu: int) -> None:
+def setup_gpu(use_gpu: int, silent=None) -> None:
     """Configure the GPU and log info."""
+    if silent is None:
+        local_msg = Printer()
+    else:
+        local_msg = Printer(no_print=silent, pretty=not silent)
     if use_gpu >= 0:
-        msg.info(f"Using GPU: {use_gpu}")
+        local_msg.info(f"Using GPU: {use_gpu}")
         require_gpu(use_gpu)
     else:
-        msg.info("Using CPU")
+        local_msg.info("Using CPU")
         if has_cupy and gpu_is_available():
-            msg.info("To switch to GPU 0, use the option: --gpu-id 0")
+            local_msg.info("To switch to GPU 0, use the option: --gpu-id 0")

spacy/cli/assemble.py

@@ -6,7 +6,6 @@ import logging
 
 from ._util import app, Arg, Opt, parse_config_overrides, show_validation_error
 from ._util import import_code
-from ..training.initialize import init_nlp
 from .. import util
 from ..util import get_sourced_components, load_model_from_config
 

spacy/cli/convert.py

@@ -115,7 +115,8 @@ def convert(
     ner_map = srsly.read_json(ner_map) if ner_map is not None else None
     doc_files = []
     for input_loc in walk_directory(Path(input_path), converter):
-        input_data = input_loc.open("r", encoding="utf-8").read()
+        with input_loc.open("r", encoding="utf-8") as infile:
+            input_data = infile.read()
         # Use converter function to convert data
         func = CONVERTERS[converter]
         docs = func(

spacy/cli/debug_model.py

@@ -1,11 +1,11 @@
-from typing import Dict, Any, Optional, Iterable
+from typing import Dict, Any, Optional
 from pathlib import Path
 import itertools
 
 from spacy.training import Example
 from spacy.util import resolve_dot_names
 from wasabi import msg
-from thinc.api import fix_random_seed, set_dropout_rate, Adam
+from thinc.api import fix_random_seed, set_dropout_rate
 from thinc.api import Model, data_validation, set_gpu_allocator
 import typer
 
@@ -133,15 +133,16 @@ def debug_model(
         _print_model(model, print_settings)
 
     # STEP 2: Updating the model and printing again
-    optimizer = Adam(0.001)
     set_dropout_rate(model, 0.2)
     # ugly hack to deal with Tok2Vec/Transformer listeners
     upstream_component = None
     if model.has_ref("tok2vec") and "tok2vec-listener" in model.get_ref("tok2vec").name:
         upstream_component = nlp.get_pipe("tok2vec")
-    if model.has_ref("tok2vec") and "transformer-listener" in model.get_ref("tok2vec").name:
+    if (
+        model.has_ref("tok2vec")
+        and "transformer-listener" in model.get_ref("tok2vec").name
+    ):
         upstream_component = nlp.get_pipe("transformer")
-    goldY = None
     for e in range(3):
         if upstream_component:
             upstream_component.update(examples)

spacy/cli/download.py

@@ -6,7 +6,7 @@ import typer
 
 from ._util import app, Arg, Opt, WHEEL_SUFFIX, SDIST_SUFFIX
 from .. import about
-from ..util import is_package, get_base_version, run_command
+from ..util import is_package, get_minor_version, run_command
 from ..errors import OLD_MODEL_SHORTCUTS
 
 
@@ -74,7 +74,7 @@ def download(model: str, direct: bool = False, sdist: bool = False, *pip_args) -
 
 
 def get_compatibility() -> dict:
-    version = get_base_version(about.__version__)
+    version = get_minor_version(about.__version__)
     r = requests.get(about.__compatibility__)
     if r.status_code != 200:
         msg.fail(

spacy/cli/evaluate.py

@@ -1,4 +1,4 @@
-from typing import Optional, List, Dict
+from typing import Optional, List, Dict, Any, Union
 from wasabi import Printer
 from pathlib import Path
 import re
@@ -60,10 +60,11 @@ def evaluate(
     displacy_path: Optional[Path] = None,
     displacy_limit: int = 25,
     silent: bool = True,
-) -> Scorer:
+    spans_key: str = "sc",
+) -> Dict[str, Any]:
     msg = Printer(no_print=silent, pretty=not silent)
     fix_random_seed()
-    setup_gpu(use_gpu)
+    setup_gpu(use_gpu, silent=silent)
     data_path = util.ensure_path(data_path)
     output_path = util.ensure_path(output)
     displacy_path = util.ensure_path(displacy_path)
@@ -90,6 +91,9 @@ def evaluate(
         "SENT P": "sents_p",
         "SENT R": "sents_r",
         "SENT F": "sents_f",
+        "SPAN P": f"spans_{spans_key}_p",
+        "SPAN R": f"spans_{spans_key}_r",
+        "SPAN F": f"spans_{spans_key}_f",
         "SPEED": "speed",
     }
     results = {}
@@ -108,27 +112,7 @@ def evaluate(
             data[re.sub(r"[\s/]", "_", key.lower())] = scores[key]
 
     msg.table(results, title="Results")
-
-    if "morph_per_feat" in scores:
-        if scores["morph_per_feat"]:
-            print_prf_per_type(msg, scores["morph_per_feat"], "MORPH", "feat")
-            data["morph_per_feat"] = scores["morph_per_feat"]
-    if "dep_las_per_type" in scores:
-        if scores["dep_las_per_type"]:
-            print_prf_per_type(msg, scores["dep_las_per_type"], "LAS", "type")
-            data["dep_las_per_type"] = scores["dep_las_per_type"]
-    if "ents_per_type" in scores:
-        if scores["ents_per_type"]:
-            print_prf_per_type(msg, scores["ents_per_type"], "NER", "type")
-            data["ents_per_type"] = scores["ents_per_type"]
-    if "cats_f_per_type" in scores:
-        if scores["cats_f_per_type"]:
-            print_prf_per_type(msg, scores["cats_f_per_type"], "Textcat F", "label")
-            data["cats_f_per_type"] = scores["cats_f_per_type"]
-    if "cats_auc_per_type" in scores:
-        if scores["cats_auc_per_type"]:
-            print_textcats_auc_per_cat(msg, scores["cats_auc_per_type"])
-            data["cats_auc_per_type"] = scores["cats_auc_per_type"]
+    data = handle_scores_per_type(scores, data, spans_key=spans_key, silent=silent)
 
     if displacy_path:
         factory_names = [nlp.get_pipe_meta(pipe).factory for pipe in nlp.pipe_names]
@@ -151,6 +135,43 @@ def evaluate(
     return data
 
 
+def handle_scores_per_type(
+    scores: Union[Scorer, Dict[str, Any]],
+    data: Dict[str, Any] = {},
+    *,
+    spans_key: str = "sc",
+    silent: bool = False,
+) -> Dict[str, Any]:
+    msg = Printer(no_print=silent, pretty=not silent)
+    if "morph_per_feat" in scores:
+        if scores["morph_per_feat"]:
+            print_prf_per_type(msg, scores["morph_per_feat"], "MORPH", "feat")
+            data["morph_per_feat"] = scores["morph_per_feat"]
+    if "dep_las_per_type" in scores:
+        if scores["dep_las_per_type"]:
+            print_prf_per_type(msg, scores["dep_las_per_type"], "LAS", "type")
+            data["dep_las_per_type"] = scores["dep_las_per_type"]
+    if "ents_per_type" in scores:
+        if scores["ents_per_type"]:
+            print_prf_per_type(msg, scores["ents_per_type"], "NER", "type")
+            data["ents_per_type"] = scores["ents_per_type"]
+    if f"spans_{spans_key}_per_type" in scores:
+        if scores[f"spans_{spans_key}_per_type"]:
+            print_prf_per_type(
+                msg, scores[f"spans_{spans_key}_per_type"], "SPANS", "type"
+            )
+            data[f"spans_{spans_key}_per_type"] = scores[f"spans_{spans_key}_per_type"]
+    if "cats_f_per_type" in scores:
+        if scores["cats_f_per_type"]:
+            print_prf_per_type(msg, scores["cats_f_per_type"], "Textcat F", "label")
+            data["cats_f_per_type"] = scores["cats_f_per_type"]
+    if "cats_auc_per_type" in scores:
+        if scores["cats_auc_per_type"]:
+            print_textcats_auc_per_cat(msg, scores["cats_auc_per_type"])
+            data["cats_auc_per_type"] = scores["cats_auc_per_type"]
+    return scores
+
+
 def render_parses(
     docs: List[Doc],
     output_path: Path,

spacy/cli/init_pipeline.py

@@ -108,6 +108,10 @@ def init_labels_cli(
         config = util.load_config(config_path, overrides=overrides)
     with show_validation_error(hint_fill=False):
         nlp = init_nlp(config, use_gpu=use_gpu)
+    _init_labels(nlp, output_path)
+
+
+def _init_labels(nlp, output_path):
     for name, component in nlp.pipeline:
         if getattr(component, "label_data", None) is not None:
             output_file = output_path / f"{name}.json"

spacy/cli/package.py

@@ -1,7 +1,7 @@
 from typing import Optional, Union, Any, Dict, List, Tuple
 import shutil
 from pathlib import Path
-from wasabi import Printer, get_raw_input
+from wasabi import Printer, MarkdownRenderer, get_raw_input
 import srsly
 import sys
 
@@ -18,7 +18,7 @@ def package_cli(
     output_dir: Path = Arg(..., help="Output parent directory", exists=True, file_okay=False),
     code_paths: str = Opt("", "--code", "-c", help="Comma-separated paths to Python file with additional code (registered functions) to be included in the package"),
     meta_path: Optional[Path] = Opt(None, "--meta-path", "--meta", "-m", help="Path to meta.json", exists=True, dir_okay=False),
-    create_meta: bool = Opt(False, "--create-meta", "-c", "-C", help="Create meta.json, even if one exists"),
+    create_meta: bool = Opt(False, "--create-meta", "-C", help="Create meta.json, even if one exists"),
     name: Optional[str] = Opt(None, "--name", "-n", help="Package name to override meta"),
     version: Optional[str] = Opt(None, "--version", "-v", help="Package version to override meta"),
     build: str = Opt("sdist", "--build", "-b", help="Comma-separated formats to build: sdist and/or wheel, or none."),
@@ -113,7 +113,7 @@ def package(
         print("\n".join(errors))
         sys.exit(1)
     model_name = meta["name"]
-    if not model_name.startswith(meta['lang'] + "_"):
+    if not model_name.startswith(meta["lang"] + "_"):
         model_name = f"{meta['lang']}_{model_name}"
     model_name_v = model_name + "-" + meta["version"]
     main_path = output_dir / model_name_v
@@ -130,9 +130,15 @@ def package(
             )
     Path.mkdir(package_path, parents=True)
     shutil.copytree(str(input_dir), str(package_path / model_name_v))
-    license_path = package_path / model_name_v / "LICENSE"
-    if license_path.exists():
-        shutil.move(str(license_path), str(main_path))
+    for file_name in FILENAMES_DOCS:
+        file_path = package_path / model_name_v / file_name
+        if file_path.exists():
+            shutil.copy(str(file_path), str(main_path))
+    readme_path = main_path / "README.md"
+    if not readme_path.exists():
+        readme = generate_readme(meta)
+        create_file(readme_path, readme)
+        create_file(package_path / model_name_v / "README.md", readme)
     imports = []
     for code_path in code_paths:
         imports.append(code_path.stem)
@@ -233,6 +239,113 @@ def generate_meta(existing_meta: Dict[str, Any], msg: Printer) -> Dict[str, Any]
     return meta
 
 
+def generate_readme(meta: Dict[str, Any]) -> str:
+    """
+    Generate a Markdown-formatted README text from a model meta.json. Used
+    within the GitHub release notes and as content for README.md file added
+    to model packages.
+    """
+    md = MarkdownRenderer()
+    lang = meta["lang"]
+    name = f"{lang}_{meta['name']}"
+    version = meta["version"]
+    pipeline = ", ".join([md.code(p) for p in meta.get("pipeline", [])])
+    components = ", ".join([md.code(p) for p in meta.get("components", [])])
+    vecs = meta.get("vectors", {})
+    vectors = f"{vecs.get('keys', 0)} keys, {vecs.get('vectors', 0)} unique vectors ({ vecs.get('width', 0)} dimensions)"
+    author = meta.get("author") or "n/a"
+    notes = meta.get("notes", "")
+    license_name = meta.get("license")
+    sources = _format_sources(meta.get("sources"))
+    description = meta.get("description")
+    label_scheme = _format_label_scheme(meta.get("labels"))
+    accuracy = _format_accuracy(meta.get("performance"))
+    table_data = [
+        (md.bold("Name"), md.code(name)),
+        (md.bold("Version"), md.code(version)),
+        (md.bold("spaCy"), md.code(meta["spacy_version"])),
+        (md.bold("Default Pipeline"), pipeline),
+        (md.bold("Components"), components),
+        (md.bold("Vectors"), vectors),
+        (md.bold("Sources"), sources or "n/a"),
+        (md.bold("License"), md.code(license_name) if license_name else "n/a"),
+        (md.bold("Author"), md.link(author, meta["url"]) if "url" in meta else author),
+    ]
+    # Put together Markdown body
+    if description:
+        md.add(description)
+    md.add(md.table(table_data, ["Feature", "Description"]))
+    if label_scheme:
+        md.add(md.title(3, "Label Scheme"))
+        md.add(label_scheme)
+    if accuracy:
+        md.add(md.title(3, "Accuracy"))
+        md.add(accuracy)
+    if notes:
+        md.add(notes)
+    return md.text
+
+
+def _format_sources(data: Any) -> str:
+    if not data or not isinstance(data, list):
+        return "n/a"
+    sources = []
+    for source in data:
+        if not isinstance(source, dict):
+            source = {"name": source}
+        name = source.get("name")
+        if not name:
+            continue
+        url = source.get("url")
+        author = source.get("author")
+        result = name if not url else "[{}]({})".format(name, url)
+        if author:
+            result += " ({})".format(author)
+        sources.append(result)
+    return "<br />".join(sources)
+
+
+def _format_accuracy(data: Dict[str, Any], exclude: List[str] = ["speed"]) -> str:
+    if not data:
+        return ""
+    md = MarkdownRenderer()
+    scalars = [(k, v) for k, v in data.items() if isinstance(v, (int, float))]
+    scores = [
+        (md.code(acc.upper()), f"{score*100:.2f}")
+        for acc, score in scalars
+        if acc not in exclude
+    ]
+    md.add(md.table(scores, ["Type", "Score"]))
+    return md.text
+
+
+def _format_label_scheme(data: Dict[str, Any]) -> str:
+    if not data:
+        return ""
+    md = MarkdownRenderer()
+    n_labels = 0
+    n_pipes = 0
+    label_data = []
+    for pipe, labels in data.items():
+        if not labels:
+            continue
+        col1 = md.bold(md.code(pipe))
+        col2 = ", ".join(
+            [md.code(label.replace("|", "\\|")) for label in labels]
+        )  # noqa: W605
+        label_data.append((col1, col2))
+        n_labels += len(labels)
+        n_pipes += 1
+    if not label_data:
+        return ""
+    label_info = f"View label scheme ({n_labels} labels for {n_pipes} components)"
+    md.add("<details>")
+    md.add(f"<summary>{label_info}</summary>")
+    md.add(md.table(label_data, ["Component", "Labels"]))
+    md.add("</details>")
+    return md.text
+
+
 TEMPLATE_SETUP = """
 #!/usr/bin/env python
 import io
@@ -247,6 +360,13 @@ def load_meta(fp):
         return json.load(f)
 
 
+def load_readme(fp):
+    if path.exists(fp):
+        with io.open(fp, encoding='utf8') as f:
+            return f.read()
+    return ""
+
+
 def list_files(data_dir):
     output = []
     for root, _, filenames in walk(data_dir):
@@ -272,6 +392,8 @@ def setup_package():
     root = path.abspath(path.dirname(__file__))
     meta_path = path.join(root, 'meta.json')
     meta = load_meta(meta_path)
+    readme_path = path.join(root, 'README.md')
+    readme = load_readme(readme_path)
     model_name = str(meta['lang'] + '_' + meta['name'])
     model_dir = path.join(model_name, model_name + '-' + meta['version'])
 
@@ -281,6 +403,7 @@ def setup_package():
     setup(
         name=model_name,
         description=meta.get('description'),
+        long_description=readme,
         author=meta.get('author'),
         author_email=meta.get('email'),
         url=meta.get('url'),
@@ -302,6 +425,8 @@ if __name__ == '__main__':
 TEMPLATE_MANIFEST = """
 include meta.json
 include LICENSE
+include LICENSES_SOURCES
+include README.md
 """.strip()
 
 
@@ -317,3 +442,6 @@ __version__ = get_model_meta(Path(__file__).parent)['version']
 def load(**overrides):
     return load_model_from_init_py(__file__, **overrides)
 """.lstrip()
+
+
+FILENAMES_DOCS = ["LICENSE", "LICENSES_SOURCES", "README.md"]

spacy/cli/templates/quickstart_training.jinja

@@ -151,14 +151,14 @@ grad_factor = 1.0
 @layers = "reduce_mean.v1"
 
 [components.textcat.model.linear_model]
-@architectures = "spacy.TextCatBOW.v1"
+@architectures = "spacy.TextCatBOW.v2"
 exclusive_classes = true
 ngram_size = 1
 no_output_layer = false
 
 {% else -%}
 [components.textcat.model]
-@architectures = "spacy.TextCatBOW.v1"
+@architectures = "spacy.TextCatBOW.v2"
 exclusive_classes = true
 ngram_size = 1
 no_output_layer = false
@@ -182,14 +182,14 @@ grad_factor = 1.0
 @layers = "reduce_mean.v1"
 
 [components.textcat_multilabel.model.linear_model]
-@architectures = "spacy.TextCatBOW.v1"
+@architectures = "spacy.TextCatBOW.v2"
 exclusive_classes = false
 ngram_size = 1
 no_output_layer = false
 
 {% else -%}
 [components.textcat_multilabel.model]
-@architectures = "spacy.TextCatBOW.v1"
+@architectures = "spacy.TextCatBOW.v2"
 exclusive_classes = false
 ngram_size = 1
 no_output_layer = false
@@ -316,14 +316,14 @@ nO = null
 width = ${components.tok2vec.model.encode.width}
 
 [components.textcat.model.linear_model]
-@architectures = "spacy.TextCatBOW.v1"
+@architectures = "spacy.TextCatBOW.v2"
 exclusive_classes = true
 ngram_size = 1
 no_output_layer = false
 
 {% else -%}
 [components.textcat.model]
-@architectures = "spacy.TextCatBOW.v1"
+@architectures = "spacy.TextCatBOW.v2"
 exclusive_classes = true
 ngram_size = 1
 no_output_layer = false
@@ -344,14 +344,14 @@ nO = null
 width = ${components.tok2vec.model.encode.width}
 
 [components.textcat_multilabel.model.linear_model]
-@architectures = "spacy.TextCatBOW.v1"
+@architectures = "spacy.TextCatBOW.v2"
 exclusive_classes = false
 ngram_size = 1
 no_output_layer = false
 
 {% else -%}
 [components.textcat_multilabel.model]
-@architectures = "spacy.TextCatBOW.v1"
+@architectures = "spacy.TextCatBOW.v2"
 exclusive_classes = false
 ngram_size = 1
 no_output_layer = false
@@ -372,7 +372,7 @@ factory = "{{ pipe }}"
 [corpora.train]
 @readers = "spacy.Corpus.v1"
 path = ${paths.train}
-max_length = {{ 500 if hardware == "gpu" else 2000 }}
+max_length = 0
 
 [corpora.dev]
 @readers = "spacy.Corpus.v1"
@@ -418,7 +418,7 @@ compound = 1.001
 
 [initialize]
 {% if use_transformer or optimize == "efficiency" or not word_vectors -%}
-vectors = null
+vectors = ${paths.vectors}
 {% else -%}
 vectors = "{{ word_vectors }}"
 {% endif -%}

spacy/cli/train.py

@@ -28,7 +28,7 @@ def train_cli(
     """
     Train or update a spaCy pipeline. Requires data in spaCy's binary format. To
     convert data from other formats, use the `spacy convert` command. The
-    config file includes all settings and hyperparameters used during traing.
+    config file includes all settings and hyperparameters used during training.
     To override settings in the config, e.g. settings that point to local
     paths or that you want to experiment with, you can override them as
     command line options. For instance, --training.batch_size 128 overrides

spacy/cli/validate.py

@@ -3,10 +3,11 @@ from pathlib import Path
 import sys
 import requests
 from wasabi import msg, Printer
+import warnings
 
 from ._util import app
 from .. import about
-from ..util import get_package_version, get_installed_models, get_base_version
+from ..util import get_package_version, get_installed_models, get_minor_version
 from ..util import get_package_path, get_model_meta, is_compatible_version
 
 
@@ -24,7 +25,7 @@ def validate_cli():
 
 def validate() -> None:
     model_pkgs, compat = get_model_pkgs()
-    spacy_version = get_base_version(about.__version__)
+    spacy_version = get_minor_version(about.__version__)
     current_compat = compat.get(spacy_version, {})
     if not current_compat:
         msg.warn(f"No compatible packages found for v{spacy_version} of spaCy")
@@ -44,8 +45,8 @@ def validate() -> None:
                 comp = msg.text("", color="green", icon="good", no_print=True)
                 version = msg.text(data["version"], color="green", no_print=True)
             else:
-                version = msg.text(data["version"], color="red", no_print=True)
-                comp = f"--> {compat.get(data['name'], ['n/a'])[0]}"
+                version = msg.text(data["version"], color="yellow", no_print=True)
+                comp = f"--> {current_compat.get(data['name'], ['n/a'])[0]}"
             rows.append((data["name"], data["spacy"], version, comp))
         msg.table(rows, header=header)
     else:
@@ -78,7 +79,9 @@ def get_model_pkgs(silent: bool = False) -> Tuple[dict, dict]:
     msg.good("Loaded compatibility table")
     compat = r.json()["spacy"]
     all_models = set()
-    installed_models = get_installed_models()
+    with warnings.catch_warnings():
+        warnings.filterwarnings("ignore", message="\\[W09[45]")
+        installed_models = get_installed_models()
     for spacy_v, models in dict(compat).items():
         all_models.update(models.keys())
         for model, model_vs in models.items():
@@ -92,7 +95,9 @@ def get_model_pkgs(silent: bool = False) -> Tuple[dict, dict]:
             spacy_version = about.__version__
         else:
             model_path = get_package_path(package)
-            model_meta = get_model_meta(model_path)
+            with warnings.catch_warnings():
+                warnings.filterwarnings("ignore", message="\\[W09[45]")
+                model_meta = get_model_meta(model_path)
             spacy_version = model_meta.get("spacy_version", "n/a")
             is_compat = is_compatible_version(about.__version__, spacy_version)
         pkgs[pkg_name] = {

spacy/displacy/__init__.py

@@ -120,7 +120,9 @@ def parse_deps(orig_doc: Doc, options: Dict[str, Any] = {}) -> Dict[str, Any]:
     doc (Doc): Document do parse.
     RETURNS (dict): Generated dependency parse keyed by words and arcs.
     """
-    doc = Doc(orig_doc.vocab).from_bytes(orig_doc.to_bytes(exclude=["user_data", "user_hooks"]))
+    doc = Doc(orig_doc.vocab).from_bytes(
+        orig_doc.to_bytes(exclude=["user_data", "user_hooks"])
+    )
     if not doc.has_annotation("DEP"):
         warnings.warn(Warnings.W005)
     if options.get("collapse_phrases", False):

spacy/errors.py

@@ -1,3 +1,6 @@
+import warnings
+
+
 def add_codes(err_cls):
     """Add error codes to string messages via class attribute names."""
 
@@ -12,6 +15,33 @@ def add_codes(err_cls):
     return ErrorsWithCodes()
 
 
+def setup_default_warnings():
+    # ignore certain numpy warnings
+    filter_warning("ignore", error_msg="numpy.dtype size changed")  # noqa
+    filter_warning("ignore", error_msg="numpy.ufunc size changed")  # noqa
+
+    # warn about entity_ruler & matcher having no patterns only once
+    for pipe in ["matcher", "entity_ruler"]:
+        filter_warning("once", error_msg=Warnings.W036.format(name=pipe))
+
+    # warn once about lemmatizer without required POS
+    filter_warning("once", error_msg="[W108]")
+
+
+def filter_warning(action: str, error_msg: str):
+    """Customize how spaCy should handle a certain warning.
+
+    error_msg (str): e.g. "W006", or a full error message
+    action (str): "default", "error", "ignore", "always", "module" or "once"
+    """
+    warnings.filterwarnings(action, message=_escape_warning_msg(error_msg))
+
+
+def _escape_warning_msg(msg):
+    """To filter with warnings.filterwarnings, the [] brackets need to be escaped"""
+    return msg.replace("[", "\\[").replace("]", "\\]")
+
+
 # fmt: off
 
 @add_codes
@@ -80,8 +110,9 @@ class Warnings:
             "@misc = \"spacy.LookupsDataLoader.v1\"\n"
             "lang = ${{nlp.lang}}\n"
             "tables = [\"lexeme_norm\"]\n")
-    W035 = ('Discarding subpattern "{pattern}" due to an unrecognized '
+    W035 = ("Discarding subpattern '{pattern}' due to an unrecognized "
             "attribute or operator.")
+    W036 = ("The component '{name}' does not have any patterns defined.")
 
     # New warnings added in v3.x
     W086 = ("Component '{listener}' will be (re)trained, but it needs the component "
@@ -119,12 +150,12 @@ class Warnings:
             "released, because the model may say it's compatible when it's "
             'not. Consider changing the "spacy_version" in your meta.json to a '
             "version range, with a lower and upper pin. For example: {example}")
-    W095 = ("Model '{model}' ({model_version}) requires spaCy {version} and is "
-            "incompatible with the current version ({current}). This may lead "
-            "to unexpected results or runtime errors. To resolve this, "
-            "download a newer compatible model or retrain your custom model "
-            "with the current spaCy version. For more details and available "
-            "updates, run: python -m spacy validate")
+    W095 = ("Model '{model}' ({model_version}) was trained with spaCy "
+            "{version} and may not be 100% compatible with the current version "
+            "({current}). If you see errors or degraded performance, download "
+            "a newer compatible model or retrain your custom model with the "
+            "current spaCy version. For more details and available updates, "
+            "run: python -m spacy validate")
     W096 = ("The method `nlp.disable_pipes` is now deprecated - use "
             "`nlp.select_pipes` instead.")
     W100 = ("Skipping unsupported morphological feature(s): '{feature}'. "
@@ -490,6 +521,18 @@ class Errors:
     E202 = ("Unsupported alignment mode '{mode}'. Supported modes: {modes}.")
 
     # New errors added in v3.x
+    E867 = ("The 'textcat' component requires at least two labels because it "
+            "uses mutually exclusive classes where exactly one label is True "
+            "for each doc. For binary classification tasks, you can use two "
+            "labels with 'textcat' (LABEL / NOT_LABEL) or alternatively, you "
+            "can use the 'textcat_multilabel' component with one label.")
+    E868 = ("Found a conflicting gold annotation in a reference document, "
+            "with the following char-based span occurring both in the gold ents "
+            "as well as in the negative spans: {span}.")
+    E869 = ("The notation '{label}' is not supported anymore. To annotate "
+            "negative NER samples, use `doc.spans[key]` instead, and "
+            "specify the key as 'incorrect_spans_key' when constructing "
+            "the NER component.")
     E870 = ("Could not serialize the DocBin because it is too large. Consider "
             "splitting up your documents into several doc bins and serializing "
             "each separately. spacy.Corpus.v1 will search recursively for all "
@@ -815,6 +858,12 @@ class Errors:
              "DependencyMatcher token patterns. The token pattern in "
              "RIGHT_ATTR should return matches that are each exactly one token "
              "long. Invalid pattern:\n{node}")
+    E1017 = ("A Doc object requires both 'deps' and 'heads' for dependency "
+             "parses. If no dependency labels are available, provide "
+             "placeholder deps such as `deps=[\"dep\"]*len(heads)`.")
+    E1018 = ("Knowledge base for component '{name}' is not set. "
+             "Make sure either `nel.initialize` or `nel.set_kb` "
+             "is called with a `kb_loader` function.")
 
 
 # Deprecated model shortcuts, only used in errors and warnings

spacy/lang/az/__init__.py

@@ -1,16 +1,11 @@
-from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS, TOKEN_MATCH
 from .stop_words import STOP_WORDS
-from .syntax_iterators import SYNTAX_ITERATORS
 from .lex_attrs import LEX_ATTRS
 from ...language import Language
 
 
 class AzerbaijaniDefaults(Language.Defaults):
-    tokenizer_exceptions = TOKENIZER_EXCEPTIONS
     lex_attr_getters = LEX_ATTRS
     stop_words = STOP_WORDS
-    token_match = TOKEN_MATCH
-    syntax_iterators = SYNTAX_ITERATORS
 
 
 class Azerbaijani(Language):

spacy/lang/bg/lex_attrs.py

@@ -22,13 +22,13 @@ _num_words = [
     "тринадесет",
     "тринайсет",
     "четиринадесет",
-    "четиринайсет"
+    "четиринайсет",
     "петнадесет",
-    "петнайсет"
+    "петнайсет",
     "шестнадесет",
     "шестнайсет",
     "седемнадесет",
-    "седемнайсет"
+    "седемнайсет",
     "осемнадесет",
     "осемнайсет",
     "деветнадесет",
@@ -36,7 +36,7 @@ _num_words = [
     "двадесет",
     "двайсет",
     "тридесет",
-    "трийсет"
+    "трийсет",
     "четиридесет",
     "четиресет",
     "петдесет",

spacy/lang/bg/tokenizer_exceptions.py

@@ -58,7 +58,6 @@ _abbr_dot_exc = [
     {ORTH: "стр.", NORM: "страница"},
     {ORTH: "ул.", NORM: "улица"},
     {ORTH: "чл.", NORM: "член"},
-
 ]
 
 for abbr in _abbr_dot_exc:

spacy/lang/ca/__init__.py

@@ -1,15 +1,23 @@
+from typing import Optional
+
+from thinc.api import Model
+
 from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS
-from .punctuation import TOKENIZER_INFIXES
+from .punctuation import TOKENIZER_INFIXES, TOKENIZER_SUFFIXES
 from .stop_words import STOP_WORDS
 from .lex_attrs import LEX_ATTRS
+from .syntax_iterators import SYNTAX_ITERATORS
 from ...language import Language
+from .lemmatizer import CatalanLemmatizer
 
 
 class CatalanDefaults(Language.Defaults):
     tokenizer_exceptions = TOKENIZER_EXCEPTIONS
     infixes = TOKENIZER_INFIXES
+    suffixes = TOKENIZER_SUFFIXES
     stop_words = STOP_WORDS
     lex_attr_getters = LEX_ATTRS
+    syntax_iterators = SYNTAX_ITERATORS
 
 
 class Catalan(Language):
@@ -17,4 +25,16 @@ class Catalan(Language):
     Defaults = CatalanDefaults
 
 
+@Catalan.factory(
+    "lemmatizer",
+    assigns=["token.lemma"],
+    default_config={"model": None, "mode": "rule", "overwrite": False},
+    default_score_weights={"lemma_acc": 1.0},
+)
+def make_lemmatizer(
+    nlp: Language, model: Optional[Model], name: str, mode: str, overwrite: bool
+):
+    return CatalanLemmatizer(nlp.vocab, model, name, mode=mode, overwrite=overwrite)
+
+
 __all__ = ["Catalan"]

spacy/lang/ca/lemmatizer.py

@@ -0,0 +1,81 @@
+from typing import List, Tuple
+
+from ...pipeline import Lemmatizer
+from ...tokens import Token
+
+
+class CatalanLemmatizer(Lemmatizer):
+    """
+    Copied from French Lemmatizer
+    Catalan language lemmatizer applies the default rule based lemmatization
+    procedure with some modifications for better Catalan language support.
+
+    The parts of speech 'ADV', 'PRON', 'DET', 'ADP' and 'AUX' are added to use
+    the rule-based lemmatization. As a last resort, the lemmatizer checks in
+    the lookup table.
+    """
+
+    @classmethod
+    def get_lookups_config(cls, mode: str) -> Tuple[List[str], List[str]]:
+        if mode == "rule":
+            required = ["lemma_lookup", "lemma_rules", "lemma_exc", "lemma_index"]
+            return (required, [])
+        else:
+            return super().get_lookups_config(mode)
+
+    def rule_lemmatize(self, token: Token) -> List[str]:
+        cache_key = (token.orth, token.pos)
+        if cache_key in self.cache:
+            return self.cache[cache_key]
+        string = token.text
+        univ_pos = token.pos_.lower()
+        if univ_pos in ("", "eol", "space"):
+            return [string.lower()]
+        elif "lemma_rules" not in self.lookups or univ_pos not in (
+            "noun",
+            "verb",
+            "adj",
+            "adp",
+            "adv",
+            "aux",
+            "cconj",
+            "det",
+            "pron",
+            "punct",
+            "sconj",
+        ):
+            return self.lookup_lemmatize(token)
+        index_table = self.lookups.get_table("lemma_index", {})
+        exc_table = self.lookups.get_table("lemma_exc", {})
+        rules_table = self.lookups.get_table("lemma_rules", {})
+        lookup_table = self.lookups.get_table("lemma_lookup", {})
+        index = index_table.get(univ_pos, {})
+        exceptions = exc_table.get(univ_pos, {})
+        rules = rules_table.get(univ_pos, [])
+        string = string.lower()
+        forms = []
+        if string in index:
+            forms.append(string)
+            self.cache[cache_key] = forms
+            return forms
+        forms.extend(exceptions.get(string, []))
+        oov_forms = []
+        if not forms:
+            for old, new in rules:
+                if string.endswith(old):
+                    form = string[: len(string) - len(old)] + new
+                    if not form:
+                        pass
+                    elif form in index or not form.isalpha():
+                        forms.append(form)
+                    else:
+                        oov_forms.append(form)
+        if not forms:
+            forms.extend(oov_forms)
+        if not forms and string in lookup_table.keys():
+            forms.append(self.lookup_lemmatize(token)[0])
+        if not forms:
+            forms.append(string)
+        forms = list(set(forms))
+        self.cache[cache_key] = forms
+        return forms

spacy/lang/ca/punctuation.py

@@ -1,12 +1,46 @@
-from ..punctuation import TOKENIZER_INFIXES
-from ..char_classes import ALPHA
+from ..char_classes import LIST_PUNCT, LIST_ELLIPSES, LIST_QUOTES, LIST_ICONS
+from ..char_classes import CURRENCY
+from ..char_classes import CONCAT_QUOTES, ALPHA_LOWER, ALPHA_UPPER, ALPHA, PUNCT
+from ..char_classes import merge_chars, _units
 
 
 ELISION = " ' ’ ".strip().replace(" ", "").replace("\n", "")
 
 
-_infixes = TOKENIZER_INFIXES + [
-    r"(?<=[{a}][{el}])(?=[{a}])".format(a=ALPHA, el=ELISION)
-]
+_infixes = (
+    LIST_ELLIPSES
+    + LIST_ICONS
+    + [
+        r"(?<=[0-9])[+\-\*^](?=[0-9-])",
+        r"(?<=[{al}{q}])\.(?=[{au}{q}])".format(
+            al=ALPHA_LOWER, au=ALPHA_UPPER, q=CONCAT_QUOTES
+        ),
+        r"(?<=[{a}]),(?=[{a}])".format(a=ALPHA),
+        r"(?<=[{a}0-9])[:<>=/](?=[{a}])".format(a=ALPHA),
+        r"(?<=[{a}][{el}])(?=[{a}0-9])".format(a=ALPHA, el=ELISION),
+    ]
+)
+
+_units = _units.replace("% ", "")
+UNITS = merge_chars(_units)
+
+_suffixes = (
+    LIST_PUNCT
+    + LIST_ELLIPSES
+    + LIST_QUOTES
+    + LIST_ICONS
+    + [r"-", "—", "–"]
+    + [
+        r"(?<=[0-9])\+",
+        r"(?<=°[FfCcKk])\.",
+        r"(?<=[0-9])(?:{c})".format(c=CURRENCY),
+        r"(?<=[0-9])(?:{u})".format(u=UNITS),
+        r"(?<=[0-9{al}{e}{p}(?:{q})])\.".format(
+            al=ALPHA_LOWER, e=r"%²\-\+", q=CONCAT_QUOTES, p=PUNCT
+        ),
+        r"(?<=[{au}][{au}])\.".format(au=ALPHA_UPPER),
+    ]
+)
 
 TOKENIZER_INFIXES = _infixes
+TOKENIZER_SUFFIXES = _suffixes

spacy/lang/ca/syntax_iterators.py

@@ -0,0 +1,46 @@
+from ...symbols import NOUN, PROPN
+from ...errors import Errors
+
+
+def noun_chunks(doclike):
+    """Detect base noun phrases from a dependency parse. Works on Doc and Span."""
+    # fmt: off
+    labels = ["nsubj", "nsubj:pass", "obj", "obl", "iobj", "ROOT", "appos", "nmod", "nmod:poss"]
+    # fmt: on
+    doc = doclike.doc  # Ensure works on both Doc and Span.
+    if not doc.has_annotation("DEP"):
+        raise ValueError(Errors.E029)
+    np_deps = [doc.vocab.strings[label] for label in labels]
+    np_label = doc.vocab.strings.add("NP")
+    prev_end = -1
+    for i, word in enumerate(doclike):
+        if word.pos not in (NOUN, PROPN):
+            continue
+        # Prevent nested chunks from being produced
+        if word.left_edge.i <= prev_end:
+            continue
+        if word.dep in np_deps:
+            left = word.left_edge.i
+            right = word.right_edge.i + 1
+            # leave prepositions and punctuation out of the left side of the chunk
+            if word.left_edge.pos_ == "ADP" or word.left_edge.pos_ == "PUNCT":
+                left = word.left_edge.i + 1
+            prev_end = word.right_edge.i
+            # leave subordinated clauses and appositions out of the chunk
+            a = word.i + 1
+            while a < word.right_edge.i:
+                paraula = doc[a]
+                if paraula.pos_ == "VERB":
+                    right = paraula.left_edge.i
+                    prev_end = paraula.left_edge.i - 1
+                elif paraula.dep_ == "appos":
+                    right = paraula.left_edge.i + 1
+                    prev_end = paraula.left_edge.i - 1
+                a += 1
+            # leave punctuation out of the right side of the chunk
+            if word.right_edge.pos_ == "PUNCT":
+                right = right - 1
+            yield left, right, np_label
+
+
+SYNTAX_ITERATORS = {"noun_chunks": noun_chunks}

spacy/lang/ca/tokenizer_exceptions.py

@@ -24,6 +24,13 @@ for exc_data in [
     {ORTH: "núm", NORM: "número"},
     {ORTH: "St.", NORM: "sant"},
     {ORTH: "Sta.", NORM: "santa"},
+    {ORTH: "'l"},
+    {ORTH: "'ls"},
+    {ORTH: "'m"},
+    {ORTH: "'n"},
+    {ORTH: "'ns"},
+    {ORTH: "'s"},
+    {ORTH: "'t"},
 ]:
     _exc[exc_data[ORTH]] = [exc_data]
 

spacy/lang/char_classes.py

@@ -260,7 +260,10 @@ _units = (
     "кг г мг м/с км/ч кПа Па мбар Кб КБ кб Мб МБ мб Гб ГБ гб Тб ТБ тб"
     "كم كم² كم³ م م² م³ سم سم² سم³ مم مم² مم³ كم غرام جرام جم كغ ملغ كوب اكواب"
 )
-_currency = r"\$ £ € ¥ ฿ US\$ C\$ A\$ ₽ ﷼ ₴"
+_currency = (
+    r"\$ £ € ¥ ฿ US\$ C\$ A\$ ₽ ﷼ ₴ ₠ ₡ ₢ ₣ ₤ ₥ ₦ ₧ ₨ ₩ ₪ ₫ € ₭ ₮ ₯ ₰ "
+    r"₱ ₲ ₳ ₴ ₵ ₶ ₷ ₸ ₹ ₺ ₻ ₼ ₽ ₾ ₿"
+)
 
 # These expressions contain various unicode variations, including characters
 # used in Chinese (see #1333, #1340, #1351) – unless there are cross-language

spacy/lang/el/lemmatizer.py

@@ -57,6 +57,6 @@ class GreekLemmatizer(Lemmatizer):
             forms.extend(oov_forms)
         if not forms:
             forms.append(string)
-        forms = list(set(forms))
+        forms = list(dict.fromkeys(forms))
         self.cache[cache_key] = forms
         return forms

spacy/lang/fi/tokenizer_exceptions.py

@@ -1,5 +1,5 @@
 from ..tokenizer_exceptions import BASE_EXCEPTIONS
-from ...symbols import ORTH
+from ...symbols import ORTH, NORM
 from ...util import update_exc
 
 
@@ -79,5 +79,34 @@ for exc_data in [
 ]:
     _exc[exc_data[ORTH]] = [exc_data]
 
+# Source: https://kaino.kotus.fi/visk/sisallys.php?p=141
+conj_contraction_bases = [
+    ("ett", "että"),
+    ("jott", "jotta"),
+    ("kosk", "koska"),
+    ("mutt", "mutta"),
+    ("vaikk", "vaikka"),
+    ("ehk", "ehkä"),
+    ("miks", "miksi"),
+    ("siks", "siksi"),
+    ("joll", "jos"),
+    ("ell", "jos"),
+]
+conj_contraction_negations = [
+    ("en", "en"),
+    ("et", "et"),
+    ("ei", "ei"),
+    ("emme", "emme"),
+    ("ette", "ette"),
+    ("eivat", "eivät"),
+    ("eivät", "eivät"),
+]
+for (base_lower, base_norm) in conj_contraction_bases:
+    for base in [base_lower, base_lower.title()]:
+        for (suffix, suffix_norm) in conj_contraction_negations:
+            _exc[base + suffix] = [
+                {ORTH: base, NORM: base_norm},
+                {ORTH: suffix, NORM: suffix_norm},
+            ]
 
 TOKENIZER_EXCEPTIONS = update_exc(BASE_EXCEPTIONS, _exc)

spacy/lang/fr/stop_words.py

@@ -1,30 +1,31 @@
 STOP_WORDS = set(
     """
 a à â abord afin ah ai aie ainsi ait allaient allons
-alors anterieur anterieure anterieures apres après as assez attendu au
-aucun aucune aujourd aujourd'hui aupres auquel aura auraient aurait auront
+alors anterieur anterieure anterieures antérieur antérieure antérieures
+apres après as assez attendu au
+aupres auquel aura auraient aurait auront
 aussi autre autrement autres autrui aux auxquelles auxquels avaient
 avais avait avant avec avoir avons ayant
 
 bas basee bat
 
-c' c’ ça car ce ceci cela celle celle-ci celle-là celles celles-ci celles-là celui
-celui-ci celui-là cent cependant certain certaine certaines certains certes ces
+c' c’ ça car ce ceci cela celle celle-ci celle-la celle-là celles celles-ci celles-la celles-là
+celui celui-ci celui-la celui-là cent cependant certain certaine certaines certains certes ces
 cet cette ceux ceux-ci ceux-là chacun chacune chaque chez ci cinq cinquantaine cinquante
 cinquantième cinquième combien comme comment compris concernant
 
-d' d’ da dans de debout dedans dehors deja delà depuis derriere
+d' d’ da dans de debout dedans dehors deja dejà delà depuis derriere
 derrière des desormais desquelles desquels dessous dessus deux deuxième
-deuxièmement devant devers devra different differentes differents différent
+deuxièmement devant devers devra different differente differentes differents différent
 différente différentes différents dire directe directement dit dite dits divers
 diverse diverses dix dix-huit dix-neuf dix-sept dixième doit doivent donc dont
-douze douzième du duquel durant dès désormais
+douze douzième du duquel durant dès déja déjà désormais
 
-effet egale egalement egales eh elle elle-même elles elles-mêmes en encore
+effet egalement eh elle elle-meme elle-même elles elles-memes elles-mêmes en encore
 enfin entre envers environ es ès est et etaient étaient etais étais etait était
-etant étant etc été etre être eu eux eux-mêmes exactement excepté
+etant étant etc etre être eu eux eux-mêmes exactement excepté également
 
-fais faisaient faisant fait façon feront font
+fais faisaient faisant fait facon façon feront font
 
 gens
 
@@ -36,45 +37,48 @@ j' j’ je jusqu jusque juste
 l' l’ la laisser laquelle le lequel les lesquelles lesquels leur leurs longtemps
 lors lorsque lui lui-meme lui-même là lès
 
-m' m’ ma maint maintenant mais malgre me meme memes merci mes mien
+m' m’ ma maint maintenant mais malgre malgré me meme memes merci mes mien
 mienne miennes miens mille moi moi-meme moi-même moindres moins
 mon même mêmes
 
 n' n’ na ne neanmoins neuvième ni nombreuses nombreux nos notamment
-notre nous nous-mêmes nouvea nul néanmoins nôtre nôtres
+notre nous nous-mêmes nouveau nul néanmoins nôtre nôtres
 
-o ô on ont onze onzième ore ou ouias oust outre
+o ô on ont onze onzième or ou ouias ouste outre
 ouvert ouverte ouverts où
 
-par parce parfois parle parlent parler parmi parseme partant
+par parce parfois parle parlent parler parmi partant
 pas pendant pense permet personne peu peut peuvent peux plus
-plusieurs plutôt possible possibles pour pourquoi
-pourrais pourrait pouvait prealable precisement premier première premièrement
-pres procedant proche près pu puis puisque
+plusieurs plutot plutôt possible possibles pour pourquoi
+pourrais pourrait pouvait prealable precisement
+premier première premièrement
+pres procedant proche près préalable précisement pu puis puisque
 
-qu' qu’ quand quant quant-à-soi quanta quarante quatorze quatre quatre-vingt
+qu' qu’ quand quant quant-à-soi quarante quatorze quatre quatre-vingt
 quatrième quatrièmement que quel quelconque quelle quelles quelqu'un quelque
 quelques quels qui quiconque quinze quoi quoique
 
 relative relativement rend rendre restant reste
-restent retour revoici revoilà
+restent retour revoici revoila revoilà
 
 s' s’ sa sait sans sauf se seize selon semblable semblaient
 semble semblent sent sept septième sera seraient serait seront ses seul seule
-seulement si sien sienne siennes siens sinon six sixième soi soi-même soit
-soixante son sont sous souvent specifique specifiques stop
+seulement seuls seules si sien sienne siennes siens sinon six sixième soi soi-meme soi-même soit
+soixante son sont sous souvent specifique specifiques spécifique spécifiques stop
 suffisant suffisante suffit suis suit suivant suivante
 suivantes suivants suivre sur surtout
 
 t' t’ ta tant te tel telle tellement telles tels tenant tend tenir tente
-tes tien tienne tiennes tiens toi toi-même ton touchant toujours tous
-tout toute toutes treize trente tres trois troisième troisièmement
+tes tien tienne tiennes tiens toi toi-meme toi-même ton touchant toujours tous
+tout toute toutes treize trente tres trois troisième troisièmement très
 tu té
 
 un une unes uns
 
-va vais vas vers via vingt voici voilà vont vos
-votre vous vous-mêmes vu vé vôtre vôtres
+va vais vas vers via vingt voici voila voilà vont vos
+votre votres vous vous-mêmes vu vé vôtre vôtres
+
+y
 
 """.split()
 )

spacy/lang/it/__init__.py

@@ -1,7 +1,11 @@
+from typing import Optional
+from thinc.api import Model
+
 from .stop_words import STOP_WORDS
 from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS
 from .punctuation import TOKENIZER_PREFIXES, TOKENIZER_INFIXES
 from ...language import Language
+from .lemmatizer import ItalianLemmatizer
 
 
 class ItalianDefaults(Language.Defaults):
@@ -16,4 +20,16 @@ class Italian(Language):
     Defaults = ItalianDefaults
 
 
+@Italian.factory(
+    "lemmatizer",
+    assigns=["token.lemma"],
+    default_config={"model": None, "mode": "pos_lookup", "overwrite": False},
+    default_score_weights={"lemma_acc": 1.0},
+)
+def make_lemmatizer(
+    nlp: Language, model: Optional[Model], name: str, mode: str, overwrite: bool
+):
+    return ItalianLemmatizer(nlp.vocab, model, name, mode=mode, overwrite=overwrite)
+
+
 __all__ = ["Italian"]

spacy/lang/it/lemmatizer.py

@@ -0,0 +1,132 @@
+from typing import List, Dict, Tuple
+
+from ...pipeline import Lemmatizer
+from ...tokens import Token
+
+
+class ItalianLemmatizer(Lemmatizer):
+    """This lemmatizer was adapted from the Polish one (version of April 2021).
+    It implements lookup lemmatization based on the morphological lexicon
+    morph-it (Baroni and Zanchetta). The table lemma_lookup with non-POS-aware
+    entries is used as a backup for words that aren't handled by morph-it."""
+
+    @classmethod
+    def get_lookups_config(cls, mode: str) -> Tuple[List[str], List[str]]:
+        if mode == "pos_lookup":
+            required = [
+                "lemma_lookup_num",
+                "lemma_lookup_det",
+                "lemma_lookup_adp",
+                "lemma_lookup_adj",
+                "lemma_lookup_noun",
+                "lemma_lookup_pron",
+                "lemma_lookup_verb",
+                "lemma_lookup_aux",
+                "lemma_lookup_adv",
+                "lemma_lookup_other",
+                "lemma_lookup",
+            ]
+            return (required, [])
+        else:
+            return super().get_lookups_config(mode)
+
+    def pos_lookup_lemmatize(self, token: Token) -> List[str]:
+        string = token.text
+        univ_pos = token.pos_
+        morphology = token.morph.to_dict()
+        lookup_pos = univ_pos.lower()
+        if univ_pos == "PROPN":
+            lookup_pos = "noun"
+        elif univ_pos == "PART":
+            lookup_pos = "pron"
+        lookup_table = self.lookups.get_table("lemma_lookup_" + lookup_pos, {})
+        if univ_pos == "NOUN":
+            return self.lemmatize_noun(string, morphology, lookup_table)
+        else:
+            if univ_pos != "PROPN":
+                string = string.lower()
+            if univ_pos == "DET":
+                return self.lemmatize_det(string, morphology, lookup_table)
+            elif univ_pos == "PRON":
+                return self.lemmatize_pron(string, morphology, lookup_table)
+            elif univ_pos == "ADP":
+                return self.lemmatize_adp(string, morphology, lookup_table)
+            elif univ_pos == "ADJ":
+                return self.lemmatize_adj(string, morphology, lookup_table)
+            else:
+                lemma = lookup_table.get(string, "")
+        if not lemma:
+            lookup_table = self.lookups.get_table("lemma_lookup_other")
+            lemma = lookup_table.get(string, "")
+        if not lemma:
+            lookup_table = self.lookups.get_table(
+                "lemma_lookup"
+            )  # "legacy" lookup table
+            lemma = lookup_table.get(string, string.lower())
+        return [lemma]
+
+    def lemmatize_det(
+        self, string: str, morphology: dict, lookup_table: Dict[str, str]
+    ) -> List[str]:
+        if string in [
+            "l'",
+            "lo",
+            "la",
+            "i",
+            "gli",
+            "le",
+        ]:
+            return ["il"]
+        if string in ["un'", "un", "una"]:
+            return ["uno"]
+        return [lookup_table.get(string, string)]
+
+    def lemmatize_pron(
+        self, string: str, morphology: dict, lookup_table: Dict[str, str]
+    ) -> List[str]:
+        if string in [
+            "l'",
+            "li",
+            "la",
+            "gli",
+            "le",
+        ]:
+            return ["lo"]
+        if string in ["un'", "un", "una"]:
+            return ["uno"]
+        lemma = lookup_table.get(string, string)
+        if lemma == "alcun":
+            lemma = "alcuno"
+        elif lemma == "qualcun":
+            lemma = "qualcuno"
+        return [lemma]
+
+    def lemmatize_adp(
+        self, string: str, morphology: dict, lookup_table: Dict[str, str]
+    ) -> List[str]:
+        if string == "d'":
+            return ["di"]
+        return [lookup_table.get(string, string)]
+
+    def lemmatize_adj(
+        self, string: str, morphology: dict, lookup_table: Dict[str, str]
+    ) -> List[str]:
+        lemma = lookup_table.get(string, string)
+        if lemma == "alcun":
+            lemma = "alcuno"
+        elif lemma == "qualcun":
+            lemma = "qualcuno"
+        return [lemma]
+
+    def lemmatize_noun(
+        self, string: str, morphology: dict, lookup_table: Dict[str, str]
+    ) -> List[str]:
+        # this method is case-sensitive, in order to work
+        # for incorrectly tagged proper names
+        if string != string.lower():
+            if string.lower() in lookup_table:
+                return [lookup_table[string.lower()]]
+            elif string in lookup_table:
+                return [lookup_table[string]]
+            return [string.lower()]
+        return [lookup_table.get(string, string)]

spacy/lang/it/tokenizer_exceptions.py

@@ -25,7 +25,7 @@ for orth in [
     "artt.",
     "att.",
     "avv.",
-    "Avv."
+    "Avv.",
     "by-pass",
     "c.d.",
     "c/c",

spacy/lang/ru/lemmatizer.py

@@ -12,7 +12,6 @@ PUNCT_RULES = {"«": '"', "»": '"'}
 
 
 class RussianLemmatizer(Lemmatizer):
-    _morph = None
 
     def __init__(
         self,
@@ -23,15 +22,16 @@ class RussianLemmatizer(Lemmatizer):
         mode: str = "pymorphy2",
         overwrite: bool = False,
     ) -> None:
-        try:
-            from pymorphy2 import MorphAnalyzer
-        except ImportError:
-            raise ImportError(
-                "The Russian lemmatizer requires the pymorphy2 library: "
-                'try to fix it with "pip install pymorphy2"'
-            ) from None
-        if RussianLemmatizer._morph is None:
-            RussianLemmatizer._morph = MorphAnalyzer()
+        if mode == "pymorphy2":
+            try:
+                from pymorphy2 import MorphAnalyzer
+            except ImportError:
+                raise ImportError(
+                    "The Russian lemmatizer mode 'pymorphy2' requires the "
+                    "pymorphy2 library. Install it with: pip install pymorphy2"
+                ) from None
+            if getattr(self, "_morph", None) is None:
+                self._morph = MorphAnalyzer()
         super().__init__(vocab, model, name, mode=mode, overwrite=overwrite)
 
     def pymorphy2_lemmatize(self, token: Token) -> List[str]:

spacy/lang/tokenizer_exceptions.py

@@ -35,8 +35,8 @@ URL_PATTERN = (
     # host & domain names
     # mods: match is case-sensitive, so include [A-Z]
     r"(?:"  # noqa: E131
-      r"(?:"
-        r"[A-Za-z0-9\u00a1-\uffff]"
+      r"(?:"  # noqa: E131
+        r"[A-Za-z0-9\u00a1-\uffff]"  # noqa: E131
         r"[A-Za-z0-9\u00a1-\uffff_-]{0,62}"
       r")?"
       r"[A-Za-z0-9\u00a1-\uffff]\."

spacy/lang/uk/lemmatizer.py

@@ -7,8 +7,6 @@ from ...vocab import Vocab
 
 
 class UkrainianLemmatizer(RussianLemmatizer):
-    _morph = None
-
     def __init__(
         self,
         vocab: Vocab,
@@ -18,14 +16,15 @@ class UkrainianLemmatizer(RussianLemmatizer):
         mode: str = "pymorphy2",
         overwrite: bool = False,
     ) -> None:
-        try:
-            from pymorphy2 import MorphAnalyzer
-        except ImportError:
-            raise ImportError(
-                "The Ukrainian lemmatizer requires the pymorphy2 library and "
-                "dictionaries: try to fix it with "
-                '"pip install pymorphy2 pymorphy2-dicts-uk"'
-            ) from None
-        if UkrainianLemmatizer._morph is None:
-            UkrainianLemmatizer._morph = MorphAnalyzer(lang="uk")
+        if mode == "pymorphy2":
+            try:
+                from pymorphy2 import MorphAnalyzer
+            except ImportError:
+                raise ImportError(
+                    "The Ukrainian lemmatizer mode 'pymorphy2' requires the "
+                    "pymorphy2 library and dictionaries. Install them with: "
+                    "pip install pymorphy2 pymorphy2-dicts-uk"
+                ) from None
+            if getattr(self, "_morph", None) is None:
+                self._morph = MorphAnalyzer(lang="uk")
         super().__init__(vocab, model, name, mode=mode, overwrite=overwrite)

spacy/language.py

@@ -1,4 +1,5 @@
-from typing import Optional, Any, Dict, Callable, Iterable, Union, List, Pattern
+from typing import Iterator, Optional, Any, Dict, Callable, Iterable, TypeVar
+from typing import Union, List, Pattern, overload
 from typing import Tuple
 from dataclasses import dataclass
 import random
@@ -687,11 +688,13 @@ class Language:
         if not isinstance(source, Language):
             raise ValueError(Errors.E945.format(name=source_name, source=type(source)))
         # Check vectors, with faster checks first
-        if self.vocab.vectors.shape != source.vocab.vectors.shape or \
-                self.vocab.vectors.key2row != source.vocab.vectors.key2row or \
-                self.vocab.vectors.to_bytes() != source.vocab.vectors.to_bytes():
-            util.logger.warning(Warnings.W113.format(name=source_name))
-        if not source_name in source.component_names:
+        if (
+            self.vocab.vectors.shape != source.vocab.vectors.shape
+            or self.vocab.vectors.key2row != source.vocab.vectors.key2row
+            or self.vocab.vectors.to_bytes() != source.vocab.vectors.to_bytes()
+        ):
+            warnings.warn(Warnings.W113.format(name=source_name))
+        if source_name not in source.component_names:
             raise KeyError(
                 Errors.E944.format(
                     name=source_name,
@@ -932,6 +935,7 @@ class Language:
         # because factory may be used for something else
         self._pipe_meta.pop(name)
         self._pipe_configs.pop(name)
+        self.meta.get("_sourced_vectors_hashes", {}).pop(name, None)
         # Make sure name is removed from the [initialize] config
         if name in self._config["initialize"]["components"]:
             self._config["initialize"]["components"].pop(name)
@@ -1428,6 +1432,21 @@ class Language:
                 except StopIteration:
                     pass
 
+    _AnyContext = TypeVar("_AnyContext")
+
+    @overload
+    def pipe(
+        self,
+        texts: Iterable[Tuple[str, _AnyContext]],
+        *,
+        as_tuples: bool = ...,
+        batch_size: Optional[int] = ...,
+        disable: Iterable[str] = ...,
+        component_cfg: Optional[Dict[str, Dict[str, Any]]] = ...,
+        n_process: int = ...,
+    ) -> Iterator[Tuple[Doc, _AnyContext]]:
+        ...
+
     def pipe(
         self,
         texts: Iterable[str],
@@ -1437,7 +1456,7 @@ class Language:
         disable: Iterable[str] = SimpleFrozenList(),
         component_cfg: Optional[Dict[str, Dict[str, Any]]] = None,
         n_process: int = 1,
-    ):
+    ) -> Iterator[Doc]:
         """Process texts as a stream, and yield `Doc` objects in order.
 
         texts (Iterable[str]): A sequence of texts to process.
@@ -1539,15 +1558,21 @@ class Language:
 
         # Cycle channels not to break the order of docs.
         # The received object is a batch of byte-encoded docs, so flatten them with chain.from_iterable.
-        byte_tuples = chain.from_iterable(recv.recv() for recv in cycle(bytedocs_recv_ch))
+        byte_tuples = chain.from_iterable(
+            recv.recv() for recv in cycle(bytedocs_recv_ch)
+        )
         try:
-            for i, (_, (byte_doc, byte_error)) in enumerate(zip(raw_texts, byte_tuples), 1):
+            for i, (_, (byte_doc, byte_error)) in enumerate(
+                zip(raw_texts, byte_tuples), 1
+            ):
                 if byte_doc is not None:
                     doc = Doc(self.vocab).from_bytes(byte_doc)
                     yield doc
                 elif byte_error is not None:
                     error = srsly.msgpack_loads(byte_error)
-                    self.default_error_handler(None, None, None, ValueError(Errors.E871.format(error=error)))
+                    self.default_error_handler(
+                        None, None, None, ValueError(Errors.E871.format(error=error))
+                    )
                 if i % batch_size == 0:
                     # tell `sender` that one batch was consumed.
                     sender.step()
@@ -1672,6 +1697,8 @@ class Language:
         # If components are loaded from a source (existing models), we cache
         # them here so they're only loaded once
         source_nlps = {}
+        source_nlp_vectors_hashes = {}
+        nlp.meta["_sourced_vectors_hashes"] = {}
         for pipe_name in config["nlp"]["pipeline"]:
             if pipe_name not in pipeline:
                 opts = ", ".join(pipeline.keys())
@@ -1696,17 +1723,33 @@ class Language:
                 else:
                     model = pipe_cfg["source"]
                     if model not in source_nlps:
-                        # We only need the components here and we need to init
-                        # model with the same vocab as the current nlp object
-                        source_nlps[model] = util.load_model(model, vocab=nlp.vocab)
+                        # We only need the components here and we intentionally
+                        # do not load the model with the same vocab because
+                        # this would cause the vectors to be copied into the
+                        # current nlp object (all the strings will be added in
+                        # create_pipe_from_source)
+                        source_nlps[model] = util.load_model(model)
                     source_name = pipe_cfg.get("component", pipe_name)
                     listeners_replaced = False
                     if "replace_listeners" in pipe_cfg:
                         for name, proc in source_nlps[model].pipeline:
                             if source_name in getattr(proc, "listening_components", []):
-                                source_nlps[model].replace_listeners(name, source_name, pipe_cfg["replace_listeners"])
+                                source_nlps[model].replace_listeners(
+                                    name, source_name, pipe_cfg["replace_listeners"]
+                                )
                                 listeners_replaced = True
-                    nlp.add_pipe(source_name, source=source_nlps[model], name=pipe_name)
+                    with warnings.catch_warnings():
+                        warnings.filterwarnings("ignore", message="\\[W113\\]")
+                        nlp.add_pipe(
+                            source_name, source=source_nlps[model], name=pipe_name
+                        )
+                    if model not in source_nlp_vectors_hashes:
+                        source_nlp_vectors_hashes[model] = hash(
+                            source_nlps[model].vocab.vectors.to_bytes()
+                        )
+                    nlp.meta["_sourced_vectors_hashes"][
+                        pipe_name
+                    ] = source_nlp_vectors_hashes[model]
                     # Delete from cache if listeners were replaced
                     if listeners_replaced:
                         del source_nlps[model]
@@ -1724,12 +1767,16 @@ class Language:
         for name, proc in nlp.pipeline:
             # Remove listeners not in the pipeline
             listener_names = getattr(proc, "listening_components", [])
-            unused_listener_names = [ll for ll in listener_names if ll not in nlp.pipe_names]
+            unused_listener_names = [
+                ll for ll in listener_names if ll not in nlp.pipe_names
+            ]
             for listener_name in unused_listener_names:
                 for listener in proc.listener_map.get(listener_name, []):
                     proc.remove_listener(listener, listener_name)
 
-            for listener in getattr(proc, "listening_components", []):  # e.g. tok2vec/transformer
+            for listener in getattr(
+                proc, "listening_components", []
+            ):  # e.g. tok2vec/transformer
                 # If it's a component sourced from another pipeline, we check if
                 # the tok2vec listeners should be replaced with standalone tok2vec
                 # models (e.g. so component can be frozen without its performance
@@ -1786,6 +1833,7 @@ class Language:
             raise ValueError(err)
         tok2vec = self.get_pipe(tok2vec_name)
         tok2vec_cfg = self.get_pipe_config(tok2vec_name)
+        tok2vec_model = tok2vec.model
         if (
             not hasattr(tok2vec, "model")
             or not hasattr(tok2vec, "listener_map")
@@ -1794,6 +1842,7 @@ class Language:
         ):
             raise ValueError(Errors.E888.format(name=tok2vec_name, pipe=type(tok2vec)))
         pipe_listeners = tok2vec.listener_map.get(pipe_name, [])
+        pipe = self.get_pipe(pipe_name)
         pipe_cfg = self._pipe_configs[pipe_name]
         if listeners:
             util.logger.debug(f"Replacing listeners of component '{pipe_name}'")
@@ -1808,7 +1857,6 @@ class Language:
                     n_listeners=len(pipe_listeners),
                 )
                 raise ValueError(err)
-            pipe = self.get_pipe(pipe_name)
             # Update the config accordingly by copying the tok2vec model to all
             # sections defined in the listener paths
             for listener_path in listeners:
@@ -1820,10 +1868,19 @@ class Language:
                         name=pipe_name, tok2vec=tok2vec_name, path=listener_path
                     )
                     raise ValueError(err)
-                util.set_dot_to_object(pipe_cfg, listener_path, tok2vec_cfg["model"])
+                new_config = tok2vec_cfg["model"]
+                if "replace_listener_cfg" in tok2vec_model.attrs:
+                    replace_func = tok2vec_model.attrs["replace_listener_cfg"]
+                    new_config = replace_func(
+                        tok2vec_cfg["model"], pipe_cfg["model"]["tok2vec"]
+                    )
+                util.set_dot_to_object(pipe_cfg, listener_path, new_config)
             # Go over the listener layers and replace them
             for listener in pipe_listeners:
-                util.replace_model_node(pipe.model, listener, tok2vec.model.copy())
+                new_model = tok2vec_model.copy()
+                if "replace_listener" in tok2vec_model.attrs:
+                    new_model = tok2vec_model.attrs["replace_listener"](new_model)
+                util.replace_model_node(pipe.model, listener, new_model)
                 tok2vec.remove_listener(listener, pipe_name)
 
     def to_disk(
@@ -1855,7 +1912,11 @@ class Language:
         util.to_disk(path, serializers, exclude)
 
     def from_disk(
-        self, path: Union[str, Path], *, exclude: Iterable[str] = SimpleFrozenList()
+        self,
+        path: Union[str, Path],
+        *,
+        exclude: Iterable[str] = SimpleFrozenList(),
+        overrides: Dict[str, Any] = SimpleFrozenDict(),
     ) -> "Language":
         """Loads state from a directory. Modifies the object in place and
         returns it. If the saved `Language` object contains a model, the
@@ -1884,7 +1945,7 @@ class Language:
         deserializers = {}
         if Path(path / "config.cfg").exists():
             deserializers["config.cfg"] = lambda p: self.config.from_disk(
-                p, interpolate=False
+                p, interpolate=False, overrides=overrides
             )
         deserializers["meta.json"] = deserialize_meta
         deserializers["vocab"] = deserialize_vocab

spacy/lexeme.pyx

@@ -163,7 +163,7 @@ cdef class Lexeme:
             self.vocab.set_vector(self.c.orth, vector)
 
     property rank:
-        """RETURNS (str): Sequential ID of the lexemes's lexical type, used
+        """RETURNS (str): Sequential ID of the lexeme's lexical type, used
             to index into tables, e.g. for word vectors."""
         def __get__(self):
             return self.c.id
@@ -205,7 +205,7 @@ cdef class Lexeme:
             self.c.lower = x
 
     property norm:
-        """RETURNS (uint64): The lexemes's norm, i.e. a normalised form of the
+        """RETURNS (uint64): The lexeme's norm, i.e. a normalised form of the
             lexeme text.
         """
         def __get__(self):
@@ -288,7 +288,7 @@ cdef class Lexeme:
             self.c.lower = self.vocab.strings.add(x)
 
     property norm_:
-        """RETURNS (str): The lexemes's norm, i.e. a normalised form of the
+        """RETURNS (str): The lexeme's norm, i.e. a normalised form of the
             lexeme text.
         """
         def __get__(self):

spacy/lookups.py

@@ -12,9 +12,7 @@ from .strings import get_string_id
 UNSET = object()
 
 
-def load_lookups(
-    lang: str, tables: List[str], strict: bool = True
-) -> 'Lookups':
+def load_lookups(lang: str, tables: List[str], strict: bool = True) -> "Lookups":
     """Load the data from the spacy-lookups-data package for a given language,
     if available. Returns an empty `Lookups` container if there's no data or if the package
     is not installed.

spacy/matcher/dependencymatcher.pyx

@@ -4,6 +4,7 @@ from collections import defaultdict
 from itertools import product
 
 import numpy
+import warnings
 
 from .matcher cimport Matcher
 from ..vocab cimport Vocab
@@ -11,7 +12,6 @@ from ..tokens.doc cimport Doc
 
 from ..errors import Errors, Warnings
 from ..tokens import Span
-from ..util import logger
 
 
 DELIMITER = "||"
@@ -282,7 +282,7 @@ cdef class DependencyMatcher:
         keys_to_position_maps = defaultdict(lambda: defaultdict(list))
         for match_id, start, end in self._matcher(doc):
             if start + 1 != end:
-                logger.warning(Warnings.W110.format(tokens=[t.text for t in doc[start:end]], pattern=self._matcher.get(match_id)[1][0][0]))
+                warnings.warn(Warnings.W110.format(tokens=[t.text for t in doc[start:end]], pattern=self._matcher.get(match_id)[1][0][0]))
             token = doc[start]
             root = ([token] + list(token.ancestors))[-1]
             keys_to_position_maps[root.i][match_id].append(start)

spacy/matcher/matcher.pyx

@@ -138,6 +138,11 @@ cdef class Matcher:
         self._filter[key] = greedy
         self._patterns[key].extend(patterns)
 
+    def _require_patterns(self) -> None:
+        """Raise a warning if this component has no patterns defined."""
+        if len(self) == 0:
+            warnings.warn(Warnings.W036.format(name="matcher"))
+
     def remove(self, key):
         """Remove a rule from the matcher. A KeyError is raised if the key does
         not exist.
@@ -215,6 +220,7 @@ cdef class Matcher:
             If with_alignments is set to True and as_spans is set to False,
             A list of `(match_id, start, end, alignments)` tuples is returned.
         """
+        self._require_patterns()
         if isinstance(doclike, Doc):
             doc = doclike
             length = len(doc)

spacy/matcher/phrasematcher.pyx

@@ -50,6 +50,8 @@ cdef class PhraseMatcher:
         if isinstance(attr, (int, long)):
             self.attr = attr
         else:
+            if attr is None:
+                attr = "ORTH"
             attr = attr.upper()
             if attr == "TEXT":
                 attr = "ORTH"

spacy/ml/_precomputable_affine.py

@@ -1,6 +1,9 @@
 from thinc.api import Model, normal_init
 
+from ..util import registry
 
+
+@registry.layers("spacy.PrecomputableAffine.v1")
 def PrecomputableAffine(nO, nI, nF, nP, dropout=0.1):
     model = Model(
         "precomputable_affine",

spacy/ml/extract_ngrams.py

@@ -1,8 +1,10 @@
 from thinc.api import Model
 
+from ..util import registry
 from ..attrs import LOWER
 
 
+@registry.layers("spacy.extract_ngrams.v1")
 def extract_ngrams(ngram_size: int, attr: int = LOWER) -> Model:
     model = Model("extract_ngrams", forward)
     model.attrs["ngram_size"] = ngram_size

spacy/ml/extract_spans.py

@@ -0,0 +1,60 @@
+from typing import Tuple, Callable
+from thinc.api import Model, to_numpy
+from thinc.types import Ragged, Ints1d
+
+from ..util import registry
+
+
+@registry.layers("spacy.extract_spans.v1")
+def extract_spans() -> Model[Tuple[Ragged, Ragged], Ragged]:
+    """Extract spans from a sequence of source arrays, as specified by an array
+    of (start, end) indices. The output is a ragged array of the
+    extracted spans.
+    """
+    return Model(
+        "extract_spans", forward, layers=[], refs={}, attrs={}, dims={}, init=init
+    )
+
+
+def init(model, X=None, Y=None):
+    pass
+
+
+def forward(
+    model: Model, source_spans: Tuple[Ragged, Ragged], is_train: bool
+) -> Tuple[Ragged, Callable]:
+    """Get subsequences from source vectors."""
+    ops = model.ops
+    X, spans = source_spans
+    assert spans.dataXd.ndim == 2
+    indices = _get_span_indices(ops, spans, X.lengths)
+    Y = Ragged(X.dataXd[indices], spans.dataXd[:, 1] - spans.dataXd[:, 0])
+    x_shape = X.dataXd.shape
+    x_lengths = X.lengths
+
+    def backprop_windows(dY: Ragged) -> Tuple[Ragged, Ragged]:
+        dX = Ragged(ops.alloc2f(*x_shape), x_lengths)
+        ops.scatter_add(dX.dataXd, indices, dY.dataXd)
+        return (dX, spans)
+
+    return Y, backprop_windows
+
+
+def _get_span_indices(ops, spans: Ragged, lengths: Ints1d) -> Ints1d:
+    """Construct a flat array that has the indices we want to extract from the
+    source data. For instance, if we want the spans (5, 9), (8, 10) the
+    indices will be [5, 6, 7, 8, 8, 9].
+    """
+    spans, lengths = _ensure_cpu(spans, lengths)
+    indices = []
+    offset = 0
+    for i, length in enumerate(lengths):
+        spans_i = spans[i].dataXd + offset
+        for j in range(spans_i.shape[0]):
+            indices.append(ops.xp.arange(spans_i[j, 0], spans_i[j, 1]))
+        offset += length
+    return ops.flatten(indices)
+
+
+def _ensure_cpu(spans: Ragged, lengths: Ints1d) -> Tuple[Ragged, Ints1d]:
+    return (Ragged(to_numpy(spans.dataXd), to_numpy(spans.lengths)), to_numpy(lengths))

spacy/ml/models/__init__.py

@@ -2,6 +2,7 @@ from .coref import *
 from .entity_linker import *  # noqa
 from .multi_task import *  # noqa
 from .parser import *  # noqa
+from .spancat import *  # noqa
 from .tagger import *  # noqa
 from .textcat import *  # noqa
 from .tok2vec import *  # noqa

spacy/ml/models/entity_linker.py

@@ -6,6 +6,7 @@ from thinc.api import Model, Maxout, Linear
 from ...util import registry
 from ...kb import KnowledgeBase, Candidate, get_candidates
 from ...vocab import Vocab
+from ...tokens import Span
 
 
 @registry.architectures("spacy.EntityLinker.v1")
@@ -44,5 +45,5 @@ def empty_kb(entity_vector_length: int) -> Callable[[Vocab], KnowledgeBase]:
 
 
 @registry.misc("spacy.CandidateGenerator.v1")
-def create_candidates() -> Callable[[KnowledgeBase, "Span"], Iterable[Candidate]]:
+def create_candidates() -> Callable[[KnowledgeBase, Span], Iterable[Candidate]]:
     return get_candidates

spacy/ml/models/multi_task.py

@@ -13,7 +13,7 @@ from functools import partial
 if TYPE_CHECKING:
     # This lets us add type hints for mypy etc. without causing circular imports
     from ...vocab import Vocab  # noqa: F401
-    from ...tokens import Doc  # noqa: F401
+    from ...tokens.doc import Doc  # noqa: F401
 
 
 @registry.architectures("spacy.PretrainVectors.v1")
@@ -205,7 +205,7 @@ def _apply_mask(
     docs: Iterable["Doc"], random_words: _RandomWords, mask_prob: float = 0.15
 ) -> Tuple[numpy.ndarray, List["Doc"]]:
     # This needs to be here to avoid circular imports
-    from ...tokens import Doc  # noqa: F811
+    from ...tokens.doc import Doc  # noqa: F811
 
     N = sum(len(doc) for doc in docs)
     mask = numpy.random.uniform(0.0, 1.0, (N,))

spacy/ml/models/parser.py

@@ -10,48 +10,7 @@ from ..tb_framework import TransitionModel
 from ...tokens import Doc
 
 
-@registry.architectures("spacy.TransitionBasedParser.v1")
-def transition_parser_v1(
-    tok2vec: Model[List[Doc], List[Floats2d]],
-    state_type: Literal["parser", "ner"],
-    extra_state_tokens: bool,
-    hidden_width: int,
-    maxout_pieces: int,
-    use_upper: bool = True,
-    nO: Optional[int] = None,
-) -> Model:
-    return build_tb_parser_model(
-        tok2vec,
-        state_type,
-        extra_state_tokens,
-        hidden_width,
-        maxout_pieces,
-        use_upper,
-        nO,
-    )
-
-
 @registry.architectures("spacy.TransitionBasedParser.v2")
-def transition_parser_v2(
-    tok2vec: Model[List[Doc], List[Floats2d]],
-    state_type: Literal["parser", "ner"],
-    extra_state_tokens: bool,
-    hidden_width: int,
-    maxout_pieces: int,
-    use_upper: bool,
-    nO: Optional[int] = None,
-) -> Model:
-    return build_tb_parser_model(
-        tok2vec,
-        state_type,
-        extra_state_tokens,
-        hidden_width,
-        maxout_pieces,
-        use_upper,
-        nO,
-    )
-
-
 def build_tb_parser_model(
     tok2vec: Model[List[Doc], List[Floats2d]],
     state_type: Literal["parser", "ner"],

spacy/ml/models/spancat.py

@@ -0,0 +1,54 @@
+from typing import List, Tuple
+from thinc.api import Model, with_getitem, chain, list2ragged, Logistic
+from thinc.api import Maxout, Linear, concatenate, glorot_uniform_init
+from thinc.api import reduce_mean, reduce_max, reduce_first, reduce_last
+from thinc.types import Ragged, Floats2d
+
+from ...util import registry
+from ...tokens import Doc
+from ..extract_spans import extract_spans
+
+
+@registry.layers.register("spacy.LinearLogistic.v1")
+def build_linear_logistic(nO=None, nI=None) -> Model[Floats2d, Floats2d]:
+    """An output layer for multi-label classification. It uses a linear layer
+    followed by a logistic activation.
+    """
+    return chain(Linear(nO=nO, nI=nI, init_W=glorot_uniform_init), Logistic())
+
+
+@registry.layers.register("spacy.mean_max_reducer.v1")
+def build_mean_max_reducer(hidden_size: int) -> Model[Ragged, Floats2d]:
+    """Reduce sequences by concatenating their mean and max pooled vectors,
+    and then combine the concatenated vectors with a hidden layer.
+    """
+    return chain(
+        concatenate(reduce_last(), reduce_first(), reduce_mean(), reduce_max()),
+        Maxout(nO=hidden_size, normalize=True, dropout=0.0),
+    )
+
+
+@registry.architectures.register("spacy.SpanCategorizer.v1")
+def build_spancat_model(
+    tok2vec: Model[List[Doc], List[Floats2d]],
+    reducer: Model[Ragged, Floats2d],
+    scorer: Model[Floats2d, Floats2d],
+) -> Model[Tuple[List[Doc], Ragged], Floats2d]:
+    """Build a span categorizer model, given a token-to-vector model, a
+    reducer model to map the sequence of vectors for each span down to a single
+    vector, and a scorer model to map the vectors to probabilities.
+
+    tok2vec (Model[List[Doc], List[Floats2d]]): The tok2vec model.
+    reducer (Model[Ragged, Floats2d]): The reducer model.
+    scorer (Model[Floats2d, Floats2d]): The scorer model.
+    """
+    model = chain(
+        with_getitem(0, chain(tok2vec, list2ragged())),
+        extract_spans(),
+        reducer,
+        scorer,
+    )
+    model.set_ref("tok2vec", tok2vec)
+    model.set_ref("reducer", reducer)
+    model.set_ref("scorer", scorer)
+    return model

spacy/ml/models/textcat.py

@@ -1,11 +1,13 @@
+from functools import partial
 from typing import Optional, List
 
 from thinc.types import Floats2d
 from thinc.api import Model, reduce_mean, Linear, list2ragged, Logistic
 from thinc.api import chain, concatenate, clone, Dropout, ParametricAttention
 from thinc.api import SparseLinear, Softmax, softmax_activation, Maxout, reduce_sum
-from thinc.api import with_cpu, Relu, residual, LayerNorm
+from thinc.api import with_cpu, Relu, residual, LayerNorm, resizable
 from thinc.layers.chain import init as init_chain
+from thinc.layers.resizable import resize_model, resize_linear_weighted
 
 from ...attrs import ORTH
 from ...util import registry
@@ -15,7 +17,10 @@ from ...tokens import Doc
 from .tok2vec import get_tok2vec_width
 
 
-@registry.architectures("spacy.TextCatCNN.v1")
+NEG_VALUE = -5000
+
+
+@registry.architectures("spacy.TextCatCNN.v2")
 def build_simple_cnn_text_classifier(
     tok2vec: Model, exclusive_classes: bool, nO: Optional[int] = None
 ) -> Model[List[Doc], Floats2d]:
@@ -25,38 +30,75 @@ def build_simple_cnn_text_classifier(
     outputs sum to 1. If exclusive_classes=False, a logistic non-linearity
     is applied instead, so that outputs are in the range [0, 1].
     """
+    fill_defaults = {"b": 0, "W": 0}
     with Model.define_operators({">>": chain}):
         cnn = tok2vec >> list2ragged() >> reduce_mean()
+        nI = tok2vec.maybe_get_dim("nO")
         if exclusive_classes:
-            output_layer = Softmax(nO=nO, nI=tok2vec.maybe_get_dim("nO"))
-            model = cnn >> output_layer
-            model.set_ref("output_layer", output_layer)
+            output_layer = Softmax(nO=nO, nI=nI)
+            fill_defaults["b"] = NEG_VALUE
+            resizable_layer = resizable(
+                output_layer,
+                resize_layer=partial(
+                    resize_linear_weighted, fill_defaults=fill_defaults
+                ),
+            )
+            model = cnn >> resizable_layer
         else:
-            linear_layer = Linear(nO=nO, nI=tok2vec.maybe_get_dim("nO"))
-            model = cnn >> linear_layer >> Logistic()
-            model.set_ref("output_layer", linear_layer)
+            output_layer = Linear(nO=nO, nI=nI)
+            resizable_layer = resizable(
+                output_layer,
+                resize_layer=partial(
+                    resize_linear_weighted, fill_defaults=fill_defaults
+                ),
+            )
+            model = cnn >> resizable_layer >> Logistic()
+        model.set_ref("output_layer", output_layer)
+        model.attrs["resize_output"] = partial(
+            resize_and_set_ref,
+            resizable_layer=resizable_layer,
+        )
     model.set_ref("tok2vec", tok2vec)
     model.set_dim("nO", nO)
     model.attrs["multi_label"] = not exclusive_classes
     return model
 
 
-@registry.architectures("spacy.TextCatBOW.v1")
+def resize_and_set_ref(model, new_nO, resizable_layer):
+    resizable_layer = resize_model(resizable_layer, new_nO)
+    model.set_ref("output_layer", resizable_layer.layers[0])
+    model.set_dim("nO", new_nO, force=True)
+    return model
+
+
+@registry.architectures("spacy.TextCatBOW.v2")
 def build_bow_text_classifier(
     exclusive_classes: bool,
     ngram_size: int,
     no_output_layer: bool,
     nO: Optional[int] = None,
 ) -> Model[List[Doc], Floats2d]:
+    fill_defaults = {"b": 0, "W": 0}
     with Model.define_operators({">>": chain}):
-        sparse_linear = SparseLinear(nO)
-        model = extract_ngrams(ngram_size, attr=ORTH) >> sparse_linear
-        model = with_cpu(model, model.ops)
+        sparse_linear = SparseLinear(nO=nO)
+        output_layer = None
         if not no_output_layer:
+            fill_defaults["b"] = NEG_VALUE
             output_layer = softmax_activation() if exclusive_classes else Logistic()
+        resizable_layer = resizable(
+            sparse_linear,
+            resize_layer=partial(resize_linear_weighted, fill_defaults=fill_defaults),
+        )
+        model = extract_ngrams(ngram_size, attr=ORTH) >> resizable_layer
+        model = with_cpu(model, model.ops)
+        if output_layer:
             model = model >> with_cpu(output_layer, output_layer.ops)
+    model.set_dim("nO", nO)
     model.set_ref("output_layer", sparse_linear)
     model.attrs["multi_label"] = not exclusive_classes
+    model.attrs["resize_output"] = partial(
+        resize_and_set_ref, resizable_layer=resizable_layer
+    )
     return model
 
 
@@ -69,9 +111,7 @@ def build_text_classifier_v2(
     exclusive_classes = not linear_model.attrs["multi_label"]
     with Model.define_operators({">>": chain, "|": concatenate}):
         width = tok2vec.maybe_get_dim("nO")
-        attention_layer = ParametricAttention(
-            width
-        )  # TODO: benchmark performance difference of this layer
+        attention_layer = ParametricAttention(width)
         maxout_layer = Maxout(nO=width, nI=width)
         norm_layer = LayerNorm(nI=width)
         cnn_model = (

spacy/ml/tb_framework.py

@@ -1,7 +1,9 @@
 from thinc.api import Model, noop
 from .parser_model import ParserStepModel
+from ..util import registry
 
 
+@registry.layers("spacy.TransitionModel.v1")
 def TransitionModel(
     tok2vec, lower, upper, resize_output, dropout=0.2, unseen_classes=set()
 ):
@@ -15,7 +17,7 @@ def TransitionModel(
     return Model(
         name="parser_model",
         forward=forward,
-        dims={"nI": tok2vec.get_dim("nI") if tok2vec.has_dim("nI") else None},
+        dims={"nI": tok2vec.maybe_get_dim("nI")},
         layers=[tok2vec, lower, upper],
         refs={"tok2vec": tok2vec, "lower": lower, "upper": upper},
         init=init,

spacy/pipeline/__init__.py

@@ -12,6 +12,7 @@ from .senter import SentenceRecognizer
 from .sentencizer import Sentencizer
 from .tagger import Tagger
 from .textcat import TextCategorizer
+from .spancat import SpanCategorizer
 from .textcat_multilabel import MultiLabel_TextCategorizer
 from .tok2vec import Tok2Vec
 from .functions import merge_entities, merge_noun_chunks, merge_subtokens
@@ -28,6 +29,7 @@ __all__ = [
     "Pipe",
     "SentenceRecognizer",
     "Sentencizer",
+    "SpanCategorizer",
     "Tagger",
     "TextCategorizer",
     "Tok2Vec",

spacy/pipeline/_parser_internals/ner.pyx

@@ -1,3 +1,5 @@
+import os
+import random
 from libc.stdint cimport int32_t
 from cymem.cymem cimport Pool
 
@@ -6,10 +8,11 @@ from thinc.extra.search cimport Beam
 
 from ...tokens.doc cimport Doc
 from ...tokens.span import Span
+from ...tokens.span cimport Span
 from ...typedefs cimport weight_t, attr_t
 from ...lexeme cimport Lexeme
 from ...attrs cimport IS_SPACE
-from ...structs cimport TokenC
+from ...structs cimport TokenC, SpanC
 from ...training.example cimport Example
 from .stateclass cimport StateClass
 from ._state cimport StateC
@@ -25,7 +28,6 @@ cdef enum:
     LAST
     UNIT
     OUT
-    ISNT
     N_MOVES
 
 
@@ -36,39 +38,62 @@ MOVE_NAMES[IN] = 'I'
 MOVE_NAMES[LAST] = 'L'
 MOVE_NAMES[UNIT] = 'U'
 MOVE_NAMES[OUT] = 'O'
-MOVE_NAMES[ISNT] = 'x'
 
 
 cdef struct GoldNERStateC:
     Transition* ner
+    SpanC* negs
     int32_t length
+    int32_t nr_neg
 
 
 cdef class BiluoGold:
     cdef Pool mem
     cdef GoldNERStateC c
 
-    def __init__(self, BiluoPushDown moves, StateClass stcls, Example example):
+    def __init__(self, BiluoPushDown moves, StateClass stcls, Example example, neg_key):
         self.mem = Pool()
-        self.c = create_gold_state(self.mem, moves, stcls.c, example)
+        self.c = create_gold_state(self.mem, moves, stcls.c, example, neg_key)
 
     def update(self, StateClass stcls):
         update_gold_state(&self.c, stcls.c)
 
 
-
 cdef GoldNERStateC create_gold_state(
     Pool mem,
     BiluoPushDown moves,
     const StateC* stcls,
-    Example example
+    Example example,
+    neg_key
 ) except *:
     cdef GoldNERStateC gs
+    cdef Span neg
+    if neg_key is not None:
+        negs = example.get_aligned_spans_y2x(
+            example.y.spans.get(neg_key, []),
+            allow_overlap=True
+        )
+    else:
+        negs = []
     assert example.x.length > 0
     gs.ner = <Transition*>mem.alloc(example.x.length, sizeof(Transition))
-    ner_tags = example.get_aligned_ner()
+    gs.negs = <SpanC*>mem.alloc(len(negs), sizeof(SpanC))
+    gs.nr_neg = len(negs)
+    ner_ents, ner_tags = example.get_aligned_ents_and_ner()
     for i, ner_tag in enumerate(ner_tags):
         gs.ner[i] = moves.lookup_transition(ner_tag)
+
+    # Prevent conflicting spans in the data. For NER, spans are equal if they have the same offsets and label.
+    neg_span_triples = {(neg_ent.start_char, neg_ent.end_char, neg_ent.label) for neg_ent in negs}
+    for pos_span in ner_ents:
+        if (pos_span.start_char, pos_span.end_char, pos_span.label) in neg_span_triples:
+            raise ValueError(Errors.E868.format(span=(pos_span.start_char, pos_span.end_char, pos_span.label_)))
+
+    # In order to handle negative samples, we need to maintain the full
+    # (start, end, label) triple. If we break it down to the 'isnt B-LOC'
+    # thing, we'll get blocked if there's an incorrect prefix.
+    for i, neg in enumerate(negs):
+        gs.negs[i] = neg.c
     return gs
 
 
@@ -156,21 +181,16 @@ cdef class BiluoPushDown(TransitionSystem):
         cdef attr_t label
         if name == '-' or name == '' or name is None:
             return Transition(clas=0, move=MISSING, label=0, score=0)
-        elif name == '!O':
-            return Transition(clas=0, move=ISNT, label=0, score=0)
         elif '-' in name:
             move_str, label_str = name.split('-', 1)
-            # Hacky way to denote 'not this entity'
+            # Deprecated, hacky way to denote 'not this entity'
             if label_str.startswith('!'):
-                label_str = label_str[1:]
-                move_str = 'x'
+                raise ValueError(Errors.E869.format(label=name))
             label = self.strings.add(label_str)
         else:
             move_str = name
             label = 0
         move = MOVE_NAMES.index(move_str)
-        if move == ISNT:
-            return Transition(clas=0, move=ISNT, label=label, score=0)
         for i in range(self.n_moves):
             if self.c[i].move == move and self.c[i].label == label:
                 return self.c[i]
@@ -220,7 +240,7 @@ cdef class BiluoPushDown(TransitionSystem):
             label_id = label_name
         if action == OUT and label_id != 0:
             return None
-        if action == MISSING or action == ISNT:
+        if action == MISSING:
             return None
         # Check we're not creating a move we already have, so that this is
         # idempotent
@@ -270,9 +290,23 @@ cdef class BiluoPushDown(TransitionSystem):
         return parses
 
     def init_gold(self, StateClass state, Example example):
-        return BiluoGold(self, state, example)
+        return BiluoGold(self, state, example, self.neg_key)
 
     def has_gold(self, Example eg, start=0, end=None):
+        # We get x and y referring to X, we want to check relative to Y,
+        # the reference
+        y_spans = eg.get_aligned_spans_x2y([eg.x[start:end]])
+        if not y_spans:
+            y_spans = [eg.y[:]]
+        y_span = y_spans[0]
+        start = y_span.start
+        end = y_span.end
+        neg_key = self.neg_key
+        if neg_key is not None:
+            # If we have any negative samples, count that as having annotation.
+            for span in eg.y.spans.get(neg_key, []):
+                if span.start >= start and span.end <= end:
+                    return True
         for word in eg.y[start:end]:
             if word.ent_iob != 0:
                 return True
@@ -306,8 +340,6 @@ cdef class BiluoPushDown(TransitionSystem):
                 n_gold += costs[i] <= 0
             else:
                 costs[i] = 9000
-        if n_gold < 1:
-            raise ValueError
 
 
 cdef class Missing:
@@ -373,23 +405,33 @@ cdef class Begin:
     @staticmethod
     cdef weight_t cost(const StateC* s, const void* _gold, attr_t label) nogil:
         gold = <GoldNERStateC*>_gold
-        cdef int g_act = gold.ner[s.B(0)].move
-        cdef attr_t g_tag = gold.ner[s.B(0)].label
+        b0 = s.B(0)
+        cdef int cost = 0
+        cdef int g_act = gold.ner[b0].move
+        cdef attr_t g_tag = gold.ner[b0].label
 
         if g_act == MISSING:
-            return 0
+            pass
         elif g_act == BEGIN:
             # B, Gold B --> Label match
-            return label != g_tag
-        # Support partial supervision in the form of "not this label"
-        elif g_act == ISNT:
-            return label == g_tag
+            cost += label != g_tag
         else:
             # B, Gold I --> False (P)
             # B, Gold L --> False (P)
             # B, Gold O --> False (P)
             # B, Gold U --> False (P)
-            return 1
+            cost += 1
+        if s.buffer_length() < 3:
+            # Handle negatives. In general we can't really do much to block
+            # B, because we don't know whether the whole entity is going to
+            # be correct or not. However, we can at least tell whether we're
+            # going to be opening an entity where there's only one possible
+            # L.
+            for span in gold.negs[:gold.nr_neg]:
+                if span.label == label and span.start == b0:
+                    cost += 1
+                    break
+        return cost
 
 
 cdef class In:
@@ -462,9 +504,6 @@ cdef class In:
         elif g_act == UNIT:
             # I, Gold U --> True iff next tag == O
             return next_act != OUT
-        # Support partial supervision in the form of "not this label"
-        elif g_act == ISNT:
-            return 0
         else:
             return 1
 
@@ -504,32 +543,41 @@ cdef class Last:
     cdef weight_t cost(const StateC* s, const void* _gold, attr_t label) nogil:
         gold = <GoldNERStateC*>_gold
         move = LAST
+        b0 = s.B(0)
+        ent_start = s.E(0)
 
-        cdef int g_act = gold.ner[s.B(0)].move
-        cdef attr_t g_tag = gold.ner[s.B(0)].label
+        cdef int g_act = gold.ner[b0].move
+        cdef attr_t g_tag = gold.ner[b0].label
+
+        cdef int cost = 0
 
         if g_act == MISSING:
-            return 0
+            pass
         elif g_act == BEGIN:
             # L, Gold B --> True
-            return 0
+            pass
         elif g_act == IN:
             # L, Gold I --> True iff this entity sunk
-            return not _entity_is_sunk(s, gold.ner)
+            cost += not _entity_is_sunk(s, gold.ner)
         elif g_act == LAST:
             # L, Gold L --> True
-            return 0
+            pass
         elif g_act == OUT:
             # L, Gold O --> True
-            return 0
+            pass
         elif g_act == UNIT:
             # L, Gold U --> True
-            return 0
-        # Support partial supervision in the form of "not this label"
-        elif g_act == ISNT:
-            return 0
+            pass
         else:
-            return 1
+            cost += 1
+        # If we have negative-example entities, integrate them into the objective,
+        # by marking actions that close an entity that we know is incorrect
+        # as costly.
+        for span in gold.negs[:gold.nr_neg]:
+            if span.label == label and (span.end-1) == b0 and span.start == ent_start:
+                cost += 1
+                break
+        return cost
 
 
 cdef class Unit:
@@ -573,21 +621,29 @@ cdef class Unit:
         gold = <GoldNERStateC*>_gold
         cdef int g_act = gold.ner[s.B(0)].move
         cdef attr_t g_tag = gold.ner[s.B(0)].label
+        cdef int cost = 0
 
         if g_act == MISSING:
-            return 0
+            pass
         elif g_act == UNIT:
             # U, Gold U --> True iff tag match
-            return label != g_tag
-        # Support partial supervision in the form of "not this label"
-        elif g_act == ISNT:
-            return label == g_tag
+            cost += label != g_tag
         else:
             # U, Gold B --> False
             # U, Gold I --> False
             # U, Gold L --> False
             # U, Gold O --> False
-            return 1
+            cost += 1
+        # If we have negative-example entities, integrate them into the objective.
+        # This is fairly straight-forward for U- entities, as we have a single
+        # action
+        cdef int b0 = s.B(0)
+        for span in gold.negs[:gold.nr_neg]:
+            if span.label == label and span.start == b0 and span.end == (b0+1):
+                cost += 1
+                break
+        return cost
+ 
 
 
 cdef class Out:
@@ -613,25 +669,24 @@ cdef class Out:
         gold = <GoldNERStateC*>_gold
         cdef int g_act = gold.ner[s.B(0)].move
         cdef attr_t g_tag = gold.ner[s.B(0)].label
-
-        if g_act == ISNT and g_tag == 0:
-            return 1
-        elif g_act == MISSING or g_act == ISNT:
-            return 0
+        cdef weight_t cost = 0
+        if g_act == MISSING:
+            pass
         elif g_act == BEGIN:
             # O, Gold B --> False
-            return 1
+            cost += 1
         elif g_act == IN:
             # O, Gold I --> True
-            return 0
+            pass
         elif g_act == LAST:
             # O, Gold L --> True
-            return 0
+            pass
         elif g_act == OUT:
             # O, Gold O --> True
-            return 0
+            pass
         elif g_act == UNIT:
             # O, Gold U --> False
-            return 1
+            cost += 1
         else:
-            return 1
+            cost += 1
+        return cost

spacy/pipeline/_parser_internals/transition_system.pxd

@@ -41,6 +41,7 @@ cdef class TransitionSystem:
     cdef public attr_t root_label
     cdef public freqs
     cdef public object labels
+    cdef public object cfg
     cdef init_state_t init_beam_state
     cdef del_state_t del_beam_state
 

spacy/pipeline/_parser_internals/transition_system.pyx

@@ -33,7 +33,14 @@ cdef int _del_state(Pool mem, void* state, void* x) except -1:
 
 
 cdef class TransitionSystem:
-    def __init__(self, StringStore string_table, labels_by_action=None, min_freq=None):
+    def __init__(
+        self,
+        StringStore string_table,
+        labels_by_action=None,
+        min_freq=None,
+        incorrect_spans_key=None
+    ):
+        self.cfg = {"neg_key": incorrect_spans_key}
         self.mem = Pool()
         self.strings = string_table
         self.n_moves = 0
@@ -49,8 +56,13 @@ cdef class TransitionSystem:
         self.del_beam_state = _del_state
 
     def __reduce__(self):
+        # TODO: This loses the 'cfg'
         return (self.__class__, (self.strings, self.labels), None, None)
 
+    @property
+    def neg_key(self):
+        return self.cfg.get("neg_key")
+
     def init_batch(self, docs):
         cdef StateClass state
         states = []
@@ -220,16 +232,21 @@ cdef class TransitionSystem:
         transitions = []
         serializers = {
             'moves': lambda: srsly.json_dumps(self.labels),
-            'strings': lambda: self.strings.to_bytes()
+            'strings': lambda: self.strings.to_bytes(),
+            'cfg': lambda: self.cfg
         }
         return util.to_bytes(serializers, exclude)
 
     def from_bytes(self, bytes_data, exclude=tuple()):
+        # We're adding a new field, 'cfg', here and we don't want to break
+        # previous models that don't have it.
+        msg = srsly.msgpack_loads(bytes_data)
         labels = {}
-        deserializers = {
-            'moves': lambda b: labels.update(srsly.json_loads(b)),
-            'strings': lambda b: self.strings.from_bytes(b)
-        }
-        msg = util.from_bytes(bytes_data, deserializers, exclude)
+        if 'moves' not in exclude:
+            labels.update(srsly.json_loads(msg['moves']))
+        if 'strings' not in exclude:
+            self.strings.from_bytes(msg['strings'])
+        if 'cfg' not in exclude and 'cfg' in msg:
+            self.cfg.update(msg['cfg'])
         self.initialize_actions(labels)
         return self

spacy/pipeline/attributeruler.py

@@ -106,7 +106,7 @@ class AttributeRuler(Pipe):
 
     def match(self, doc: Doc):
         matches = self.matcher(doc, allow_missing=True)
-        # Sort by the attribute ID, so that later rules have precendence
+        # Sort by the attribute ID, so that later rules have precedence
         matches = [
             (int(self.vocab.strings[m_id]), m_id, s, e) for m_id, s, e in matches
         ]

spacy/pipeline/dep_parser.pyx

@@ -3,6 +3,7 @@ from collections import defaultdict
 from typing import Optional, Iterable
 from thinc.api import Model, Config
 
+from ._parser_internals.transition_system import TransitionSystem
 from .transition_parser cimport Parser
 from ._parser_internals.arc_eager cimport ArcEager
 
@@ -59,7 +60,7 @@ def make_parser(
     nlp: Language,
     name: str,
     model: Model,
-    moves: Optional[list],
+    moves: Optional[TransitionSystem],
     update_with_oracle_cut_size: int,
     learn_tokens: bool,
     min_action_freq: int
@@ -85,13 +86,13 @@ def make_parser(
     model (Model): The model for the transition-based parser. The model needs
         to have a specific substructure of named components --- see the
         spacy.ml.tb_framework.TransitionModel for details.
-    moves (List[str]): A list of transition names. Inferred from the data if not
-        provided.
-    update_with_oracle_cut_size (int):
-        During training, cut long sequences into shorter segments by creating
-        intermediate states based on the gold-standard history. The model is
-        not very sensitive to this parameter, so you usually won't need to change
-        it. 100 is a good default.
+    moves (Optional[TransitionSystem]): This defines how the parse-state is created,
+        updated and evaluated. If 'moves' is None, a new instance is
+        created with `self.TransitionSystem()`. Defaults to `None`.
+    update_with_oracle_cut_size (int): During training, cut long sequences into
+        shorter segments by creating intermediate states based on the gold-standard
+        history. The model is not very sensitive to this parameter, so you usually
+        won't need to change it. 100 is a good default.
     learn_tokens (bool): Whether to learn to merge subtokens that are split
         relative to the gold standard. Experimental.
     min_action_freq (int): The minimum frequency of labelled actions to retain.
@@ -112,6 +113,9 @@ def make_parser(
         beam_width=1,
         beam_density=0.0,
         beam_update_prob=0.0,
+        # At some point in the future we can try to implement support for
+        # partial annotations, perhaps only in the beam objective.
+        incorrect_spans_key=None
     )
 
 @Language.factory(
@@ -140,7 +144,7 @@ def make_beam_parser(
     nlp: Language,
     name: str,
     model: Model,
-    moves: Optional[list],
+    moves: Optional[TransitionSystem],
     update_with_oracle_cut_size: int,
     learn_tokens: bool,
     min_action_freq: int,
@@ -165,8 +169,13 @@ def make_beam_parser(
     model (Model): The model for the transition-based parser. The model needs
         to have a specific substructure of named components --- see the
         spacy.ml.tb_framework.TransitionModel for details.
-    moves (List[str]): A list of transition names. Inferred from the data if not
-        provided.
+    moves (Optional[TransitionSystem]): This defines how the parse-state is created,
+        updated and evaluated. If 'moves' is None, a new instance is
+        created with `self.TransitionSystem()`. Defaults to `None`.
+    update_with_oracle_cut_size (int): During training, cut long sequences into
+        shorter segments by creating intermediate states based on the gold-standard
+        history. The model is not very sensitive to this parameter, so you usually
+        won't need to change it. 100 is a good default.
     beam_width (int): The number of candidate analyses to maintain.
     beam_density (float): The minimum ratio between the scores of the first and
         last candidates in the beam. This allows the parser to avoid exploring
@@ -195,7 +204,10 @@ def make_beam_parser(
         beam_update_prob=beam_update_prob,
         multitasks=[],
         learn_tokens=learn_tokens,
-        min_action_freq=min_action_freq
+        min_action_freq=min_action_freq,
+        # At some point in the future we can try to implement support for
+        # partial annotations, perhaps only in the beam objective.
+        incorrect_spans_key=None
     )
 
 
@@ -206,6 +218,39 @@ cdef class DependencyParser(Parser):
     """
     TransitionSystem = ArcEager
 
+    def __init__(
+        self,
+        vocab,
+        model,
+        name="parser",
+        moves=None,
+        *,
+        update_with_oracle_cut_size=100,
+        min_action_freq=30,
+        learn_tokens=False,
+        beam_width=1,
+        beam_density=0.0,
+        beam_update_prob=0.0,
+        multitasks=tuple(),
+        incorrect_spans_key=None,
+    ):
+        """Create a DependencyParser.
+        """
+        super().__init__(
+            vocab,
+            model,
+            name,
+            moves,
+            update_with_oracle_cut_size=update_with_oracle_cut_size,
+            min_action_freq=min_action_freq,
+            learn_tokens=learn_tokens,
+            beam_width=beam_width,
+            beam_density=beam_density,
+            beam_update_prob=beam_update_prob,
+            multitasks=multitasks,
+            incorrect_spans_key=incorrect_spans_key,
+        )
+
     @property
     def postprocesses(self):
         output = [nonproj.deprojectivize]

spacy/pipeline/entity_linker.py

@@ -9,7 +9,7 @@ import warnings
 
 from ..kb import KnowledgeBase, Candidate
 from ..ml import empty_kb
-from ..tokens import Doc
+from ..tokens import Doc, Span
 from .pipe import deserialize_config
 from .trainable_pipe import TrainablePipe
 from ..language import Language
@@ -67,7 +67,7 @@ def make_entity_linker(
     incl_prior: bool,
     incl_context: bool,
     entity_vector_length: int,
-    get_candidates: Callable[[KnowledgeBase, "Span"], Iterable[Candidate]],
+    get_candidates: Callable[[KnowledgeBase, Span], Iterable[Candidate]],
 ):
     """Construct an EntityLinker component.
 
@@ -114,7 +114,7 @@ class EntityLinker(TrainablePipe):
         incl_prior: bool,
         incl_context: bool,
         entity_vector_length: int,
-        get_candidates: Callable[[KnowledgeBase, "Span"], Iterable[Candidate]],
+        get_candidates: Callable[[KnowledgeBase, Span], Iterable[Candidate]],
     ) -> None:
         """Initialize an entity linker.
 
@@ -127,7 +127,7 @@ class EntityLinker(TrainablePipe):
         incl_prior (bool): Whether or not to include prior probabilities from the KB in the model.
         incl_context (bool): Whether or not to include the local context in the model.
         entity_vector_length (int): Size of encoding vectors in the KB.
-        get_candidates (Callable[[KnowledgeBase, "Span"], Iterable[Candidate]]): Function that
+        get_candidates (Callable[[KnowledgeBase, Span], Iterable[Candidate]]): Function that
             produces a list of candidates, given a certain knowledge base and a textual mention.
 
         DOCS: https://spacy.io/api/entitylinker#init
@@ -142,7 +142,7 @@ class EntityLinker(TrainablePipe):
         self.get_candidates = get_candidates
         self.cfg = {}
         self.distance = CosineDistance(normalize=False)
-        # how many neightbour sentences to take into account
+        # how many neighbour sentences to take into account
         # create an empty KB by default. If you want to load a predefined one, specify it in 'initialize'.
         self.kb = empty_kb(entity_vector_length)(self.vocab)
 
@@ -156,6 +156,8 @@ class EntityLinker(TrainablePipe):
 
     def validate_kb(self) -> None:
         # Raise an error if the knowledge base is not initialized.
+        if self.kb is None:
+            raise ValueError(Errors.E1018.format(name=self.name))
         if len(self.kb) == 0:
             raise ValueError(Errors.E139.format(name=self.name))
 
@@ -305,11 +307,9 @@ class EntityLinker(TrainablePipe):
                     sent = ent.sent
                     sent_index = sentences.index(sent)
                     assert sent_index >= 0
-                    # get n_neightbour sentences, clipped to the length of the document
+                    # get n_neighbour sentences, clipped to the length of the document
                     start_sentence = max(0, sent_index - self.n_sents)
-                    end_sentence = min(
-                        len(sentences) - 1, sent_index + self.n_sents
-                    )
+                    end_sentence = min(len(sentences) - 1, sent_index + self.n_sents)
                     start_token = sentences[start_sentence].start
                     end_token = sentences[end_sentence].end
                     sent_doc = doc[start_token:end_token].as_doc()
@@ -335,22 +335,16 @@ class EntityLinker(TrainablePipe):
                         else:
                             random.shuffle(candidates)
                             # set all prior probabilities to 0 if incl_prior=False
-                            prior_probs = xp.asarray(
-                                [c.prior_prob for c in candidates]
-                            )
+                            prior_probs = xp.asarray([c.prior_prob for c in candidates])
                             if not self.incl_prior:
-                                prior_probs = xp.asarray(
-                                    [0.0 for _ in candidates]
-                                )
+                                prior_probs = xp.asarray([0.0 for _ in candidates])
                             scores = prior_probs
                             # add in similarity from the context
                             if self.incl_context:
                                 entity_encodings = xp.asarray(
                                     [c.entity_vector for c in candidates]
                                 )
-                                entity_norm = xp.linalg.norm(
-                                    entity_encodings, axis=1
-                                )
+                                entity_norm = xp.linalg.norm(entity_encodings, axis=1)
                                 if len(entity_encodings) != len(prior_probs):
                                     raise RuntimeError(
                                         Errors.E147.format(
@@ -359,14 +353,12 @@ class EntityLinker(TrainablePipe):
                                         )
                                     )
                                 # cosine similarity
-                                sims = xp.dot(
-                                    entity_encodings, sentence_encoding_t
-                                ) / (sentence_norm * entity_norm)
+                                sims = xp.dot(entity_encodings, sentence_encoding_t) / (
+                                    sentence_norm * entity_norm
+                                )
                                 if sims.shape != prior_probs.shape:
                                     raise ValueError(Errors.E161)
-                                scores = (
-                                    prior_probs + sims - (prior_probs * sims)
-                                )
+                                scores = prior_probs + sims - (prior_probs * sims)
                             # TODO: thresholding
                             best_index = scores.argmax().item()
                             best_candidate = candidates[best_index]
@@ -481,7 +473,8 @@ class EntityLinker(TrainablePipe):
 
         def load_model(p):
             try:
-                self.model.from_bytes(p.open("rb").read())
+                with p.open("rb") as infile:
+                    self.model.from_bytes(infile.read())
             except AttributeError:
                 raise ValueError(Errors.E149) from None
 

spacy/pipeline/entityruler.py

@@ -1,3 +1,4 @@
+import warnings
 from typing import Optional, Union, List, Dict, Tuple, Iterable, Any, Callable, Sequence
 from collections import defaultdict
 from pathlib import Path
@@ -6,7 +7,7 @@ import srsly
 from .pipe import Pipe
 from ..training import Example
 from ..language import Language
-from ..errors import Errors
+from ..errors import Errors, Warnings
 from ..util import ensure_path, to_disk, from_disk, SimpleFrozenList
 from ..tokens import Doc, Span
 from ..matcher import Matcher, PhraseMatcher
@@ -100,17 +101,12 @@ class EntityRuler(Pipe):
         self.overwrite = overwrite_ents
         self.token_patterns = defaultdict(list)
         self.phrase_patterns = defaultdict(list)
+        self._validate = validate
         self.matcher = Matcher(nlp.vocab, validate=validate)
-        if phrase_matcher_attr is not None:
-            if phrase_matcher_attr.upper() == "TEXT":
-                phrase_matcher_attr = "ORTH"
-            self.phrase_matcher_attr = phrase_matcher_attr
-            self.phrase_matcher = PhraseMatcher(
-                nlp.vocab, attr=self.phrase_matcher_attr, validate=validate
-            )
-        else:
-            self.phrase_matcher_attr = None
-            self.phrase_matcher = PhraseMatcher(nlp.vocab, validate=validate)
+        self.phrase_matcher_attr = phrase_matcher_attr
+        self.phrase_matcher = PhraseMatcher(
+            nlp.vocab, attr=self.phrase_matcher_attr, validate=validate
+        )
         self.ent_id_sep = ent_id_sep
         self._ent_ids = defaultdict(dict)
         if patterns is not None:
@@ -143,7 +139,10 @@ class EntityRuler(Pipe):
             error_handler(self.name, self, [doc], e)
 
     def match(self, doc: Doc):
-        matches = list(self.matcher(doc)) + list(self.phrase_matcher(doc))
+        self._require_patterns()
+        with warnings.catch_warnings():
+            warnings.filterwarnings("ignore", message="\\[W036")
+            matches = list(self.matcher(doc)) + list(self.phrase_matcher(doc))
         matches = set(
             [(m_id, start, end) for m_id, start, end in matches if start != end]
         )
@@ -277,9 +276,7 @@ class EntityRuler(Pipe):
                 if self == pipe:
                     current_index = i
                     break
-            subsequent_pipes = [
-                pipe for pipe in self.nlp.pipe_names[current_index + 1 :]
-            ]
+            subsequent_pipes = [pipe for pipe in self.nlp.pipe_names[current_index:]]
         except ValueError:
             subsequent_pipes = []
         with self.nlp.select_pipes(disable=subsequent_pipes):
@@ -300,7 +297,7 @@ class EntityRuler(Pipe):
                 self.nlp.pipe(phrase_pattern_texts),
                 phrase_pattern_ids,
             ):
-                phrase_pattern = {"label": label, "pattern": pattern, "id": ent_id}
+                phrase_pattern = {"label": label, "pattern": pattern}
                 if ent_id:
                     phrase_pattern["id"] = ent_id
                 phrase_patterns.append(phrase_pattern)
@@ -314,20 +311,27 @@ class EntityRuler(Pipe):
                 pattern = entry["pattern"]
                 if isinstance(pattern, Doc):
                     self.phrase_patterns[label].append(pattern)
+                    self.phrase_matcher.add(label, [pattern])
                 elif isinstance(pattern, list):
                     self.token_patterns[label].append(pattern)
+                    self.matcher.add(label, [pattern])
                 else:
                     raise ValueError(Errors.E097.format(pattern=pattern))
-            for label, patterns in self.token_patterns.items():
-                self.matcher.add(label, patterns)
-            for label, patterns in self.phrase_patterns.items():
-                self.phrase_matcher.add(label, patterns)
 
     def clear(self) -> None:
         """Reset all patterns."""
         self.token_patterns = defaultdict(list)
         self.phrase_patterns = defaultdict(list)
         self._ent_ids = defaultdict(dict)
+        self.matcher = Matcher(self.nlp.vocab, validate=self._validate)
+        self.phrase_matcher = PhraseMatcher(
+            self.nlp.vocab, attr=self.phrase_matcher_attr, validate=self._validate
+        )
+
+    def _require_patterns(self) -> None:
+        """Raise a warning if this component has no patterns defined."""
+        if len(self) == 0:
+            warnings.warn(Warnings.W036.format(name=self.name))
 
     def _split_label(self, label: str) -> Tuple[str, str]:
         """Split Entity label into ent_label and ent_id if it contains self.ent_id_sep
@@ -373,10 +377,9 @@ class EntityRuler(Pipe):
             self.add_patterns(cfg.get("patterns", cfg))
             self.overwrite = cfg.get("overwrite", False)
             self.phrase_matcher_attr = cfg.get("phrase_matcher_attr", None)
-            if self.phrase_matcher_attr is not None:
-                self.phrase_matcher = PhraseMatcher(
-                    self.nlp.vocab, attr=self.phrase_matcher_attr
-                )
+            self.phrase_matcher = PhraseMatcher(
+                self.nlp.vocab, attr=self.phrase_matcher_attr
+            )
             self.ent_id_sep = cfg.get("ent_id_sep", DEFAULT_ENT_ID_SEP)
         else:
             self.add_patterns(cfg)
@@ -427,10 +430,9 @@ class EntityRuler(Pipe):
             self.phrase_matcher_attr = cfg.get("phrase_matcher_attr")
             self.ent_id_sep = cfg.get("ent_id_sep", DEFAULT_ENT_ID_SEP)
 
-            if self.phrase_matcher_attr is not None:
-                self.phrase_matcher = PhraseMatcher(
-                    self.nlp.vocab, attr=self.phrase_matcher_attr
-                )
+            self.phrase_matcher = PhraseMatcher(
+                self.nlp.vocab, attr=self.phrase_matcher_attr
+            )
             from_disk(path, deserializers_patterns, {})
         return self
 

spacy/pipeline/lemmatizer.py

@@ -2,6 +2,8 @@ from typing import Optional, List, Dict, Any, Callable, Iterable, Union, Tuple
 from thinc.api import Model
 from pathlib import Path
 
+import warnings
+
 from .pipe import Pipe
 from ..errors import Errors, Warnings
 from ..language import Language
@@ -182,7 +184,7 @@ class Lemmatizer(Pipe):
         univ_pos = token.pos_.lower()
         if univ_pos in ("", "eol", "space"):
             if univ_pos == "":
-                logger.warning(Warnings.W108.format(text=string))
+                warnings.warn(Warnings.W108.format(text=string))
             return [string.lower()]
         # See Issue #435 for example of where this logic is requied.
         if self.is_base_form(token):

spacy/pipeline/ner.pyx

@@ -3,6 +3,7 @@ from collections import defaultdict
 from typing import Optional, Iterable
 from thinc.api import Model, Config
 
+from ._parser_internals.transition_system import TransitionSystem
 from .transition_parser cimport Parser
 from ._parser_internals.ner cimport BiluoPushDown
 
@@ -40,6 +41,7 @@ DEFAULT_NER_MODEL = Config().from_str(default_model_config)["model"]
         "moves": None,
         "update_with_oracle_cut_size": 100,
         "model": DEFAULT_NER_MODEL,
+        "incorrect_spans_key": None
     },
     default_score_weights={"ents_f": 1.0, "ents_p": 0.0, "ents_r": 0.0, "ents_per_type": None},
 
@@ -48,8 +50,9 @@ def make_ner(
     nlp: Language,
     name: str,
     model: Model,
-    moves: Optional[list],
+    moves: Optional[TransitionSystem],
     update_with_oracle_cut_size: int,
+    incorrect_spans_key: Optional[str]=None
 ):
     """Create a transition-based EntityRecognizer component. The entity recognizer
     identifies non-overlapping labelled spans of tokens.
@@ -67,13 +70,16 @@ def make_ner(
     model (Model): The model for the transition-based parser. The model needs
         to have a specific substructure of named components --- see the
         spacy.ml.tb_framework.TransitionModel for details.
-    moves (list[str]): A list of transition names. Inferred from the data if not
-        provided.
-    update_with_oracle_cut_size (int):
-        During training, cut long sequences into shorter segments by creating
-        intermediate states based on the gold-standard history. The model is
-        not very sensitive to this parameter, so you usually won't need to change
-        it. 100 is a good default.
+    moves (Optional[TransitionSystem]): This defines how the parse-state is created,
+        updated and evaluated. If 'moves' is None, a new instance is
+        created with `self.TransitionSystem()`. Defaults to `None`.
+    update_with_oracle_cut_size (int): During training, cut long sequences into
+        shorter segments by creating intermediate states based on the gold-standard
+        history. The model is not very sensitive to this parameter, so you usually
+        won't need to change it. 100 is a good default.
+    incorrect_spans_key (Optional[str]): Identifies spans that are known
+        to be incorrect entity annotations. The incorrect entity annotations
+        can be stored in the span group, under this key.
     """
     return EntityRecognizer(
         nlp.vocab,
@@ -81,9 +87,8 @@ def make_ner(
         name,
         moves=moves,
         update_with_oracle_cut_size=update_with_oracle_cut_size,
+        incorrect_spans_key=incorrect_spans_key,
         multitasks=[],
-        min_action_freq=1,
-        learn_tokens=False,
         beam_width=1,
         beam_density=0.0,
         beam_update_prob=0.0,
@@ -98,7 +103,8 @@ def make_ner(
         "model": DEFAULT_NER_MODEL,
         "beam_density": 0.01,
         "beam_update_prob": 0.5,
-        "beam_width": 32
+        "beam_width": 32,
+        "incorrect_spans_key": None
     },
     default_score_weights={"ents_f": 1.0, "ents_p": 0.0, "ents_r": 0.0, "ents_per_type": None},
 )
@@ -106,11 +112,12 @@ def make_beam_ner(
     nlp: Language,
     name: str,
     model: Model,
-    moves: Optional[list],
+    moves: Optional[TransitionSystem],
     update_with_oracle_cut_size: int,
     beam_width: int,
     beam_density: float,
     beam_update_prob: float,
+    incorrect_spans_key: Optional[str]=None
 ):
     """Create a transition-based EntityRecognizer component that uses beam-search.
     The entity recognizer identifies non-overlapping labelled spans of tokens.
@@ -128,13 +135,13 @@ def make_beam_ner(
     model (Model): The model for the transition-based parser. The model needs
         to have a specific substructure of named components --- see the
         spacy.ml.tb_framework.TransitionModel for details.
-    moves (list[str]): A list of transition names. Inferred from the data if not
-        provided.
-    update_with_oracle_cut_size (int):
-        During training, cut long sequences into shorter segments by creating
-        intermediate states based on the gold-standard history. The model is
-        not very sensitive to this parameter, so you usually won't need to change
-        it. 100 is a good default.
+    moves (Optional[TransitionSystem]): This defines how the parse-state is created,
+        updated and evaluated. If 'moves' is None, a new instance is
+        created with `self.TransitionSystem()`. Defaults to `None`.
+    update_with_oracle_cut_size (int): During training, cut long sequences into
+        shorter segments by creating intermediate states based on the gold-standard
+        history. The model is not very sensitive to this parameter, so you usually
+        won't need to change it. 100 is a good default.
     beam_width (int): The number of candidate analyses to maintain.
     beam_density (float): The minimum ratio between the scores of the first and
         last candidates in the beam. This allows the parser to avoid exploring
@@ -144,6 +151,8 @@ def make_beam_ner(
     beam_update_prob (float): The chance of making a beam update, instead of a
         greedy update. Greedy updates are an approximation for the beam updates,
         and are faster to compute.
+    incorrect_spans_key (Optional[str]): Optional key into span groups of
+        entities known to be non-entities.
     """
     return EntityRecognizer(
         nlp.vocab,
@@ -152,11 +161,10 @@ def make_beam_ner(
         moves=moves,
         update_with_oracle_cut_size=update_with_oracle_cut_size,
         multitasks=[],
-        min_action_freq=1,
-        learn_tokens=False,
         beam_width=beam_width,
         beam_density=beam_density,
         beam_update_prob=beam_update_prob,
+        incorrect_spans_key=incorrect_spans_key
     )
 
 
@@ -167,6 +175,37 @@ cdef class EntityRecognizer(Parser):
     """
     TransitionSystem = BiluoPushDown
 
+    def __init__(
+        self,
+        vocab,
+        model,
+        name="ner",
+        moves=None,
+        *,
+        update_with_oracle_cut_size=100,
+        beam_width=1,
+        beam_density=0.0,
+        beam_update_prob=0.0,
+        multitasks=tuple(),
+        incorrect_spans_key=None,
+    ):
+        """Create an EntityRecognizer.
+        """
+        super().__init__(
+            vocab,
+            model,
+            name,
+            moves,
+            update_with_oracle_cut_size=update_with_oracle_cut_size,
+            min_action_freq=1,   # not relevant for NER
+            learn_tokens=False,  # not relevant for NER
+            beam_width=beam_width,
+            beam_density=beam_density,
+            beam_update_prob=beam_update_prob,
+            multitasks=multitasks,
+            incorrect_spans_key=incorrect_spans_key,
+        )
+
     def add_multitask_objective(self, mt_component):
         """Register another component as a multi-task objective. Experimental."""
         self._multitasks.append(mt_component)

spacy/pipeline/spancat.py

@@ -0,0 +1,412 @@
+import numpy
+from typing import List, Dict, Callable, Tuple, Optional, Iterable, Any
+from thinc.api import Config, Model, get_current_ops, set_dropout_rate, Ops
+from thinc.api import Optimizer
+from thinc.types import Ragged, Ints2d, Floats2d
+
+from ..scorer import Scorer
+from ..language import Language
+from .trainable_pipe import TrainablePipe
+from ..tokens import Doc, SpanGroup, Span
+from ..vocab import Vocab
+from ..training import Example, validate_examples
+from ..errors import Errors
+from ..util import registry
+
+
+spancat_default_config = """
+[model]
+@architectures = "spacy.SpanCategorizer.v1"
+scorer = {"@layers": "spacy.LinearLogistic.v1"}
+
+[model.reducer]
+@layers = spacy.mean_max_reducer.v1
+hidden_size = 128
+
+[model.tok2vec]
+@architectures = "spacy.Tok2Vec.v1"
+
+[model.tok2vec.embed]
+@architectures = "spacy.MultiHashEmbed.v1"
+width = 96
+rows = [5000, 2000, 1000, 1000]
+attrs = ["ORTH", "PREFIX", "SUFFIX", "SHAPE"]
+include_static_vectors = false
+
+[model.tok2vec.encode]
+@architectures = "spacy.MaxoutWindowEncoder.v1"
+width = ${model.tok2vec.embed.width}
+window_size = 1
+maxout_pieces = 3
+depth = 4
+"""
+
+DEFAULT_SPANCAT_MODEL = Config().from_str(spancat_default_config)["model"]
+
+
+@registry.misc("spacy.ngram_suggester.v1")
+def build_ngram_suggester(sizes: List[int]) -> Callable[[List[Doc]], Ragged]:
+    """Suggest all spans of the given lengths. Spans are returned as a ragged
+    array of integers. The array has two columns, indicating the start and end
+    position."""
+
+    def ngram_suggester(docs: List[Doc], *, ops: Optional[Ops] = None) -> Ragged:
+        if ops is None:
+            ops = get_current_ops()
+        spans = []
+        lengths = []
+        for doc in docs:
+            starts = ops.xp.arange(len(doc), dtype="i")
+            starts = starts.reshape((-1, 1))
+            length = 0
+            for size in sizes:
+                if size <= len(doc):
+                    starts_size = starts[: len(doc) - (size - 1)]
+                    spans.append(ops.xp.hstack((starts_size, starts_size + size)))
+                    length += spans[-1].shape[0]
+                if spans:
+                    assert spans[-1].ndim == 2, spans[-1].shape
+            lengths.append(length)
+        if len(spans) > 0:
+            output = Ragged(ops.xp.vstack(spans), ops.asarray(lengths, dtype="i"))
+        else:
+            output = Ragged(ops.xp.zeros((0, 0)), ops.asarray(lengths, dtype="i"))
+
+        assert output.dataXd.ndim == 2
+        return output
+
+    return ngram_suggester
+
+
+@Language.factory(
+    "spancat",
+    assigns=["doc.spans"],
+    default_config={
+        "threshold": 0.5,
+        "spans_key": "sc",
+        "max_positive": None,
+        "model": DEFAULT_SPANCAT_MODEL,
+        "suggester": {"@misc": "spacy.ngram_suggester.v1", "sizes": [1, 2, 3]},
+    },
+    default_score_weights={"spans_sc_f": 1.0, "spans_sc_p": 0.0, "spans_sc_r": 0.0},
+)
+def make_spancat(
+    nlp: Language,
+    name: str,
+    suggester: Callable[[List[Doc]], Ragged],
+    model: Model[Tuple[List[Doc], Ragged], Floats2d],
+    spans_key: str,
+    threshold: float = 0.5,
+    max_positive: Optional[int] = None,
+) -> "SpanCategorizer":
+    """Create a SpanCategorizer component. The span categorizer consists of two
+    parts: a suggester function that proposes candidate spans, and a labeller
+    model that predicts one or more labels for each span.
+
+    suggester (Callable[List[Doc], Ragged]): A function that suggests spans.
+        Spans are returned as a ragged array with two integer columns, for the
+        start and end positions.
+    model (Model[Tuple[List[Doc], Ragged], Floats2d]): A model instance that
+        is given a list of documents and (start, end) indices representing
+        candidate span offsets. The model predicts a probability for each category
+        for each span.
+    spans_key (str): Key of the doc.spans dict to save the spans under. During
+        initialization and training, the component will look for spans on the
+        reference document under the same key.
+    threshold (float): Minimum probability to consider a prediction positive.
+        Spans with a positive prediction will be saved on the Doc. Defaults to
+        0.5.
+    max_positive (Optional[int]): Maximum number of labels to consider positive
+        per span. Defaults to None, indicating no limit.
+    """
+    return SpanCategorizer(
+        nlp.vocab,
+        suggester=suggester,
+        model=model,
+        spans_key=spans_key,
+        threshold=threshold,
+        max_positive=max_positive,
+        name=name,
+    )
+
+
+class SpanCategorizer(TrainablePipe):
+    """Pipeline component to label spans of text.
+
+    DOCS: https://spacy.io/api/spancategorizer
+    """
+
+    def __init__(
+        self,
+        vocab: Vocab,
+        model: Model[Tuple[List[Doc], Ragged], Floats2d],
+        suggester: Callable[[List[Doc]], Ragged],
+        name: str = "spancat",
+        *,
+        spans_key: str = "spans",
+        threshold: float = 0.5,
+        max_positive: Optional[int] = None,
+    ) -> None:
+        """Initialize the span categorizer.
+
+        DOCS: https://spacy.io/api/spancategorizer#init
+        """
+        self.cfg = {
+            "labels": [],
+            "spans_key": spans_key,
+            "threshold": threshold,
+            "max_positive": max_positive,
+        }
+        self.vocab = vocab
+        self.suggester = suggester
+        self.model = model
+        self.name = name
+
+    @property
+    def key(self) -> str:
+        """Key of the doc.spans dict to save the spans under. During
+        initialization and training, the component will look for spans on the
+        reference document under the same key.
+        """
+        return self.cfg["spans_key"]
+
+    def add_label(self, label: str) -> int:
+        """Add a new label to the pipe.
+
+        label (str): The label to add.
+        RETURNS (int): 0 if label is already present, otherwise 1.
+
+        DOCS: https://spacy.io/api/spancategorizer#add_label
+        """
+        if not isinstance(label, str):
+            raise ValueError(Errors.E187)
+        if label in self.labels:
+            return 0
+        self._allow_extra_label()
+        self.cfg["labels"].append(label)
+        self.vocab.strings.add(label)
+        return 1
+
+    @property
+    def labels(self) -> Tuple[str]:
+        """RETURNS (Tuple[str]): The labels currently added to the component.
+
+        DOCS: https://spacy.io/api/spancategorizer#labels
+        """
+        return tuple(self.cfg["labels"])
+
+    @property
+    def label_data(self) -> List[str]:
+        """RETURNS (List[str]): Information about the component's labels.
+
+        DOCS: https://spacy.io/api/spancategorizer#label_data
+        """
+        return list(self.labels)
+
+    def predict(self, docs: Iterable[Doc]):
+        """Apply the pipeline's model to a batch of docs, without modifying them.
+
+        docs (Iterable[Doc]): The documents to predict.
+        RETURNS: The models prediction for each document.
+
+        DOCS: https://spacy.io/api/spancategorizer#predict
+        """
+        indices = self.suggester(docs, ops=self.model.ops)
+        scores = self.model.predict((docs, indices))
+        return (indices, scores)
+
+    def set_annotations(self, docs: Iterable[Doc], indices_scores) -> None:
+        """Modify a batch of Doc objects, using pre-computed scores.
+
+        docs (Iterable[Doc]): The documents to modify.
+        scores: The scores to set, produced by SpanCategorizer.predict.
+
+        DOCS: https://spacy.io/api/spancategorizer#set_annotations
+        """
+        labels = self.labels
+        indices, scores = indices_scores
+        offset = 0
+        for i, doc in enumerate(docs):
+            indices_i = indices[i].dataXd
+            doc.spans[self.key] = self._make_span_group(
+                doc, indices_i, scores[offset : offset + indices.lengths[i]], labels
+            )
+            offset += indices.lengths[i]
+
+    def update(
+        self,
+        examples: Iterable[Example],
+        *,
+        drop: float = 0.0,
+        sgd: Optional[Optimizer] = None,
+        losses: Optional[Dict[str, float]] = None,
+    ) -> Dict[str, float]:
+        """Learn from a batch of documents and gold-standard information,
+        updating the pipe's model. Delegates to predict and get_loss.
+
+        examples (Iterable[Example]): A batch of Example objects.
+        drop (float): The dropout rate.
+        sgd (thinc.api.Optimizer): The optimizer.
+        losses (Dict[str, float]): Optional record of the loss during training.
+            Updated using the component name as the key.
+        RETURNS (Dict[str, float]): The updated losses dictionary.
+
+        DOCS: https://spacy.io/api/spancategorizer#update
+        """
+        if losses is None:
+            losses = {}
+        losses.setdefault(self.name, 0.0)
+        validate_examples(examples, "SpanCategorizer.update")
+        self._validate_categories(examples)
+        if not any(len(eg.predicted) if eg.predicted else 0 for eg in examples):
+            # Handle cases where there are no tokens in any docs.
+            return losses
+        docs = [eg.predicted for eg in examples]
+        spans = self.suggester(docs, ops=self.model.ops)
+        if spans.lengths.sum() == 0:
+            return losses
+        set_dropout_rate(self.model, drop)
+        scores, backprop_scores = self.model.begin_update((docs, spans))
+        loss, d_scores = self.get_loss(examples, (spans, scores))
+        backprop_scores(d_scores)
+        if sgd is not None:
+            self.finish_update(sgd)
+        losses[self.name] += loss
+        return losses
+
+    def get_loss(
+        self, examples: Iterable[Example], spans_scores: Tuple[Ragged, Ragged]
+    ) -> Tuple[float, float]:
+        """Find the loss and gradient of loss for the batch of documents and
+        their predicted scores.
+
+        examples (Iterable[Examples]): The batch of examples.
+        spans_scores: Scores representing the model's predictions.
+        RETURNS (Tuple[float, float]): The loss and the gradient.
+
+        DOCS: https://spacy.io/api/spancategorizer#get_loss
+        """
+        spans, scores = spans_scores
+        spans = Ragged(
+            self.model.ops.to_numpy(spans.data), self.model.ops.to_numpy(spans.lengths)
+        )
+        label_map = {label: i for i, label in enumerate(self.labels)}
+        target = numpy.zeros(scores.shape, dtype=scores.dtype)
+        offset = 0
+        for i, eg in enumerate(examples):
+            # Map (start, end) offset of spans to the row in the d_scores array,
+            # so that we can adjust the gradient for predictions that were
+            # in the gold standard.
+            spans_index = {}
+            spans_i = spans[i].dataXd
+            for j in range(spans.lengths[i]):
+                start = int(spans_i[j, 0])
+                end = int(spans_i[j, 1])
+                spans_index[(start, end)] = offset + j
+            for gold_span in self._get_aligned_spans(eg):
+                key = (gold_span.start, gold_span.end)
+                if key in spans_index:
+                    row = spans_index[key]
+                    k = label_map[gold_span.label_]
+                    target[row, k] = 1.0
+            # The target is a flat array for all docs. Track the position
+            # we're at within the flat array.
+            offset += spans.lengths[i]
+        target = self.model.ops.asarray(target, dtype="f")
+        # The target will have the values 0 (for untrue predictions) or 1
+        # (for true predictions).
+        # The scores should be in the range [0, 1].
+        # If the prediction is 0.9 and it's true, the gradient
+        # will be -0.1 (0.9 - 1.0).
+        # If the prediction is 0.9 and it's false, the gradient will be
+        # 0.9 (0.9 - 0.0)
+        d_scores = scores - target
+        loss = float((d_scores ** 2).sum())
+        return loss, d_scores
+
+    def initialize(
+        self,
+        get_examples: Callable[[], Iterable[Example]],
+        *,
+        nlp: Language = None,
+        labels: Optional[List[str]] = None,
+    ) -> None:
+        """Initialize the pipe for training, using a representative set
+        of data examples.
+
+        get_examples (Callable[[], Iterable[Example]]): Function that
+            returns a representative sample of gold-standard Example objects.
+        nlp (Language): The current nlp object the component is part of.
+        labels: The labels to add to the component, typically generated by the
+            `init labels` command. If no labels are provided, the get_examples
+            callback is used to extract the labels from the data.
+
+        DOCS: https://spacy.io/api/spancategorizer#initialize
+        """
+        subbatch = []
+        if labels is not None:
+            for label in labels:
+                self.add_label(label)
+        for eg in get_examples():
+            if labels is None:
+                for span in eg.reference.spans.get(self.key, []):
+                    self.add_label(span.label_)
+            if len(subbatch) < 10:
+                subbatch.append(eg)
+        self._require_labels()
+        if subbatch:
+            docs = [eg.x for eg in subbatch]
+            spans = self.suggester(docs)
+            Y = self.model.ops.alloc2f(spans.dataXd.shape[0], len(self.labels))
+            self.model.initialize(X=(docs, spans), Y=Y)
+        else:
+            self.model.initialize()
+
+    def score(self, examples: Iterable[Example], **kwargs) -> Dict[str, Any]:
+        """Score a batch of examples.
+
+        examples (Iterable[Example]): The examples to score.
+        RETURNS (Dict[str, Any]): The scores, produced by Scorer.score_cats.
+
+        DOCS: https://spacy.io/api/spancategorizer#score
+        """
+        validate_examples(examples, "SpanCategorizer.score")
+        self._validate_categories(examples)
+        kwargs = dict(kwargs)
+        attr_prefix = "spans_"
+        kwargs.setdefault("attr", f"{attr_prefix}{self.key}")
+        kwargs.setdefault("labels", self.labels)
+        kwargs.setdefault("multi_label", True)
+        kwargs.setdefault("threshold", self.cfg["threshold"])
+        kwargs.setdefault(
+            "getter", lambda doc, key: doc.spans.get(key[len(attr_prefix) :], [])
+        )
+        kwargs.setdefault("has_annotation", lambda doc: self.key in doc.spans)
+        return Scorer.score_spans(examples, **kwargs)
+
+    def _validate_categories(self, examples):
+        # TODO
+        pass
+
+    def _get_aligned_spans(self, eg: Example):
+        return eg.get_aligned_spans_y2x(eg.reference.spans.get(self.key, []))
+
+    def _make_span_group(
+        self, doc: Doc, indices: Ints2d, scores: Floats2d, labels: List[str]
+    ) -> SpanGroup:
+        spans = SpanGroup(doc, name=self.key)
+        max_positive = self.cfg["max_positive"]
+        threshold = self.cfg["threshold"]
+        for i in range(indices.shape[0]):
+            start = int(indices[i, 0])
+            end = int(indices[i, 1])
+            positives = []
+            for j, score in enumerate(scores[i]):
+                if score >= threshold:
+                    positives.append((score, start, end, labels[j]))
+            positives.sort(reverse=True)
+            if max_positive:
+                positives = positives[:max_positive]
+            for score, start, end, label in positives:
+                spans.append(Span(doc, start, end, label=label))
+        return spans

spacy/pipeline/textcat.py

@@ -35,7 +35,7 @@ maxout_pieces = 3
 depth = 2
 
 [model.linear_model]
-@architectures = "spacy.TextCatBOW.v1"
+@architectures = "spacy.TextCatBOW.v2"
 exclusive_classes = true
 ngram_size = 1
 no_output_layer = false
@@ -44,7 +44,7 @@ DEFAULT_SINGLE_TEXTCAT_MODEL = Config().from_str(single_label_default_config)["m
 
 single_label_bow_config = """
 [model]
-@architectures = "spacy.TextCatBOW.v1"
+@architectures = "spacy.TextCatBOW.v2"
 exclusive_classes = true
 ngram_size = 1
 no_output_layer = false
@@ -52,7 +52,7 @@ no_output_layer = false
 
 single_label_cnn_config = """
 [model]
-@architectures = "spacy.TextCatCNN.v1"
+@architectures = "spacy.TextCatCNN.v2"
 exclusive_classes = true
 
 [model.tok2vec]
@@ -298,6 +298,10 @@ class TextCategorizer(TrainablePipe):
             return 0
         self._allow_extra_label()
         self.cfg["labels"].append(label)
+        if self.model and "resize_output" in self.model.attrs:
+            self.model = self.model.attrs["resize_output"](
+                self.model, len(self.cfg["labels"])
+            )
         self.vocab.strings.add(label)
         return 1
 
@@ -332,6 +336,8 @@ class TextCategorizer(TrainablePipe):
         else:
             for label in labels:
                 self.add_label(label)
+        if len(self.labels) < 2:
+            raise ValueError(Errors.E867)
         if positive_label is not None:
             if positive_label not in self.labels:
                 err = Errors.E920.format(pos_label=positive_label, labels=self.labels)

spacy/pipeline/textcat_multilabel.py

@@ -35,7 +35,7 @@ maxout_pieces = 3
 depth = 2
 
 [model.linear_model]
-@architectures = "spacy.TextCatBOW.v1"
+@architectures = "spacy.TextCatBOW.v2"
 exclusive_classes = false
 ngram_size = 1
 no_output_layer = false
@@ -44,7 +44,7 @@ DEFAULT_MULTI_TEXTCAT_MODEL = Config().from_str(multi_label_default_config)["mod
 
 multi_label_bow_config = """
 [model]
-@architectures = "spacy.TextCatBOW.v1"
+@architectures = "spacy.TextCatBOW.v2"
 exclusive_classes = false
 ngram_size = 1
 no_output_layer = false
@@ -52,7 +52,7 @@ no_output_layer = false
 
 multi_label_cnn_config = """
 [model]
-@architectures = "spacy.TextCatCNN.v1"
+@architectures = "spacy.TextCatCNN.v2"
 exclusive_classes = false
 
 [model.tok2vec]

spacy/pipeline/tok2vec.py

@@ -173,6 +173,7 @@ class Tok2Vec(TrainablePipe):
             for i in range(len(one_d_tokvecs)):
                 d_tokvecs[i] += one_d_tokvecs[i]
                 losses[self.name] += float((one_d_tokvecs[i] ** 2).sum())
+            return [self.model.ops.alloc2f(*t2v.shape) for t2v in tokvecs]
 
         def backprop(one_d_tokvecs):
             """Callback to actually do the backprop. Passed to last listener."""

spacy/pipeline/trainable_pipe.pyx

@@ -101,7 +101,8 @@ cdef class TrainablePipe(Pipe):
 
     def update(self,
                examples: Iterable["Example"],
-               *, drop: float=0.0,
+               *,
+               drop: float=0.0,
                sgd: Optimizer=None,
                losses: Optional[Dict[str, float]]=None) -> Dict[str, float]:
         """Learn from a batch of documents and gold-standard information,
@@ -213,7 +214,12 @@ cdef class TrainablePipe(Pipe):
 
     def _allow_extra_label(self) -> None:
         """Raise an error if the component can not add any more labels."""
-        if self.model.has_dim("nO") and self.model.get_dim("nO") == len(self.labels):
+        nO = None
+        if self.model.has_dim("nO"):
+            nO = self.model.get_dim("nO")
+        elif self.model.has_ref("output_layer") and self.model.get_ref("output_layer").has_dim("nO"):
+            nO = self.model.get_ref("output_layer").get_dim("nO")
+        if nO is not None and nO == len(self.labels):
             if not self.is_resizable:
                 raise ValueError(Errors.E922.format(name=self.name, nO=self.model.get_dim("nO")))
 
@@ -324,7 +330,8 @@ cdef class TrainablePipe(Pipe):
 
         def load_model(p):
             try:
-                self.model.from_bytes(p.open("rb").read())
+                with open(p, "rb") as mfile:
+                    self.model.from_bytes(mfile.read())
             except AttributeError:
                 raise ValueError(Errors.E149) from None
 

spacy/pipeline/transition_parser.pyx

@@ -29,6 +29,7 @@ from ..training import validate_examples, validate_get_examples
 from ..errors import Errors, Warnings
 from .. import util
 
+
 cdef class Parser(TrainablePipe):
     """
     Base class of the DependencyParser and EntityRecognizer.
@@ -48,15 +49,43 @@ cdef class Parser(TrainablePipe):
         beam_density=0.0,
         beam_update_prob=0.0,
         multitasks=tuple(),
+        incorrect_spans_key=None
     ):
         """Create a Parser.
 
         vocab (Vocab): The vocabulary object. Must be shared with documents
             to be processed. The value is set to the `.vocab` attribute.
-        **cfg: Configuration parameters. Set to the `.cfg` attribute.
-             If it doesn't include a value for 'moves',  a new instance is
-             created with `self.TransitionSystem()`. This defines how the
-             parse-state is created, updated and evaluated.
+        model (Model): The model for the transition-based parser. The model needs
+            to have a specific substructure of named components --- see the
+            spacy.ml.tb_framework.TransitionModel for details.
+        name (str): The name of the pipeline component
+        moves (Optional[TransitionSystem]): This defines how the parse-state is created,
+            updated and evaluated. If 'moves' is None, a new instance is
+            created with `self.TransitionSystem()`. Defaults to `None`.
+        update_with_oracle_cut_size (int): During training, cut long sequences into
+            shorter segments by creating intermediate states based on the gold-standard
+            history. The model is not very sensitive to this parameter, so you usually
+            won't need to change it. 100 is a good default.
+        min_action_freq (int): The minimum frequency of labelled actions to retain.
+            Rarer labelled actions have their label backed-off to "dep". While this
+            primarily affects the label accuracy, it can also affect the attachment
+            structure, as the labels are used to represent the pseudo-projectivity
+            transformation.
+        learn_tokens (bool): Whether to learn to merge subtokens that are split
+            relative to the gold standard. Experimental.
+        beam_width (int): The number of candidate analyses to maintain.
+        beam_density (float): The minimum ratio between the scores of the first and
+            last candidates in the beam. This allows the parser to avoid exploring
+            candidates that are too far behind. This is mostly intended to improve
+            efficiency, but it can also improve accuracy as deeper search is not
+            always better.
+        beam_update_prob (float): The chance of making a beam update, instead of a
+            greedy update. Greedy updates are an approximation for the beam updates,
+            and are faster to compute.
+        multitasks: additional multi-tasking components. Experimental.
+        incorrect_spans_key (Optional[str]): Identifies spans that are known
+            to be incorrect entity annotations. The incorrect entity annotations
+            can be stored in the span group, under this key.
         """
         self.vocab = vocab
         self.name = name
@@ -68,11 +97,16 @@ cdef class Parser(TrainablePipe):
             "learn_tokens": learn_tokens,
             "beam_width": beam_width,
             "beam_density": beam_density,
-            "beam_update_prob": beam_update_prob
+            "beam_update_prob": beam_update_prob,
+            "incorrect_spans_key": incorrect_spans_key
         }
         if moves is None:
-            # defined by EntityRecognizer as a BiluoPushDown
-            moves = self.TransitionSystem(self.vocab.strings)
+            # EntityRecognizer -> BiluoPushDown
+            # DependencyParser -> ArcEager
+            moves = self.TransitionSystem(
+                self.vocab.strings,
+                incorrect_spans_key=incorrect_spans_key
+            )
         self.moves = moves
         self.model = model
         if self.moves.n_moves != 0:
@@ -118,6 +152,10 @@ cdef class Parser(TrainablePipe):
         # Available for subclasses, e.g. to deprojectivize
         return []
 
+    @property
+    def incorrect_spans_key(self):
+        return self.cfg["incorrect_spans_key"]
+
     def add_label(self, label):
         resized = False
         for action in self.moves.action_types:
@@ -326,7 +364,6 @@ cdef class Parser(TrainablePipe):
         )
         for multitask in self._multitasks:
             multitask.update(examples, drop=drop, sgd=sgd)
-    
         n_examples = len([eg for eg in examples if self.moves.has_gold(eg)])
         if n_examples == 0:
             return losses
@@ -554,7 +591,7 @@ cdef class Parser(TrainablePipe):
                 self._resize()
                 self.model.from_bytes(bytes_data)
             except AttributeError:
-                raise ValueError(Errors.E149) from None
+                raise ValueError(Errors.E149)
         return self
 
     def to_bytes(self, exclude=tuple()):

spacy/schemas.py

@@ -4,7 +4,7 @@ from enum import Enum
 from pydantic import BaseModel, Field, ValidationError, validator, create_model
 from pydantic import StrictStr, StrictInt, StrictFloat, StrictBool
 from pydantic.main import ModelMetaclass
-from thinc.api import Optimizer, ConfigValidationError
+from thinc.api import Optimizer, ConfigValidationError, Model
 from thinc.config import Promise
 from collections import defaultdict
 import inspect
@@ -17,6 +17,7 @@ if TYPE_CHECKING:
     # This lets us add type hints for mypy etc. without causing circular imports
     from .language import Language  # noqa: F401
     from .training import Example  # noqa: F401
+    from .vocab import Vocab  # noqa: F401
 
 
 # fmt: off
@@ -354,7 +355,7 @@ class ConfigSchemaPretrain(BaseModel):
     batcher: Batcher = Field(..., title="Batcher for the training data")
     component: str = Field(..., title="Component to find the layer to pretrain")
     layer: str = Field(..., title="Layer to pretrain. Whole model if empty.")
-    objective: Callable[["Vocab", "Model"], "Model"] = Field(..., title="A function that creates the pretraining objective.")
+    objective: Callable[["Vocab", Model], Model] = Field(..., title="A function that creates the pretraining objective.")
     # fmt: on
 
     class Config:

spacy/scorer.py

@@ -365,7 +365,9 @@ class Scorer:
                 gold_spans.add(gold_span)
                 gold_per_type[span.label_].add(gold_span)
             pred_per_type = {label: set() for label in labels}
-            for span in example.get_aligned_spans_x2y(getter(pred_doc, attr), allow_overlap):
+            for span in example.get_aligned_spans_x2y(
+                getter(pred_doc, attr), allow_overlap
+            ):
                 if labeled:
                     pred_span = (span.label_, span.start, span.end - 1)
                 else:
@@ -381,10 +383,10 @@ class Scorer:
             score.score_set(pred_spans, gold_spans)
         # Assemble final result
         final_scores = {
-                f"{attr}_p": None,
-                f"{attr}_r": None,
-                f"{attr}_f": None,
-            }
+            f"{attr}_p": None,
+            f"{attr}_r": None,
+            f"{attr}_f": None,
+        }
         if labeled:
             final_scores[f"{attr}_per_type"] = None
         if len(score) > 0:
@@ -392,7 +394,9 @@ class Scorer:
             final_scores[f"{attr}_r"] = score.recall
             final_scores[f"{attr}_f"] = score.fscore
             if labeled:
-                final_scores[f"{attr}_per_type"] = {k: v.to_dict() for k, v in score_per_type.items()}
+                final_scores[f"{attr}_per_type"] = {
+                    k: v.to_dict() for k, v in score_per_type.items()
+                }
         return final_scores
 
     @staticmethod

spacy/tests/conftest.py

@@ -281,6 +281,13 @@ def uk_tokenizer():
     return get_lang_class("uk")().tokenizer
 
 
+@pytest.fixture
+def uk_lemmatizer():
+    pytest.importorskip("pymorphy2")
+    pytest.importorskip("pymorphy2_dicts_uk")
+    return get_lang_class("uk")().add_pipe("lemmatizer")
+
+
 @pytest.fixture(scope="session")
 def ur_tokenizer():
     return get_lang_class("ur")().tokenizer

spacy/tests/doc/test_add_entities.py

@@ -18,14 +18,9 @@ def _ner_example(ner):
 def test_doc_add_entities_set_ents_iob(en_vocab):
     text = ["This", "is", "a", "lion"]
     doc = Doc(en_vocab, words=text)
-    config = {
-        "learn_tokens": False,
-        "min_action_freq": 30,
-        "update_with_oracle_cut_size": 100,
-    }
     cfg = {"model": DEFAULT_NER_MODEL}
     model = registry.resolve(cfg, validate=True)["model"]
-    ner = EntityRecognizer(en_vocab, model, **config)
+    ner = EntityRecognizer(en_vocab, model)
     ner.initialize(lambda: [_ner_example(ner)])
     ner(doc)
 
@@ -40,14 +35,9 @@ def test_ents_reset(en_vocab):
     """Ensure that resetting doc.ents does not change anything"""
     text = ["This", "is", "a", "lion"]
     doc = Doc(en_vocab, words=text)
-    config = {
-        "learn_tokens": False,
-        "min_action_freq": 30,
-        "update_with_oracle_cut_size": 100,
-    }
     cfg = {"model": DEFAULT_NER_MODEL}
     model = registry.resolve(cfg, validate=True)["model"]
-    ner = EntityRecognizer(en_vocab, model, **config)
+    ner = EntityRecognizer(en_vocab, model)
     ner.initialize(lambda: [_ner_example(ner)])
     ner(doc)
     orig_iobs = [t.ent_iob_ for t in doc]

spacy/tests/doc/test_creation.py

@@ -63,3 +63,10 @@ def test_create_from_words_and_text(vocab):
         words = [" ", " ", "'", "dogs", "'", "\n\n", "run"]
         text = "  'dogs'\n\nrun  "
         (words, spaces) = util.get_words_and_spaces(words + ["away"], text)
+
+
+def test_create_with_heads_and_no_deps(vocab):
+    words = "I like ginger".split()
+    heads = list(range(len(words)))
+    with pytest.raises(ValueError):
+        doc = Doc(vocab, words=words, heads=heads)

spacy/tests/doc/test_doc_api.py

@@ -2,8 +2,6 @@ import weakref
 
 import pytest
 import numpy
-import logging
-import mock
 
 from spacy.lang.xx import MultiLanguage
 from spacy.tokens import Doc, Span, Token
@@ -158,13 +156,10 @@ def test_doc_api_serialize(en_tokenizer, text):
     def inner_func(d1, d2):
         return "hello!"
 
-    logger = logging.getLogger("spacy")
-    with mock.patch.object(logger, "warning") as mock_warning:
-        _ = tokens.to_bytes()  # noqa: F841
-        mock_warning.assert_not_called()
+    _ = tokens.to_bytes()  # noqa: F841
+    with pytest.warns(UserWarning):
         tokens.user_hooks["similarity"] = inner_func
         _ = tokens.to_bytes()  # noqa: F841
-        mock_warning.assert_called_once()
 
 
 def test_doc_api_set_ents(en_tokenizer):
@@ -351,17 +346,25 @@ def test_doc_from_array_morph(en_vocab):
 
 @pytest.mark.usefixtures("clean_underscore")
 def test_doc_api_from_docs(en_tokenizer, de_tokenizer):
-    en_texts = ["Merging the docs is fun.", "", "They don't think alike."]
+    en_texts = [
+        "Merging the docs is fun.",
+        "",
+        "They don't think alike. ",
+        "Another doc.",
+    ]
     en_texts_without_empty = [t for t in en_texts if len(t)]
     de_text = "Wie war die Frage?"
     en_docs = [en_tokenizer(text) for text in en_texts]
     en_docs[0].spans["group"] = [en_docs[0][1:4]]
     en_docs[2].spans["group"] = [en_docs[2][1:4]]
-    span_group_texts = sorted([en_docs[0][1:4].text, en_docs[2][1:4].text])
+    en_docs[3].spans["group"] = [en_docs[3][0:1]]
+    span_group_texts = sorted(
+        [en_docs[0][1:4].text, en_docs[2][1:4].text, en_docs[3][0:1].text]
+    )
     de_doc = de_tokenizer(de_text)
     Token.set_extension("is_ambiguous", default=False)
-    en_docs[0][2]._.is_ambiguous = True # docs
-    en_docs[2][3]._.is_ambiguous = True # think
+    en_docs[0][2]._.is_ambiguous = True  # docs
+    en_docs[2][3]._.is_ambiguous = True  # think
     assert Doc.from_docs([]) is None
     assert de_doc is not Doc.from_docs([de_doc])
     assert str(de_doc) == str(Doc.from_docs([de_doc]))
@@ -371,24 +374,25 @@ def test_doc_api_from_docs(en_tokenizer, de_tokenizer):
 
     m_doc = Doc.from_docs(en_docs)
     assert len(en_texts_without_empty) == len(list(m_doc.sents))
-    assert len(str(m_doc)) > len(en_texts[0]) + len(en_texts[1])
-    assert str(m_doc) == " ".join(en_texts_without_empty)
+    assert len(m_doc.text) > len(en_texts[0]) + len(en_texts[1])
+    assert m_doc.text == " ".join([t.strip() for t in en_texts_without_empty])
     p_token = m_doc[len(en_docs[0]) - 1]
     assert p_token.text == "." and bool(p_token.whitespace_)
     en_docs_tokens = [t for doc in en_docs for t in doc]
     assert len(m_doc) == len(en_docs_tokens)
     think_idx = len(en_texts[0]) + 1 + en_texts[2].index("think")
-    assert m_doc[2]._.is_ambiguous == True
+    assert m_doc[2]._.is_ambiguous is True
     assert m_doc[9].idx == think_idx
-    assert m_doc[9]._.is_ambiguous == True
+    assert m_doc[9]._.is_ambiguous is True
     assert not any([t._.is_ambiguous for t in m_doc[3:8]])
     assert "group" in m_doc.spans
     assert span_group_texts == sorted([s.text for s in m_doc.spans["group"]])
+    assert bool(m_doc[11].whitespace_)
 
     m_doc = Doc.from_docs(en_docs, ensure_whitespace=False)
     assert len(en_texts_without_empty) == len(list(m_doc.sents))
-    assert len(str(m_doc)) == sum(len(t) for t in en_texts)
-    assert str(m_doc) == "".join(en_texts)
+    assert len(m_doc.text) == sum(len(t) for t in en_texts)
+    assert m_doc.text == "".join(en_texts_without_empty)
     p_token = m_doc[len(en_docs[0]) - 1]
     assert p_token.text == "." and not bool(p_token.whitespace_)
     en_docs_tokens = [t for doc in en_docs for t in doc]
@@ -397,11 +401,12 @@ def test_doc_api_from_docs(en_tokenizer, de_tokenizer):
     assert m_doc[9].idx == think_idx
     assert "group" in m_doc.spans
     assert span_group_texts == sorted([s.text for s in m_doc.spans["group"]])
+    assert bool(m_doc[11].whitespace_)
 
     m_doc = Doc.from_docs(en_docs, attrs=["lemma", "length", "pos"])
-    assert len(str(m_doc)) > len(en_texts[0]) + len(en_texts[1])
+    assert len(m_doc.text) > len(en_texts[0]) + len(en_texts[1])
     # space delimiter considered, although spacy attribute was missing
-    assert str(m_doc) == " ".join(en_texts_without_empty)
+    assert m_doc.text == " ".join([t.strip() for t in en_texts_without_empty])
     p_token = m_doc[len(en_docs[0]) - 1]
     assert p_token.text == "." and bool(p_token.whitespace_)
     en_docs_tokens = [t for doc in en_docs for t in doc]
@@ -414,6 +419,16 @@ def test_doc_api_from_docs(en_tokenizer, de_tokenizer):
     # can merge empty docs
     doc = Doc.from_docs([en_tokenizer("")] * 10)
 
+    # empty but set spans keys are preserved
+    en_docs = [en_tokenizer(text) for text in en_texts]
+    m_doc = Doc.from_docs(en_docs)
+    assert "group" not in m_doc.spans
+    for doc in en_docs:
+        doc.spans["group"] = []
+    m_doc = Doc.from_docs(en_docs)
+    assert "group" in m_doc.spans
+    assert len(m_doc.spans["group"]) == 0
+
 
 def test_doc_api_from_docs_ents(en_tokenizer):
     texts = ["Merging the docs is fun.", "They don't think alike."]

spacy/tests/doc/test_retokenize_merge.py

@@ -108,9 +108,12 @@ def test_doc_retokenize_spans_merge_tokens_default_attrs(en_vocab):
     words = ["The", "players", "start", "."]
     lemmas = [t.lower() for t in words]
     heads = [1, 2, 2, 2]
+    deps = ["dep"] * len(heads)
     tags = ["DT", "NN", "VBZ", "."]
     pos = ["DET", "NOUN", "VERB", "PUNCT"]
-    doc = Doc(en_vocab, words=words, tags=tags, pos=pos, heads=heads, lemmas=lemmas)
+    doc = Doc(
+        en_vocab, words=words, tags=tags, pos=pos, heads=heads, deps=deps, lemmas=lemmas
+    )
     assert len(doc) == 4
     assert doc[0].text == "The"
     assert doc[0].tag_ == "DT"
@@ -123,7 +126,9 @@ def test_doc_retokenize_spans_merge_tokens_default_attrs(en_vocab):
     assert doc[0].tag_ == "NN"
     assert doc[0].pos_ == "NOUN"
     assert doc[0].lemma_ == "the players"
-    doc = Doc(en_vocab, words=words, tags=tags, pos=pos, heads=heads, lemmas=lemmas)
+    doc = Doc(
+        en_vocab, words=words, tags=tags, pos=pos, heads=heads, deps=deps, lemmas=lemmas
+    )
     assert len(doc) == 4
     assert doc[0].text == "The"
     assert doc[0].tag_ == "DT"
@@ -190,8 +195,9 @@ def test_doc_retokenize_span_np_merges(en_tokenizer):
 
     text = "displaCy is a lightweight and modern dependency parse tree visualization tool built with CSS3 and JavaScript."
     heads = [1, 1, 10, 7, 3, 3, 7, 10, 9, 10, 1, 10, 11, 12, 13, 13, 1]
+    deps = ["dep"] * len(heads)
     tokens = en_tokenizer(text)
-    doc = Doc(tokens.vocab, words=[t.text for t in tokens], heads=heads)
+    doc = Doc(tokens.vocab, words=[t.text for t in tokens], heads=heads, deps=deps)
     with doc.retokenize() as retokenizer:
         for ent in doc.ents:
             attrs = {"tag": ent.label_, "lemma": ent.lemma_, "ent_type": ent.label_}
@@ -199,8 +205,9 @@ def test_doc_retokenize_span_np_merges(en_tokenizer):
 
     text = "One test with entities like New York City so the ents list is not void"
     heads = [1, 1, 1, 2, 3, 6, 7, 4, 12, 11, 11, 12, 1, 12, 12]
+    deps = ["dep"] * len(heads)
     tokens = en_tokenizer(text)
-    doc = Doc(tokens.vocab, words=[t.text for t in tokens], heads=heads)
+    doc = Doc(tokens.vocab, words=[t.text for t in tokens], heads=heads, deps=deps)
     with doc.retokenize() as retokenizer:
         for ent in doc.ents:
             retokenizer.merge(ent)
@@ -210,6 +217,7 @@ def test_doc_retokenize_spans_entity_merge(en_tokenizer):
     # fmt: off
     text = "Stewart Lee is a stand up comedian who lives in England and loves Joe Pasquale.\n"
     heads = [1, 2, 2, 4, 6, 4, 2, 8, 6, 8, 9, 8, 8, 14, 12, 2, 15]
+    deps = ["dep"] * len(heads)
     tags = ["NNP", "NNP", "VBZ", "DT", "VB", "RP", "NN", "WP", "VBZ", "IN", "NNP", "CC", "VBZ", "NNP", "NNP", ".", "SP"]
     ents = [("PERSON", 0, 2), ("GPE", 10, 11), ("PERSON", 13, 15)]
     ents = ["O"] * len(heads)
@@ -221,7 +229,12 @@ def test_doc_retokenize_spans_entity_merge(en_tokenizer):
     # fmt: on
     tokens = en_tokenizer(text)
     doc = Doc(
-        tokens.vocab, words=[t.text for t in tokens], heads=heads, tags=tags, ents=ents
+        tokens.vocab,
+        words=[t.text for t in tokens],
+        heads=heads,
+        deps=deps,
+        tags=tags,
+        ents=ents,
     )
     assert len(doc) == 17
     with doc.retokenize() as retokenizer:
@@ -471,7 +484,7 @@ def test_doc_retokenize_merge_without_parse_keeps_sents(en_tokenizer):
     assert len(list(doc.sents)) == 2
     with doc.retokenize() as retokenizer:
         retokenizer.merge(doc[3:6])
-    assert doc[3].is_sent_start == None
+    assert doc[3].is_sent_start is None
 
     # merging over a sentence boundary and setting sent_start
     doc = Doc(tokens.vocab, words=[t.text for t in tokens], sent_starts=sent_starts)

spacy/tests/doc/test_retokenize_split.py

@@ -44,7 +44,8 @@ def test_doc_retokenize_split_lemmas(en_vocab):
     # If lemmas are not set, leave unset
     words = ["LosAngeles", "start", "."]
     heads = [1, 2, 2]
-    doc = Doc(en_vocab, words=words, heads=heads)
+    deps = ["dep"] * len(heads)
+    doc = Doc(en_vocab, words=words, heads=heads, deps=deps)
     with doc.retokenize() as retokenizer:
         retokenizer.split(
             doc[0],
@@ -57,7 +58,8 @@ def test_doc_retokenize_split_lemmas(en_vocab):
     # If lemmas are set, use split orth as default lemma
     words = ["LosAngeles", "start", "."]
     heads = [1, 2, 2]
-    doc = Doc(en_vocab, words=words, heads=heads)
+    deps = ["dep"] * len(heads)
+    doc = Doc(en_vocab, words=words, heads=heads, deps=deps)
     for t in doc:
         t.lemma_ = "a"
     with doc.retokenize() as retokenizer:

spacy/tests/doc/test_span.py

@@ -20,7 +20,17 @@ def doc(en_tokenizer):
             "O", "O", "O", "O", "O"]
     # fmt: on
     tokens = en_tokenizer(text)
-    return Doc(tokens.vocab, words=[t.text for t in tokens], heads=heads, deps=deps, ents=ents)
+    lemmas = [t.text for t in tokens]  # this is not correct, just a placeholder
+    spaces = [bool(t.whitespace_) for t in tokens]
+    return Doc(
+        tokens.vocab,
+        words=[t.text for t in tokens],
+        spaces=spaces,
+        heads=heads,
+        deps=deps,
+        ents=ents,
+        lemmas=lemmas,
+    )
 
 
 @pytest.fixture
@@ -84,7 +94,7 @@ def test_spans_span_sent(doc, doc_not_parsed):
     """Test span.sent property"""
     assert len(list(doc.sents))
     assert doc[:2].sent.root.text == "is"
-    assert doc[:2].sent.text == "This is a sentence ."
+    assert doc[:2].sent.text == "This is a sentence."
     assert doc[6:7].sent.root.left_edge.text == "This"
     # test on manual sbd
     doc_not_parsed[0].is_sent_start = True
@@ -249,7 +259,7 @@ def test_span_as_doc(doc):
 
 @pytest.mark.usefixtures("clean_underscore")
 def test_span_as_doc_user_data(doc):
-    """Test that the user_data can be preserved (but not by default). """
+    """Test that the user_data can be preserved (but not by default)."""
     my_key = "my_info"
     my_value = 342
     doc.user_data[my_key] = my_value
@@ -286,7 +296,6 @@ def test_span_attrs_writable(doc):
 
 
 def test_span_ents_property(doc):
-    """Test span.ents for the """
     doc.ents = [
         (doc.vocab.strings["PRODUCT"], 0, 1),
         (doc.vocab.strings["PRODUCT"], 7, 8),
@@ -308,7 +317,7 @@ def test_span_ents_property(doc):
     assert sentences[1].ents[0].start == 7
     assert sentences[1].ents[0].end == 8
     # Third sentence ents, Also tests end of sentence
-    assert sentences[2].ents[0].text == "a third ."
+    assert sentences[2].ents[0].text == "a third."
     assert sentences[2].ents[0].label_ == "PRODUCT"
     assert sentences[2].ents[0].start == 11
     assert sentences[2].ents[0].end == 14
@@ -361,6 +370,12 @@ def test_span_boundaries(doc):
         span[5]
 
 
+def test_span_lemma(doc):
+    # span lemmas should have the same number of spaces as the span
+    sp = doc[1:5]
+    assert len(sp.text.split(" ")) == len(sp.lemma_.split(" "))
+
+
 def test_sent(en_tokenizer):
     doc = en_tokenizer("Check span.sent raises error if doc is not sentencized.")
     span = doc[1:3]

spacy/tests/doc/test_token_api.py

@@ -95,7 +95,8 @@ def test_doc_token_api_ancestors(en_vocab):
     # the structure of this sentence depends on the English annotation scheme
     words = ["Yesterday", "I", "saw", "a", "dog", "that", "barked", "loudly", "."]
     heads = [2, 2, 2, 4, 2, 6, 4, 6, 2]
-    doc = Doc(en_vocab, words=words, heads=heads)
+    deps = ["dep"] * len(heads)
+    doc = Doc(en_vocab, words=words, heads=heads, deps=deps)
     assert [t.text for t in doc[6].ancestors] == ["dog", "saw"]
     assert [t.text for t in doc[1].ancestors] == ["saw"]
     assert [t.text for t in doc[2].ancestors] == []
@@ -146,7 +147,7 @@ def test_doc_token_api_head_setter(en_vocab):
     assert doc[4].left_edge.i == 0
     assert doc[2].left_edge.i == 0
     # head token must be from the same document
-    doc2 = Doc(en_vocab, words=words, heads=heads)
+    doc2 = Doc(en_vocab, words=words, heads=heads, deps=["dep"] * len(heads))
     with pytest.raises(ValueError):
         doc[0].head = doc2[0]
     # test sentence starts when two sentences are joined
@@ -254,7 +255,7 @@ def test_token_api_non_conjuncts(en_vocab):
 
 
 def test_missing_head_dep(en_vocab):
-    """ Check that the Doc constructor and Example.from_dict parse missing information the same"""
+    """Check that the Doc constructor and Example.from_dict parse missing information the same"""
     heads = [1, 1, 1, 1, 2, None]  # element 5 is missing
     deps = ["", "ROOT", "dobj", "cc", "conj", None]  # element 0 and 5 are missing
     words = ["I", "like", "London", "and", "Berlin", "."]

spacy/tests/lang/bg/test_text.py

@@ -1,5 +1,5 @@
 import pytest
-from spacy.lang.bg.lex_attrs import like_num
+
 
 @pytest.mark.parametrize(
     "word,match",