mirror of
https://github.com/explosion/spaCy.git
synced 2024-12-24 17:06:29 +03:00
Merge remote-tracking branch 'upstream/master' into develop
This commit is contained in:
commit
a3f6d6bce1
7
.github/azure-steps.yml
vendored
7
.github/azure-steps.yml
vendored
|
@ -27,7 +27,6 @@ steps:
|
||||||
|
|
||||||
- script: python -m mypy spacy
|
- script: python -m mypy spacy
|
||||||
displayName: 'Run mypy'
|
displayName: 'Run mypy'
|
||||||
condition: ne(variables['python_version'], '3.10')
|
|
||||||
|
|
||||||
- task: DeleteFiles@1
|
- task: DeleteFiles@1
|
||||||
inputs:
|
inputs:
|
||||||
|
@ -41,7 +40,7 @@ steps:
|
||||||
|
|
||||||
- bash: |
|
- bash: |
|
||||||
${{ parameters.prefix }} SDIST=$(python -c "import os;print(os.listdir('./dist')[-1])" 2>&1)
|
${{ parameters.prefix }} SDIST=$(python -c "import os;print(os.listdir('./dist')[-1])" 2>&1)
|
||||||
${{ parameters.prefix }} python -m pip install dist/$SDIST
|
${{ parameters.prefix }} SPACY_NUM_BUILD_JOBS=2 python -m pip install dist/$SDIST
|
||||||
displayName: "Install from sdist"
|
displayName: "Install from sdist"
|
||||||
|
|
||||||
- script: |
|
- script: |
|
||||||
|
@ -111,7 +110,7 @@ steps:
|
||||||
condition: eq(variables['python_version'], '3.8')
|
condition: eq(variables['python_version'], '3.8')
|
||||||
|
|
||||||
- script: |
|
- script: |
|
||||||
${{ parameters.prefix }} python -m pip install thinc-apple-ops
|
${{ parameters.prefix }} python -m pip install --pre thinc-apple-ops
|
||||||
${{ parameters.prefix }} python -m pytest --pyargs spacy
|
${{ parameters.prefix }} python -m pytest --pyargs spacy
|
||||||
displayName: "Run CPU tests with thinc-apple-ops"
|
displayName: "Run CPU tests with thinc-apple-ops"
|
||||||
condition: and(startsWith(variables['imageName'], 'macos'), eq(variables['python.version'], '3.9'))
|
condition: and(startsWith(variables['imageName'], 'macos'), eq(variables['python.version'], '3.10'))
|
||||||
|
|
106
.github/contributors/Lucaterre.md
vendored
Normal file
106
.github/contributors/Lucaterre.md
vendored
Normal file
|
@ -0,0 +1,106 @@
|
||||||
|
# spaCy contributor agreement
|
||||||
|
|
||||||
|
This spaCy Contributor Agreement (**"SCA"**) is based on the
|
||||||
|
[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf).
|
||||||
|
The SCA applies to any contribution that you make to any product or project
|
||||||
|
managed by us (the **"project"**), and sets out the intellectual property rights
|
||||||
|
you grant to us in the contributed materials. The term **"us"** shall mean
|
||||||
|
[ExplosionAI GmbH](https://explosion.ai/legal). The term
|
||||||
|
**"you"** shall mean the person or entity identified below.
|
||||||
|
|
||||||
|
If you agree to be bound by these terms, fill in the information requested
|
||||||
|
below and include the filled-in version with your first pull request, under the
|
||||||
|
folder [`.github/contributors/`](/.github/contributors/). The name of the file
|
||||||
|
should be your GitHub username, with the extension `.md`. For example, the user
|
||||||
|
example_user would create the file `.github/contributors/example_user.md`.
|
||||||
|
|
||||||
|
Read this agreement carefully before signing. These terms and conditions
|
||||||
|
constitute a binding legal agreement.
|
||||||
|
|
||||||
|
## Contributor Agreement
|
||||||
|
|
||||||
|
1. The term "contribution" or "contributed materials" means any source code,
|
||||||
|
object code, patch, tool, sample, graphic, specification, manual,
|
||||||
|
documentation, or any other material posted or submitted by you to the project.
|
||||||
|
|
||||||
|
2. With respect to any worldwide copyrights, or copyright applications and
|
||||||
|
registrations, in your contribution:
|
||||||
|
|
||||||
|
* you hereby assign to us joint ownership, and to the extent that such
|
||||||
|
assignment is or becomes invalid, ineffective or unenforceable, you hereby
|
||||||
|
grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge,
|
||||||
|
royalty-free, unrestricted license to exercise all rights under those
|
||||||
|
copyrights. This includes, at our option, the right to sublicense these same
|
||||||
|
rights to third parties through multiple levels of sublicensees or other
|
||||||
|
licensing arrangements;
|
||||||
|
|
||||||
|
* you agree that each of us can do all things in relation to your
|
||||||
|
contribution as if each of us were the sole owners, and if one of us makes
|
||||||
|
a derivative work of your contribution, the one who makes the derivative
|
||||||
|
work (or has it made will be the sole owner of that derivative work;
|
||||||
|
|
||||||
|
* you agree that you will not assert any moral rights in your contribution
|
||||||
|
against us, our licensees or transferees;
|
||||||
|
|
||||||
|
* you agree that we may register a copyright in your contribution and
|
||||||
|
exercise all ownership rights associated with it; and
|
||||||
|
|
||||||
|
* you agree that neither of us has any duty to consult with, obtain the
|
||||||
|
consent of, pay or render an accounting to the other for any use or
|
||||||
|
distribution of your contribution.
|
||||||
|
|
||||||
|
3. With respect to any patents you own, or that you can license without payment
|
||||||
|
to any third party, you hereby grant to us a perpetual, irrevocable,
|
||||||
|
non-exclusive, worldwide, no-charge, royalty-free license to:
|
||||||
|
|
||||||
|
* make, have made, use, sell, offer to sell, import, and otherwise transfer
|
||||||
|
your contribution in whole or in part, alone or in combination with or
|
||||||
|
included in any product, work or materials arising out of the project to
|
||||||
|
which your contribution was submitted, and
|
||||||
|
|
||||||
|
* at our option, to sublicense these same rights to third parties through
|
||||||
|
multiple levels of sublicensees or other licensing arrangements.
|
||||||
|
|
||||||
|
4. Except as set out above, you keep all right, title, and interest in your
|
||||||
|
contribution. The rights that you grant to us under these terms are effective
|
||||||
|
on the date you first submitted a contribution to us, even if your submission
|
||||||
|
took place before the date you sign these terms.
|
||||||
|
|
||||||
|
5. You covenant, represent, warrant and agree that:
|
||||||
|
|
||||||
|
* Each contribution that you submit is and shall be an original work of
|
||||||
|
authorship and you can legally grant the rights set out in this SCA;
|
||||||
|
|
||||||
|
* to the best of your knowledge, each contribution will not violate any
|
||||||
|
third party's copyrights, trademarks, patents, or other intellectual
|
||||||
|
property rights; and
|
||||||
|
|
||||||
|
* each contribution shall be in compliance with U.S. export control laws and
|
||||||
|
other applicable export and import laws. You agree to notify us if you
|
||||||
|
become aware of any circumstance which would make any of the foregoing
|
||||||
|
representations inaccurate in any respect. We may publicly disclose your
|
||||||
|
participation in the project, including the fact that you have signed the SCA.
|
||||||
|
|
||||||
|
6. This SCA is governed by the laws of the State of California and applicable
|
||||||
|
U.S. Federal law. Any choice of law rules will not apply.
|
||||||
|
|
||||||
|
7. Please place an “x” on one of the applicable statement below. Please do NOT
|
||||||
|
mark both statements:
|
||||||
|
|
||||||
|
* [x] I am signing on behalf of myself as an individual and no other person
|
||||||
|
or entity, including my employer, has or will have rights with respect to my
|
||||||
|
contributions.
|
||||||
|
|
||||||
|
* [ ] I am signing on behalf of my employer or a legal entity and I have the
|
||||||
|
actual authority to contractually bind that entity.
|
||||||
|
|
||||||
|
## Contributor Details
|
||||||
|
|
||||||
|
| Field | Entry |
|
||||||
|
|------------------------------- |---------------|
|
||||||
|
| Name | Lucas Terriel |
|
||||||
|
| Company name (if applicable) | |
|
||||||
|
| Title or role (if applicable) | |
|
||||||
|
| Date | 2022-06-20 |
|
||||||
|
| GitHub username | Lucaterre |
|
||||||
|
| Website (optional) | |
|
67
.github/spacy_universe_alert.py
vendored
Normal file
67
.github/spacy_universe_alert.py
vendored
Normal file
|
@ -0,0 +1,67 @@
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
import json
|
||||||
|
from datetime import datetime
|
||||||
|
|
||||||
|
from slack_sdk.web.client import WebClient
|
||||||
|
|
||||||
|
CHANNEL = "#alerts-universe"
|
||||||
|
SLACK_TOKEN = os.environ.get("SLACK_BOT_TOKEN", "ENV VAR not available!")
|
||||||
|
DATETIME_FORMAT = "%Y-%m-%dT%H:%M:%SZ"
|
||||||
|
|
||||||
|
client = WebClient(SLACK_TOKEN)
|
||||||
|
github_context = json.loads(sys.argv[1])
|
||||||
|
|
||||||
|
event = github_context['event']
|
||||||
|
pr_title = event['pull_request']["title"]
|
||||||
|
pr_link = event['pull_request']["patch_url"].replace(".patch", "")
|
||||||
|
pr_author_url = event['sender']["html_url"]
|
||||||
|
pr_author_name = pr_author_url.rsplit('/')[-1]
|
||||||
|
pr_created_at_dt = datetime.strptime(
|
||||||
|
event['pull_request']["created_at"],
|
||||||
|
DATETIME_FORMAT
|
||||||
|
)
|
||||||
|
pr_created_at = pr_created_at_dt.strftime("%c")
|
||||||
|
pr_updated_at_dt = datetime.strptime(
|
||||||
|
event['pull_request']["updated_at"],
|
||||||
|
DATETIME_FORMAT
|
||||||
|
)
|
||||||
|
pr_updated_at = pr_updated_at_dt.strftime("%c")
|
||||||
|
|
||||||
|
blocks = [
|
||||||
|
{
|
||||||
|
"type": "section",
|
||||||
|
"text": {
|
||||||
|
"type": "mrkdwn",
|
||||||
|
"text": "📣 New spaCy Universe Project Alert ✨"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "section",
|
||||||
|
"fields": [
|
||||||
|
{
|
||||||
|
"type": "mrkdwn",
|
||||||
|
"text": f"*Pull Request:*\n<{pr_link}|{pr_title}>"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "mrkdwn",
|
||||||
|
"text": f"*Author:*\n<{pr_author_url}|{pr_author_name}>"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "mrkdwn",
|
||||||
|
"text": f"*Created at:*\n {pr_created_at}"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "mrkdwn",
|
||||||
|
"text": f"*Last Updated:*\n {pr_updated_at}"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
client.chat_postMessage(
|
||||||
|
channel=CHANNEL,
|
||||||
|
text="spaCy universe project PR alert",
|
||||||
|
blocks=blocks
|
||||||
|
)
|
2
.github/workflows/explosionbot.yml
vendored
2
.github/workflows/explosionbot.yml
vendored
|
@ -23,5 +23,5 @@ jobs:
|
||||||
env:
|
env:
|
||||||
INPUT_TOKEN: ${{ secrets.EXPLOSIONBOT_TOKEN }}
|
INPUT_TOKEN: ${{ secrets.EXPLOSIONBOT_TOKEN }}
|
||||||
INPUT_BK_TOKEN: ${{ secrets.BUILDKITE_SECRET }}
|
INPUT_BK_TOKEN: ${{ secrets.BUILDKITE_SECRET }}
|
||||||
ENABLED_COMMANDS: "test_gpu,test_slow"
|
ENABLED_COMMANDS: "test_gpu,test_slow,test_slow_gpu"
|
||||||
ALLOWED_TEAMS: "spaCy"
|
ALLOWED_TEAMS: "spaCy"
|
||||||
|
|
1
.github/workflows/gputests.yml
vendored
1
.github/workflows/gputests.yml
vendored
|
@ -10,6 +10,7 @@ jobs:
|
||||||
fail-fast: false
|
fail-fast: false
|
||||||
matrix:
|
matrix:
|
||||||
branch: [master, v4]
|
branch: [master, v4]
|
||||||
|
if: github.repository_owner == 'explosion'
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
steps:
|
steps:
|
||||||
- name: Trigger buildkite build
|
- name: Trigger buildkite build
|
||||||
|
|
1
.github/workflows/slowtests.yml
vendored
1
.github/workflows/slowtests.yml
vendored
|
@ -10,6 +10,7 @@ jobs:
|
||||||
fail-fast: false
|
fail-fast: false
|
||||||
matrix:
|
matrix:
|
||||||
branch: [master, v4]
|
branch: [master, v4]
|
||||||
|
if: github.repository_owner == 'explosion'
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
steps:
|
steps:
|
||||||
- name: Checkout
|
- name: Checkout
|
||||||
|
|
30
.github/workflows/spacy_universe_alert.yml
vendored
Normal file
30
.github/workflows/spacy_universe_alert.yml
vendored
Normal file
|
@ -0,0 +1,30 @@
|
||||||
|
name: spaCy universe project alert
|
||||||
|
|
||||||
|
on:
|
||||||
|
pull_request_target:
|
||||||
|
paths:
|
||||||
|
- "website/meta/universe.json"
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
build:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
|
||||||
|
steps:
|
||||||
|
- name: Dump GitHub context
|
||||||
|
env:
|
||||||
|
GITHUB_CONTEXT: ${{ toJson(github) }}
|
||||||
|
PR_NUMBER: ${{github.event.number}}
|
||||||
|
run: |
|
||||||
|
echo "$GITHUB_CONTEXT"
|
||||||
|
|
||||||
|
- uses: actions/checkout@v1
|
||||||
|
- uses: actions/setup-python@v1
|
||||||
|
- name: Install Bernadette app dependency and send an alert
|
||||||
|
env:
|
||||||
|
SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
|
||||||
|
GITHUB_CONTEXT: ${{ toJson(github) }}
|
||||||
|
CHANNEL: "#alerts-universe"
|
||||||
|
run: |
|
||||||
|
pip install slack-sdk==3.17.2 aiohttp==3.8.1
|
||||||
|
echo "$CHANNEL"
|
||||||
|
python .github/spacy_universe_alert.py "$GITHUB_CONTEXT"
|
|
@ -271,7 +271,8 @@ except: # noqa: E722
|
||||||
|
|
||||||
### Python conventions
|
### Python conventions
|
||||||
|
|
||||||
All Python code must be written **compatible with Python 3.6+**.
|
All Python code must be written **compatible with Python 3.6+**. More detailed
|
||||||
|
code conventions can be found in the [developer docs](https://github.com/explosion/spaCy/blob/master/extra/DEVELOPER_DOCS/Code%20Conventions.md).
|
||||||
|
|
||||||
#### I/O and handling paths
|
#### I/O and handling paths
|
||||||
|
|
||||||
|
|
|
@ -1,4 +1,4 @@
|
||||||
recursive-include spacy *.pyi *.pyx *.pxd *.txt *.cfg *.jinja *.toml
|
recursive-include spacy *.pyi *.pyx *.pxd *.txt *.cfg *.jinja *.toml *.hh
|
||||||
include LICENSE
|
include LICENSE
|
||||||
include README.md
|
include README.md
|
||||||
include pyproject.toml
|
include pyproject.toml
|
||||||
|
|
|
@ -16,7 +16,7 @@ production-ready [**training system**](https://spacy.io/usage/training) and easy
|
||||||
model packaging, deployment and workflow management. spaCy is commercial
|
model packaging, deployment and workflow management. spaCy is commercial
|
||||||
open-source software, released under the MIT license.
|
open-source software, released under the MIT license.
|
||||||
|
|
||||||
💫 **Version 3.2 out now!**
|
💫 **Version 3.4.0 out now!**
|
||||||
[Check out the release notes here.](https://github.com/explosion/spaCy/releases)
|
[Check out the release notes here.](https://github.com/explosion/spaCy/releases)
|
||||||
|
|
||||||
[![Azure Pipelines](https://img.shields.io/azure-devops/build/explosion-ai/public/8/master.svg?logo=azure-pipelines&style=flat-square&label=build)](https://dev.azure.com/explosion-ai/public/_build?definitionId=8)
|
[![Azure Pipelines](https://img.shields.io/azure-devops/build/explosion-ai/public/8/master.svg?logo=azure-pipelines&style=flat-square&label=build)](https://dev.azure.com/explosion-ai/public/_build?definitionId=8)
|
||||||
|
|
|
@ -1,6 +1,8 @@
|
||||||
# build version constraints for use with wheelwright + multibuild
|
# build version constraints for use with wheelwright + multibuild
|
||||||
numpy==1.15.0; python_version<='3.7'
|
numpy==1.15.0; python_version<='3.7' and platform_machine!='aarch64'
|
||||||
numpy==1.17.3; python_version=='3.8'
|
numpy==1.19.2; python_version<='3.7' and platform_machine=='aarch64'
|
||||||
|
numpy==1.17.3; python_version=='3.8' and platform_machine!='aarch64'
|
||||||
|
numpy==1.19.2; python_version=='3.8' and platform_machine=='aarch64'
|
||||||
numpy==1.19.3; python_version=='3.9'
|
numpy==1.19.3; python_version=='3.9'
|
||||||
numpy==1.21.3; python_version=='3.10'
|
numpy==1.21.3; python_version=='3.10'
|
||||||
numpy; python_version>='3.11'
|
numpy; python_version>='3.11'
|
||||||
|
|
|
@ -455,6 +455,10 @@ Regression tests are tests that refer to bugs reported in specific issues. They
|
||||||
|
|
||||||
The test suite also provides [fixtures](https://github.com/explosion/spaCy/blob/master/spacy/tests/conftest.py) for different language tokenizers that can be used as function arguments of the same name and will be passed in automatically. Those should only be used for tests related to those specific languages. We also have [test utility functions](https://github.com/explosion/spaCy/blob/master/spacy/tests/util.py) for common operations, like creating a temporary file.
|
The test suite also provides [fixtures](https://github.com/explosion/spaCy/blob/master/spacy/tests/conftest.py) for different language tokenizers that can be used as function arguments of the same name and will be passed in automatically. Those should only be used for tests related to those specific languages. We also have [test utility functions](https://github.com/explosion/spaCy/blob/master/spacy/tests/util.py) for common operations, like creating a temporary file.
|
||||||
|
|
||||||
|
### Testing Cython Code
|
||||||
|
|
||||||
|
If you're developing Cython code (`.pyx` files), those extensions will need to be built before the test runner can test that code - otherwise it's going to run the tests with stale code from the last time the extension was built. You can build the extensions locally with `python setup.py build_ext -i`.
|
||||||
|
|
||||||
### Constructing objects and state
|
### Constructing objects and state
|
||||||
|
|
||||||
Test functions usually follow the same simple structure: they set up some state, perform the operation you want to test and `assert` conditions that you expect to be true, usually before and after the operation.
|
Test functions usually follow the same simple structure: they set up some state, perform the operation you want to test and `assert` conditions that you expect to be true, usually before and after the operation.
|
||||||
|
|
|
@ -16,21 +16,41 @@ To summon the robot, write a github comment on the issue/PR you wish to test. Th
|
||||||
|
|
||||||
Some things to note:
|
Some things to note:
|
||||||
|
|
||||||
* The `@explosion-bot please` must be the beginning of the command - you cannot add anything in front of this or else the robot won't know how to parse it. Adding anything at the end aside from the test name will also confuse the robot, so keep it simple!
|
- The `@explosion-bot please` must be the beginning of the command - you cannot add anything in front of this or else the robot won't know how to parse it. Adding anything at the end aside from the test name will also confuse the robot, so keep it simple!
|
||||||
* The command name (such as `test_gpu`) must be one of the tests that the bot knows how to run. The available commands are documented in the bot's [workflow config](https://github.com/explosion/spaCy/blob/master/.github/workflows/explosionbot.yml#L26) and must match exactly one of the commands listed there.
|
- The command name (such as `test_gpu`) must be one of the tests that the bot knows how to run. The available commands are documented in the bot's [workflow config](https://github.com/explosion/spaCy/blob/master/.github/workflows/explosionbot.yml#L26) and must match exactly one of the commands listed there.
|
||||||
* The robot can't do multiple things at once, so if you want it to run multiple tests, you'll have to summon it with one comment per test.
|
- The robot can't do multiple things at once, so if you want it to run multiple tests, you'll have to summon it with one comment per test.
|
||||||
* For the `test_gpu` command, you can specify an optional thinc branch (from the spaCy repo) or a spaCy branch (from the thinc repo) with either the `--thinc-branch` or `--spacy-branch` flags. By default, the bot will pull in the PR branch from the repo where the command was issued, and the main branch of the other repository. However, if you need to run against another branch, you can say (for example):
|
|
||||||
|
|
||||||
```
|
### Examples
|
||||||
@explosion-bot please test_gpu --thinc-branch develop
|
|
||||||
```
|
- Execute spaCy slow GPU tests with a custom thinc branch from a spaCy PR:
|
||||||
You can also specify a branch from an unmerged PR:
|
|
||||||
```
|
```
|
||||||
@explosion-bot please test_gpu --thinc-branch refs/pull/633/head
|
@explosion-bot please test_slow_gpu --thinc-branch <branch_name>
|
||||||
```
|
```
|
||||||
|
|
||||||
|
`branch_name` can either be a named branch, e.g: `develop`, or an unmerged PR, e.g: `refs/pull/<pr_number>/head`.
|
||||||
|
|
||||||
|
- Execute spaCy Transformers GPU tests from a spaCy PR:
|
||||||
|
|
||||||
|
```
|
||||||
|
@explosion-bot please test_gpu --run-on spacy-transformers --run-on-branch master --spacy-branch current_pr
|
||||||
|
```
|
||||||
|
|
||||||
|
This will launch the GPU pipeline for the `spacy-transformers` repo on its `master` branch, using the current spaCy PR's branch to build spaCy. The name of the repository passed to `--run-on` is case-sensitive, e.g: use `spaCy` instead of `spacy`.
|
||||||
|
|
||||||
|
- General info about supported commands.
|
||||||
|
|
||||||
|
```
|
||||||
|
@explosion-bot please info
|
||||||
|
```
|
||||||
|
|
||||||
|
- Help text for a specific command
|
||||||
|
```
|
||||||
|
@explosion-bot please <command> --help
|
||||||
|
```
|
||||||
|
|
||||||
## Troubleshooting
|
## Troubleshooting
|
||||||
|
|
||||||
If the robot isn't responding to commands as expected, you can check its logs in the [Github Action](https://github.com/explosion/spaCy/actions/workflows/explosionbot.yml).
|
If the robot isn't responding to commands as expected, you can check its logs in the [Github Action](https://github.com/explosion/spaCy/actions/workflows/explosionbot.yml).
|
||||||
|
|
||||||
For each command sent to the bot, there should be a run of the `explosion-bot` workflow. In the `Install and run explosion-bot` step, towards the ends of the logs you should see info about the configuration that the bot was run with, as well as any errors that the bot encountered.
|
For each command sent to the bot, there should be a run of the `explosion-bot` workflow. In the `Install and run explosion-bot` step, towards the ends of the logs you should see info about the configuration that the bot was run with, as well as any errors that the bot encountered.
|
||||||
|
|
|
@ -5,8 +5,7 @@ requires = [
|
||||||
"cymem>=2.0.2,<2.1.0",
|
"cymem>=2.0.2,<2.1.0",
|
||||||
"preshed>=3.0.2,<3.1.0",
|
"preshed>=3.0.2,<3.1.0",
|
||||||
"murmurhash>=0.28.0,<1.1.0",
|
"murmurhash>=0.28.0,<1.1.0",
|
||||||
"thinc>=8.0.14,<8.1.0",
|
"thinc>=8.1.0,<8.2.0",
|
||||||
"blis>=0.4.0,<0.8.0",
|
|
||||||
"pathy",
|
"pathy",
|
||||||
"numpy>=1.15.0",
|
"numpy>=1.15.0",
|
||||||
]
|
]
|
||||||
|
|
|
@ -3,8 +3,7 @@ spacy-legacy>=3.0.9,<3.1.0
|
||||||
spacy-loggers>=1.0.0,<2.0.0
|
spacy-loggers>=1.0.0,<2.0.0
|
||||||
cymem>=2.0.2,<2.1.0
|
cymem>=2.0.2,<2.1.0
|
||||||
preshed>=3.0.2,<3.1.0
|
preshed>=3.0.2,<3.1.0
|
||||||
thinc>=8.0.14,<8.1.0
|
thinc>=8.1.0,<8.2.0
|
||||||
blis>=0.4.0,<0.8.0
|
|
||||||
ml_datasets>=0.2.0,<0.3.0
|
ml_datasets>=0.2.0,<0.3.0
|
||||||
murmurhash>=0.28.0,<1.1.0
|
murmurhash>=0.28.0,<1.1.0
|
||||||
wasabi>=0.9.1,<1.1.0
|
wasabi>=0.9.1,<1.1.0
|
||||||
|
@ -16,13 +15,13 @@ pathy>=0.3.5
|
||||||
numpy>=1.15.0
|
numpy>=1.15.0
|
||||||
requests>=2.13.0,<3.0.0
|
requests>=2.13.0,<3.0.0
|
||||||
tqdm>=4.38.0,<5.0.0
|
tqdm>=4.38.0,<5.0.0
|
||||||
pydantic>=1.7.4,!=1.8,!=1.8.1,<1.9.0
|
pydantic>=1.7.4,!=1.8,!=1.8.1,<1.10.0
|
||||||
jinja2
|
jinja2
|
||||||
langcodes>=3.2.0,<4.0.0
|
langcodes>=3.2.0,<4.0.0
|
||||||
# Official Python utilities
|
# Official Python utilities
|
||||||
setuptools
|
setuptools
|
||||||
packaging>=20.0
|
packaging>=20.0
|
||||||
typing_extensions>=3.7.4.1,<4.0.0.0; python_version < "3.8"
|
typing_extensions>=3.7.4.1,<4.2.0; python_version < "3.8"
|
||||||
# Development dependencies
|
# Development dependencies
|
||||||
pre-commit>=2.13.0
|
pre-commit>=2.13.0
|
||||||
cython>=0.25,<3.0
|
cython>=0.25,<3.0
|
||||||
|
@ -31,7 +30,7 @@ pytest-timeout>=1.3.0,<2.0.0
|
||||||
mock>=2.0.0,<3.0.0
|
mock>=2.0.0,<3.0.0
|
||||||
flake8>=3.8.0,<3.10.0
|
flake8>=3.8.0,<3.10.0
|
||||||
hypothesis>=3.27.0,<7.0.0
|
hypothesis>=3.27.0,<7.0.0
|
||||||
mypy==0.910
|
mypy>=0.910,<0.970; platform_machine!='aarch64'
|
||||||
types-dataclasses>=0.1.3; python_version < "3.7"
|
types-dataclasses>=0.1.3; python_version < "3.7"
|
||||||
types-mock>=0.1.1
|
types-mock>=0.1.1
|
||||||
types-requests
|
types-requests
|
||||||
|
|
17
setup.cfg
17
setup.cfg
|
@ -38,7 +38,7 @@ setup_requires =
|
||||||
cymem>=2.0.2,<2.1.0
|
cymem>=2.0.2,<2.1.0
|
||||||
preshed>=3.0.2,<3.1.0
|
preshed>=3.0.2,<3.1.0
|
||||||
murmurhash>=0.28.0,<1.1.0
|
murmurhash>=0.28.0,<1.1.0
|
||||||
thinc>=8.0.14,<8.1.0
|
thinc>=8.1.0,<8.2.0
|
||||||
install_requires =
|
install_requires =
|
||||||
# Our libraries
|
# Our libraries
|
||||||
spacy-legacy>=3.0.9,<3.1.0
|
spacy-legacy>=3.0.9,<3.1.0
|
||||||
|
@ -46,8 +46,7 @@ install_requires =
|
||||||
murmurhash>=0.28.0,<1.1.0
|
murmurhash>=0.28.0,<1.1.0
|
||||||
cymem>=2.0.2,<2.1.0
|
cymem>=2.0.2,<2.1.0
|
||||||
preshed>=3.0.2,<3.1.0
|
preshed>=3.0.2,<3.1.0
|
||||||
thinc>=8.0.14,<8.1.0
|
thinc>=8.1.0,<8.2.0
|
||||||
blis>=0.4.0,<0.8.0
|
|
||||||
wasabi>=0.9.1,<1.1.0
|
wasabi>=0.9.1,<1.1.0
|
||||||
srsly>=2.4.3,<3.0.0
|
srsly>=2.4.3,<3.0.0
|
||||||
catalogue>=2.0.6,<2.1.0
|
catalogue>=2.0.6,<2.1.0
|
||||||
|
@ -57,12 +56,12 @@ install_requires =
|
||||||
tqdm>=4.38.0,<5.0.0
|
tqdm>=4.38.0,<5.0.0
|
||||||
numpy>=1.15.0
|
numpy>=1.15.0
|
||||||
requests>=2.13.0,<3.0.0
|
requests>=2.13.0,<3.0.0
|
||||||
pydantic>=1.7.4,!=1.8,!=1.8.1,<1.9.0
|
pydantic>=1.7.4,!=1.8,!=1.8.1,<1.10.0
|
||||||
jinja2
|
jinja2
|
||||||
# Official Python utilities
|
# Official Python utilities
|
||||||
setuptools
|
setuptools
|
||||||
packaging>=20.0
|
packaging>=20.0
|
||||||
typing_extensions>=3.7.4,<4.0.0.0; python_version < "3.8"
|
typing_extensions>=3.7.4,<4.2.0; python_version < "3.8"
|
||||||
langcodes>=3.2.0,<4.0.0
|
langcodes>=3.2.0,<4.0.0
|
||||||
|
|
||||||
[options.entry_points]
|
[options.entry_points]
|
||||||
|
@ -104,14 +103,18 @@ cuda114 =
|
||||||
cupy-cuda114>=5.0.0b4,<11.0.0
|
cupy-cuda114>=5.0.0b4,<11.0.0
|
||||||
cuda115 =
|
cuda115 =
|
||||||
cupy-cuda115>=5.0.0b4,<11.0.0
|
cupy-cuda115>=5.0.0b4,<11.0.0
|
||||||
|
cuda116 =
|
||||||
|
cupy-cuda116>=5.0.0b4,<11.0.0
|
||||||
|
cuda117 =
|
||||||
|
cupy-cuda117>=5.0.0b4,<11.0.0
|
||||||
apple =
|
apple =
|
||||||
thinc-apple-ops>=0.0.4,<1.0.0
|
thinc-apple-ops>=0.1.0.dev0,<1.0.0
|
||||||
# Language tokenizers with external dependencies
|
# Language tokenizers with external dependencies
|
||||||
ja =
|
ja =
|
||||||
sudachipy>=0.5.2,!=0.6.1
|
sudachipy>=0.5.2,!=0.6.1
|
||||||
sudachidict_core>=20211220
|
sudachidict_core>=20211220
|
||||||
ko =
|
ko =
|
||||||
natto-py==0.9.0
|
natto-py>=0.9.0
|
||||||
th =
|
th =
|
||||||
pythainlp>=2.0
|
pythainlp>=2.0
|
||||||
|
|
||||||
|
|
8
setup.py
8
setup.py
|
@ -126,6 +126,8 @@ class build_ext_options:
|
||||||
|
|
||||||
class build_ext_subclass(build_ext, build_ext_options):
|
class build_ext_subclass(build_ext, build_ext_options):
|
||||||
def build_extensions(self):
|
def build_extensions(self):
|
||||||
|
if self.parallel is None and os.environ.get("SPACY_NUM_BUILD_JOBS") is not None:
|
||||||
|
self.parallel = int(os.environ.get("SPACY_NUM_BUILD_JOBS"))
|
||||||
build_ext_options.build_options(self)
|
build_ext_options.build_options(self)
|
||||||
build_ext.build_extensions(self)
|
build_ext.build_extensions(self)
|
||||||
|
|
||||||
|
@ -206,7 +208,11 @@ def setup_package():
|
||||||
for name in MOD_NAMES:
|
for name in MOD_NAMES:
|
||||||
mod_path = name.replace(".", "/") + ".pyx"
|
mod_path = name.replace(".", "/") + ".pyx"
|
||||||
ext = Extension(
|
ext = Extension(
|
||||||
name, [mod_path], language="c++", include_dirs=include_dirs, extra_compile_args=["-std=c++11"]
|
name,
|
||||||
|
[mod_path],
|
||||||
|
language="c++",
|
||||||
|
include_dirs=include_dirs,
|
||||||
|
extra_compile_args=["-std=c++11"],
|
||||||
)
|
)
|
||||||
ext_modules.append(ext)
|
ext_modules.append(ext)
|
||||||
print("Cythonizing sources")
|
print("Cythonizing sources")
|
||||||
|
|
|
@ -32,6 +32,7 @@ def load(
|
||||||
*,
|
*,
|
||||||
vocab: Union[Vocab, bool] = True,
|
vocab: Union[Vocab, bool] = True,
|
||||||
disable: Iterable[str] = util.SimpleFrozenList(),
|
disable: Iterable[str] = util.SimpleFrozenList(),
|
||||||
|
enable: Iterable[str] = util.SimpleFrozenList(),
|
||||||
exclude: Iterable[str] = util.SimpleFrozenList(),
|
exclude: Iterable[str] = util.SimpleFrozenList(),
|
||||||
config: Union[Dict[str, Any], Config] = util.SimpleFrozenDict(),
|
config: Union[Dict[str, Any], Config] = util.SimpleFrozenDict(),
|
||||||
) -> Language:
|
) -> Language:
|
||||||
|
@ -42,6 +43,8 @@ def load(
|
||||||
disable (Iterable[str]): Names of pipeline components to disable. Disabled
|
disable (Iterable[str]): Names of pipeline components to disable. Disabled
|
||||||
pipes will be loaded but they won't be run unless you explicitly
|
pipes will be loaded but they won't be run unless you explicitly
|
||||||
enable them by calling nlp.enable_pipe.
|
enable them by calling nlp.enable_pipe.
|
||||||
|
enable (Iterable[str]): Names of pipeline components to enable. All other
|
||||||
|
pipes will be disabled (but can be enabled later using nlp.enable_pipe).
|
||||||
exclude (Iterable[str]): Names of pipeline components to exclude. Excluded
|
exclude (Iterable[str]): Names of pipeline components to exclude. Excluded
|
||||||
components won't be loaded.
|
components won't be loaded.
|
||||||
config (Dict[str, Any] / Config): Config overrides as nested dict or dict
|
config (Dict[str, Any] / Config): Config overrides as nested dict or dict
|
||||||
|
@ -49,7 +52,12 @@ def load(
|
||||||
RETURNS (Language): The loaded nlp object.
|
RETURNS (Language): The loaded nlp object.
|
||||||
"""
|
"""
|
||||||
return util.load_model(
|
return util.load_model(
|
||||||
name, vocab=vocab, disable=disable, exclude=exclude, config=config
|
name,
|
||||||
|
vocab=vocab,
|
||||||
|
disable=disable,
|
||||||
|
enable=enable,
|
||||||
|
exclude=exclude,
|
||||||
|
config=config,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -1,6 +1,6 @@
|
||||||
# fmt: off
|
# fmt: off
|
||||||
__title__ = "spacy"
|
__title__ = "spacy"
|
||||||
__version__ = "3.3.0"
|
__version__ = "3.4.1"
|
||||||
__download_url__ = "https://github.com/explosion/spacy-models/releases/download"
|
__download_url__ = "https://github.com/explosion/spacy-models/releases/download"
|
||||||
__compatibility__ = "https://raw.githubusercontent.com/explosion/spacy-models/master/compatibility.json"
|
__compatibility__ = "https://raw.githubusercontent.com/explosion/spacy-models/master/compatibility.json"
|
||||||
__projects__ = "https://github.com/explosion/projects"
|
__projects__ = "https://github.com/explosion/projects"
|
||||||
|
|
|
@ -12,7 +12,7 @@ from click.parser import split_arg_string
|
||||||
from typer.main import get_command
|
from typer.main import get_command
|
||||||
from contextlib import contextmanager
|
from contextlib import contextmanager
|
||||||
from thinc.api import Config, ConfigValidationError, require_gpu
|
from thinc.api import Config, ConfigValidationError, require_gpu
|
||||||
from thinc.util import has_cupy, gpu_is_available
|
from thinc.util import gpu_is_available
|
||||||
from configparser import InterpolationError
|
from configparser import InterpolationError
|
||||||
import os
|
import os
|
||||||
|
|
||||||
|
@ -462,6 +462,23 @@ def git_sparse_checkout(repo, subpath, dest, branch):
|
||||||
shutil.move(str(source_path), str(dest))
|
shutil.move(str(source_path), str(dest))
|
||||||
|
|
||||||
|
|
||||||
|
def git_repo_branch_exists(repo: str, branch: str) -> bool:
|
||||||
|
"""Uses 'git ls-remote' to check if a repository and branch exists
|
||||||
|
|
||||||
|
repo (str): URL to get repo.
|
||||||
|
branch (str): Branch on repo to check.
|
||||||
|
RETURNS (bool): True if repo:branch exists.
|
||||||
|
"""
|
||||||
|
get_git_version()
|
||||||
|
cmd = f"git ls-remote {repo} {branch}"
|
||||||
|
# We might be tempted to use `--exit-code` with `git ls-remote`, but
|
||||||
|
# `run_command` handles the `returncode` for us, so we'll rely on
|
||||||
|
# the fact that stdout returns '' if the requested branch doesn't exist
|
||||||
|
ret = run_command(cmd, capture=True)
|
||||||
|
exists = ret.stdout != ""
|
||||||
|
return exists
|
||||||
|
|
||||||
|
|
||||||
def get_git_version(
|
def get_git_version(
|
||||||
error: str = "Could not run 'git'. Make sure it's installed and the executable is available.",
|
error: str = "Could not run 'git'. Make sure it's installed and the executable is available.",
|
||||||
) -> Tuple[int, int]:
|
) -> Tuple[int, int]:
|
||||||
|
@ -554,5 +571,5 @@ def setup_gpu(use_gpu: int, silent=None) -> None:
|
||||||
require_gpu(use_gpu)
|
require_gpu(use_gpu)
|
||||||
else:
|
else:
|
||||||
local_msg.info("Using CPU")
|
local_msg.info("Using CPU")
|
||||||
if has_cupy and gpu_is_available():
|
if gpu_is_available():
|
||||||
local_msg.info("To switch to GPU 0, use the option: --gpu-id 0")
|
local_msg.info("To switch to GPU 0, use the option: --gpu-id 0")
|
||||||
|
|
|
@ -6,10 +6,11 @@ import sys
|
||||||
import srsly
|
import srsly
|
||||||
from wasabi import Printer, MESSAGES, msg
|
from wasabi import Printer, MESSAGES, msg
|
||||||
import typer
|
import typer
|
||||||
|
import math
|
||||||
|
|
||||||
from ._util import app, Arg, Opt, show_validation_error, parse_config_overrides
|
from ._util import app, Arg, Opt, show_validation_error, parse_config_overrides
|
||||||
from ._util import import_code, debug_cli
|
from ._util import import_code, debug_cli
|
||||||
from ..training import Example
|
from ..training import Example, remove_bilu_prefix
|
||||||
from ..training.initialize import get_sourced_components
|
from ..training.initialize import get_sourced_components
|
||||||
from ..schemas import ConfigSchemaTraining
|
from ..schemas import ConfigSchemaTraining
|
||||||
from ..pipeline._parser_internals import nonproj
|
from ..pipeline._parser_internals import nonproj
|
||||||
|
@ -30,6 +31,12 @@ DEP_LABEL_THRESHOLD = 20
|
||||||
# Minimum number of expected examples to train a new pipeline
|
# Minimum number of expected examples to train a new pipeline
|
||||||
BLANK_MODEL_MIN_THRESHOLD = 100
|
BLANK_MODEL_MIN_THRESHOLD = 100
|
||||||
BLANK_MODEL_THRESHOLD = 2000
|
BLANK_MODEL_THRESHOLD = 2000
|
||||||
|
# Arbitrary threshold where SpanCat performs well
|
||||||
|
SPAN_DISTINCT_THRESHOLD = 1
|
||||||
|
# Arbitrary threshold where SpanCat performs well
|
||||||
|
BOUNDARY_DISTINCT_THRESHOLD = 1
|
||||||
|
# Arbitrary threshold for filtering span lengths during reporting (percentage)
|
||||||
|
SPAN_LENGTH_THRESHOLD_PERCENTAGE = 90
|
||||||
|
|
||||||
|
|
||||||
@debug_cli.command(
|
@debug_cli.command(
|
||||||
|
@ -247,6 +254,69 @@ def debug_data(
|
||||||
msg.warn(f"No examples for texts WITHOUT new label '{label}'")
|
msg.warn(f"No examples for texts WITHOUT new label '{label}'")
|
||||||
has_no_neg_warning = True
|
has_no_neg_warning = True
|
||||||
|
|
||||||
|
with msg.loading("Obtaining span characteristics..."):
|
||||||
|
span_characteristics = _get_span_characteristics(
|
||||||
|
train_dataset, gold_train_data, spans_key
|
||||||
|
)
|
||||||
|
|
||||||
|
msg.info(f"Span characteristics for spans_key '{spans_key}'")
|
||||||
|
msg.info("SD = Span Distinctiveness, BD = Boundary Distinctiveness")
|
||||||
|
_print_span_characteristics(span_characteristics)
|
||||||
|
|
||||||
|
_span_freqs = _get_spans_length_freq_dist(
|
||||||
|
gold_train_data["spans_length"][spans_key]
|
||||||
|
)
|
||||||
|
_filtered_span_freqs = _filter_spans_length_freq_dist(
|
||||||
|
_span_freqs, threshold=SPAN_LENGTH_THRESHOLD_PERCENTAGE
|
||||||
|
)
|
||||||
|
|
||||||
|
msg.info(
|
||||||
|
f"Over {SPAN_LENGTH_THRESHOLD_PERCENTAGE}% of spans have lengths of 1 -- "
|
||||||
|
f"{max(_filtered_span_freqs.keys())} "
|
||||||
|
f"(min={span_characteristics['min_length']}, max={span_characteristics['max_length']}). "
|
||||||
|
f"The most common span lengths are: {_format_freqs(_filtered_span_freqs)}. "
|
||||||
|
"If you are using the n-gram suggester, note that omitting "
|
||||||
|
"infrequent n-gram lengths can greatly improve speed and "
|
||||||
|
"memory usage."
|
||||||
|
)
|
||||||
|
|
||||||
|
msg.text(
|
||||||
|
f"Full distribution of span lengths: {_format_freqs(_span_freqs)}",
|
||||||
|
show=verbose,
|
||||||
|
)
|
||||||
|
|
||||||
|
# Add report regarding span characteristics
|
||||||
|
if span_characteristics["avg_sd"] < SPAN_DISTINCT_THRESHOLD:
|
||||||
|
msg.warn("Spans may not be distinct from the rest of the corpus")
|
||||||
|
else:
|
||||||
|
msg.good("Spans are distinct from the rest of the corpus")
|
||||||
|
|
||||||
|
p_spans = span_characteristics["p_spans"].values()
|
||||||
|
all_span_tokens: Counter = sum(p_spans, Counter())
|
||||||
|
most_common_spans = [w for w, _ in all_span_tokens.most_common(10)]
|
||||||
|
msg.text(
|
||||||
|
"10 most common span tokens: {}".format(
|
||||||
|
_format_labels(most_common_spans)
|
||||||
|
),
|
||||||
|
show=verbose,
|
||||||
|
)
|
||||||
|
|
||||||
|
# Add report regarding span boundary characteristics
|
||||||
|
if span_characteristics["avg_bd"] < BOUNDARY_DISTINCT_THRESHOLD:
|
||||||
|
msg.warn("Boundary tokens are not distinct from the rest of the corpus")
|
||||||
|
else:
|
||||||
|
msg.good("Boundary tokens are distinct from the rest of the corpus")
|
||||||
|
|
||||||
|
p_bounds = span_characteristics["p_bounds"].values()
|
||||||
|
all_span_bound_tokens: Counter = sum(p_bounds, Counter())
|
||||||
|
most_common_bounds = [w for w, _ in all_span_bound_tokens.most_common(10)]
|
||||||
|
msg.text(
|
||||||
|
"10 most common span boundary tokens: {}".format(
|
||||||
|
_format_labels(most_common_bounds)
|
||||||
|
),
|
||||||
|
show=verbose,
|
||||||
|
)
|
||||||
|
|
||||||
if has_low_data_warning:
|
if has_low_data_warning:
|
||||||
msg.text(
|
msg.text(
|
||||||
f"To train a new span type, your data should include at "
|
f"To train a new span type, your data should include at "
|
||||||
|
@ -291,7 +361,7 @@ def debug_data(
|
||||||
if label != "-"
|
if label != "-"
|
||||||
]
|
]
|
||||||
labels_with_counts = _format_labels(labels_with_counts, counts=True)
|
labels_with_counts = _format_labels(labels_with_counts, counts=True)
|
||||||
msg.text(f"Labels in train data: {_format_labels(labels)}", show=verbose)
|
msg.text(f"Labels in train data: {labels_with_counts}", show=verbose)
|
||||||
missing_labels = model_labels - labels
|
missing_labels = model_labels - labels
|
||||||
if missing_labels:
|
if missing_labels:
|
||||||
msg.warn(
|
msg.warn(
|
||||||
|
@ -647,6 +717,9 @@ def _compile_gold(
|
||||||
"words": Counter(),
|
"words": Counter(),
|
||||||
"roots": Counter(),
|
"roots": Counter(),
|
||||||
"spancat": dict(),
|
"spancat": dict(),
|
||||||
|
"spans_length": dict(),
|
||||||
|
"spans_per_type": dict(),
|
||||||
|
"sb_per_type": dict(),
|
||||||
"ws_ents": 0,
|
"ws_ents": 0,
|
||||||
"boundary_cross_ents": 0,
|
"boundary_cross_ents": 0,
|
||||||
"n_words": 0,
|
"n_words": 0,
|
||||||
|
@ -685,21 +758,66 @@ def _compile_gold(
|
||||||
# "Illegal" whitespace entity
|
# "Illegal" whitespace entity
|
||||||
data["ws_ents"] += 1
|
data["ws_ents"] += 1
|
||||||
if label.startswith(("B-", "U-")):
|
if label.startswith(("B-", "U-")):
|
||||||
combined_label = label.split("-")[1]
|
combined_label = remove_bilu_prefix(label)
|
||||||
data["ner"][combined_label] += 1
|
data["ner"][combined_label] += 1
|
||||||
if sent_starts[i] == True and label.startswith(("I-", "L-")):
|
if sent_starts[i] and label.startswith(("I-", "L-")):
|
||||||
data["boundary_cross_ents"] += 1
|
data["boundary_cross_ents"] += 1
|
||||||
elif label == "-":
|
elif label == "-":
|
||||||
data["ner"]["-"] += 1
|
data["ner"]["-"] += 1
|
||||||
if "spancat" in factory_names:
|
if "spancat" in factory_names:
|
||||||
for span_key in list(eg.reference.spans.keys()):
|
for spans_key in list(eg.reference.spans.keys()):
|
||||||
if span_key not in data["spancat"]:
|
# Obtain the span frequency
|
||||||
data["spancat"][span_key] = Counter()
|
if spans_key not in data["spancat"]:
|
||||||
for i, span in enumerate(eg.reference.spans[span_key]):
|
data["spancat"][spans_key] = Counter()
|
||||||
|
for i, span in enumerate(eg.reference.spans[spans_key]):
|
||||||
if span.label_ is None:
|
if span.label_ is None:
|
||||||
continue
|
continue
|
||||||
else:
|
else:
|
||||||
data["spancat"][span_key][span.label_] += 1
|
data["spancat"][spans_key][span.label_] += 1
|
||||||
|
|
||||||
|
# Obtain the span length
|
||||||
|
if spans_key not in data["spans_length"]:
|
||||||
|
data["spans_length"][spans_key] = dict()
|
||||||
|
for span in gold.spans[spans_key]:
|
||||||
|
if span.label_ is None:
|
||||||
|
continue
|
||||||
|
if span.label_ not in data["spans_length"][spans_key]:
|
||||||
|
data["spans_length"][spans_key][span.label_] = []
|
||||||
|
data["spans_length"][spans_key][span.label_].append(len(span))
|
||||||
|
|
||||||
|
# Obtain spans per span type
|
||||||
|
if spans_key not in data["spans_per_type"]:
|
||||||
|
data["spans_per_type"][spans_key] = dict()
|
||||||
|
for span in gold.spans[spans_key]:
|
||||||
|
if span.label_ not in data["spans_per_type"][spans_key]:
|
||||||
|
data["spans_per_type"][spans_key][span.label_] = []
|
||||||
|
data["spans_per_type"][spans_key][span.label_].append(span)
|
||||||
|
|
||||||
|
# Obtain boundary tokens per span type
|
||||||
|
window_size = 1
|
||||||
|
if spans_key not in data["sb_per_type"]:
|
||||||
|
data["sb_per_type"][spans_key] = dict()
|
||||||
|
for span in gold.spans[spans_key]:
|
||||||
|
if span.label_ not in data["sb_per_type"][spans_key]:
|
||||||
|
# Creating a data structure that holds the start and
|
||||||
|
# end tokens for each span type
|
||||||
|
data["sb_per_type"][spans_key][span.label_] = {
|
||||||
|
"start": [],
|
||||||
|
"end": [],
|
||||||
|
}
|
||||||
|
for offset in range(window_size):
|
||||||
|
sb_start_idx = span.start - (offset + 1)
|
||||||
|
if sb_start_idx >= 0:
|
||||||
|
data["sb_per_type"][spans_key][span.label_]["start"].append(
|
||||||
|
gold[sb_start_idx : sb_start_idx + 1]
|
||||||
|
)
|
||||||
|
|
||||||
|
sb_end_idx = span.end + (offset + 1)
|
||||||
|
if sb_end_idx <= len(gold):
|
||||||
|
data["sb_per_type"][spans_key][span.label_]["end"].append(
|
||||||
|
gold[sb_end_idx - 1 : sb_end_idx]
|
||||||
|
)
|
||||||
|
|
||||||
if "textcat" in factory_names or "textcat_multilabel" in factory_names:
|
if "textcat" in factory_names or "textcat_multilabel" in factory_names:
|
||||||
data["cats"].update(gold.cats)
|
data["cats"].update(gold.cats)
|
||||||
if any(val not in (0, 1) for val in gold.cats.values()):
|
if any(val not in (0, 1) for val in gold.cats.values()):
|
||||||
|
@ -770,6 +888,16 @@ def _format_labels(
|
||||||
return ", ".join([f"'{l}'" for l in cast(Iterable[str], labels)])
|
return ", ".join([f"'{l}'" for l in cast(Iterable[str], labels)])
|
||||||
|
|
||||||
|
|
||||||
|
def _format_freqs(freqs: Dict[int, float], sort: bool = True) -> str:
|
||||||
|
if sort:
|
||||||
|
freqs = dict(sorted(freqs.items()))
|
||||||
|
|
||||||
|
_freqs = [(str(k), v) for k, v in freqs.items()]
|
||||||
|
return ", ".join(
|
||||||
|
[f"{l} ({c}%)" for l, c in cast(Iterable[Tuple[str, float]], _freqs)]
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def _get_examples_without_label(
|
def _get_examples_without_label(
|
||||||
data: Sequence[Example],
|
data: Sequence[Example],
|
||||||
label: str,
|
label: str,
|
||||||
|
@ -780,7 +908,7 @@ def _get_examples_without_label(
|
||||||
for eg in data:
|
for eg in data:
|
||||||
if component == "ner":
|
if component == "ner":
|
||||||
labels = [
|
labels = [
|
||||||
label.split("-")[1]
|
remove_bilu_prefix(label)
|
||||||
for label in eg.get_aligned_ner()
|
for label in eg.get_aligned_ner()
|
||||||
if label not in ("O", "-", None)
|
if label not in ("O", "-", None)
|
||||||
]
|
]
|
||||||
|
@ -824,3 +952,158 @@ def _get_labels_from_spancat(nlp: Language) -> Dict[str, Set[str]]:
|
||||||
labels[pipe.key] = set()
|
labels[pipe.key] = set()
|
||||||
labels[pipe.key].update(pipe.labels)
|
labels[pipe.key].update(pipe.labels)
|
||||||
return labels
|
return labels
|
||||||
|
|
||||||
|
|
||||||
|
def _gmean(l: List) -> float:
|
||||||
|
"""Compute geometric mean of a list"""
|
||||||
|
return math.exp(math.fsum(math.log(i) for i in l) / len(l))
|
||||||
|
|
||||||
|
|
||||||
|
def _wgt_average(metric: Dict[str, float], frequencies: Counter) -> float:
|
||||||
|
total = sum(value * frequencies[span_type] for span_type, value in metric.items())
|
||||||
|
return total / sum(frequencies.values())
|
||||||
|
|
||||||
|
|
||||||
|
def _get_distribution(docs, normalize: bool = True) -> Counter:
|
||||||
|
"""Get the frequency distribution given a set of Docs"""
|
||||||
|
word_counts: Counter = Counter()
|
||||||
|
for doc in docs:
|
||||||
|
for token in doc:
|
||||||
|
# Normalize the text
|
||||||
|
t = token.text.lower().replace("``", '"').replace("''", '"')
|
||||||
|
word_counts[t] += 1
|
||||||
|
if normalize:
|
||||||
|
total = sum(word_counts.values(), 0.0)
|
||||||
|
word_counts = Counter({k: v / total for k, v in word_counts.items()})
|
||||||
|
return word_counts
|
||||||
|
|
||||||
|
|
||||||
|
def _get_kl_divergence(p: Counter, q: Counter) -> float:
|
||||||
|
"""Compute the Kullback-Leibler divergence from two frequency distributions"""
|
||||||
|
total = 0.0
|
||||||
|
for word, p_word in p.items():
|
||||||
|
total += p_word * math.log(p_word / q[word])
|
||||||
|
return total
|
||||||
|
|
||||||
|
|
||||||
|
def _format_span_row(span_data: List[Dict], labels: List[str]) -> List[Any]:
|
||||||
|
"""Compile into one list for easier reporting"""
|
||||||
|
d = {
|
||||||
|
label: [label] + list(round(d[label], 2) for d in span_data) for label in labels
|
||||||
|
}
|
||||||
|
return list(d.values())
|
||||||
|
|
||||||
|
|
||||||
|
def _get_span_characteristics(
|
||||||
|
examples: List[Example], compiled_gold: Dict[str, Any], spans_key: str
|
||||||
|
) -> Dict[str, Any]:
|
||||||
|
"""Obtain all span characteristics"""
|
||||||
|
data_labels = compiled_gold["spancat"][spans_key]
|
||||||
|
# Get lengths
|
||||||
|
span_length = {
|
||||||
|
label: _gmean(l)
|
||||||
|
for label, l in compiled_gold["spans_length"][spans_key].items()
|
||||||
|
}
|
||||||
|
min_lengths = [min(l) for l in compiled_gold["spans_length"][spans_key].values()]
|
||||||
|
max_lengths = [max(l) for l in compiled_gold["spans_length"][spans_key].values()]
|
||||||
|
|
||||||
|
# Get relevant distributions: corpus, spans, span boundaries
|
||||||
|
p_corpus = _get_distribution([eg.reference for eg in examples], normalize=True)
|
||||||
|
p_spans = {
|
||||||
|
label: _get_distribution(spans, normalize=True)
|
||||||
|
for label, spans in compiled_gold["spans_per_type"][spans_key].items()
|
||||||
|
}
|
||||||
|
p_bounds = {
|
||||||
|
label: _get_distribution(sb["start"] + sb["end"], normalize=True)
|
||||||
|
for label, sb in compiled_gold["sb_per_type"][spans_key].items()
|
||||||
|
}
|
||||||
|
|
||||||
|
# Compute for actual span characteristics
|
||||||
|
span_distinctiveness = {
|
||||||
|
label: _get_kl_divergence(freq_dist, p_corpus)
|
||||||
|
for label, freq_dist in p_spans.items()
|
||||||
|
}
|
||||||
|
sb_distinctiveness = {
|
||||||
|
label: _get_kl_divergence(freq_dist, p_corpus)
|
||||||
|
for label, freq_dist in p_bounds.items()
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
"sd": span_distinctiveness,
|
||||||
|
"bd": sb_distinctiveness,
|
||||||
|
"lengths": span_length,
|
||||||
|
"min_length": min(min_lengths),
|
||||||
|
"max_length": max(max_lengths),
|
||||||
|
"avg_sd": _wgt_average(span_distinctiveness, data_labels),
|
||||||
|
"avg_bd": _wgt_average(sb_distinctiveness, data_labels),
|
||||||
|
"avg_length": _wgt_average(span_length, data_labels),
|
||||||
|
"labels": list(data_labels.keys()),
|
||||||
|
"p_spans": p_spans,
|
||||||
|
"p_bounds": p_bounds,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _print_span_characteristics(span_characteristics: Dict[str, Any]):
|
||||||
|
"""Print all span characteristics into a table"""
|
||||||
|
headers = ("Span Type", "Length", "SD", "BD")
|
||||||
|
# Prepare table data with all span characteristics
|
||||||
|
table_data = [
|
||||||
|
span_characteristics["lengths"],
|
||||||
|
span_characteristics["sd"],
|
||||||
|
span_characteristics["bd"],
|
||||||
|
]
|
||||||
|
table = _format_span_row(
|
||||||
|
span_data=table_data, labels=span_characteristics["labels"]
|
||||||
|
)
|
||||||
|
# Prepare table footer with weighted averages
|
||||||
|
footer_data = [
|
||||||
|
span_characteristics["avg_length"],
|
||||||
|
span_characteristics["avg_sd"],
|
||||||
|
span_characteristics["avg_bd"],
|
||||||
|
]
|
||||||
|
footer = ["Wgt. Average"] + [str(round(f, 2)) for f in footer_data]
|
||||||
|
msg.table(table, footer=footer, header=headers, divider=True)
|
||||||
|
|
||||||
|
|
||||||
|
def _get_spans_length_freq_dist(
|
||||||
|
length_dict: Dict, threshold=SPAN_LENGTH_THRESHOLD_PERCENTAGE
|
||||||
|
) -> Dict[int, float]:
|
||||||
|
"""Get frequency distribution of spans length under a certain threshold"""
|
||||||
|
all_span_lengths = []
|
||||||
|
for _, lengths in length_dict.items():
|
||||||
|
all_span_lengths.extend(lengths)
|
||||||
|
|
||||||
|
freq_dist: Counter = Counter()
|
||||||
|
for i in all_span_lengths:
|
||||||
|
if freq_dist.get(i):
|
||||||
|
freq_dist[i] += 1
|
||||||
|
else:
|
||||||
|
freq_dist[i] = 1
|
||||||
|
|
||||||
|
# We will be working with percentages instead of raw counts
|
||||||
|
freq_dist_percentage = {}
|
||||||
|
for span_length, count in freq_dist.most_common():
|
||||||
|
percentage = (count / len(all_span_lengths)) * 100.0
|
||||||
|
percentage = round(percentage, 2)
|
||||||
|
freq_dist_percentage[span_length] = percentage
|
||||||
|
|
||||||
|
return freq_dist_percentage
|
||||||
|
|
||||||
|
|
||||||
|
def _filter_spans_length_freq_dist(
|
||||||
|
freq_dist: Dict[int, float], threshold: int
|
||||||
|
) -> Dict[int, float]:
|
||||||
|
"""Filter frequency distribution with respect to a threshold
|
||||||
|
|
||||||
|
We're going to filter all the span lengths that fall
|
||||||
|
around a percentage threshold when summed.
|
||||||
|
"""
|
||||||
|
total = 0.0
|
||||||
|
filtered_freq_dist = {}
|
||||||
|
for span_length, dist in freq_dist.items():
|
||||||
|
if total >= threshold:
|
||||||
|
break
|
||||||
|
else:
|
||||||
|
filtered_freq_dist[span_length] = dist
|
||||||
|
total += dist
|
||||||
|
return filtered_freq_dist
|
||||||
|
|
|
@ -7,6 +7,7 @@ import typer
|
||||||
from ._util import app, Arg, Opt, WHEEL_SUFFIX, SDIST_SUFFIX
|
from ._util import app, Arg, Opt, WHEEL_SUFFIX, SDIST_SUFFIX
|
||||||
from .. import about
|
from .. import about
|
||||||
from ..util import is_package, get_minor_version, run_command
|
from ..util import is_package, get_minor_version, run_command
|
||||||
|
from ..util import is_prerelease_version
|
||||||
from ..errors import OLD_MODEL_SHORTCUTS
|
from ..errors import OLD_MODEL_SHORTCUTS
|
||||||
|
|
||||||
|
|
||||||
|
@ -74,7 +75,10 @@ def download(model: str, direct: bool = False, sdist: bool = False, *pip_args) -
|
||||||
|
|
||||||
|
|
||||||
def get_compatibility() -> dict:
|
def get_compatibility() -> dict:
|
||||||
version = get_minor_version(about.__version__)
|
if is_prerelease_version(about.__version__):
|
||||||
|
version: Optional[str] = about.__version__
|
||||||
|
else:
|
||||||
|
version = get_minor_version(about.__version__)
|
||||||
r = requests.get(about.__compatibility__)
|
r = requests.get(about.__compatibility__)
|
||||||
if r.status_code != 200:
|
if r.status_code != 200:
|
||||||
msg.fail(
|
msg.fail(
|
||||||
|
|
|
@ -10,6 +10,7 @@ from jinja2 import Template
|
||||||
from .. import util
|
from .. import util
|
||||||
from ..language import DEFAULT_CONFIG_PRETRAIN_PATH
|
from ..language import DEFAULT_CONFIG_PRETRAIN_PATH
|
||||||
from ..schemas import RecommendationSchema
|
from ..schemas import RecommendationSchema
|
||||||
|
from ..util import SimpleFrozenList
|
||||||
from ._util import init_cli, Arg, Opt, show_validation_error, COMMAND
|
from ._util import init_cli, Arg, Opt, show_validation_error, COMMAND
|
||||||
from ._util import string_to_list, import_code
|
from ._util import string_to_list, import_code
|
||||||
|
|
||||||
|
@ -24,16 +25,30 @@ class Optimizations(str, Enum):
|
||||||
accuracy = "accuracy"
|
accuracy = "accuracy"
|
||||||
|
|
||||||
|
|
||||||
|
class InitValues:
|
||||||
|
"""
|
||||||
|
Default values for initialization. Dedicated class to allow synchronized default values for init_config_cli() and
|
||||||
|
init_config(), i.e. initialization calls via CLI respectively Python.
|
||||||
|
"""
|
||||||
|
|
||||||
|
lang = "en"
|
||||||
|
pipeline = SimpleFrozenList(["tagger", "parser", "ner"])
|
||||||
|
optimize = Optimizations.efficiency
|
||||||
|
gpu = False
|
||||||
|
pretraining = False
|
||||||
|
force_overwrite = False
|
||||||
|
|
||||||
|
|
||||||
@init_cli.command("config")
|
@init_cli.command("config")
|
||||||
def init_config_cli(
|
def init_config_cli(
|
||||||
# fmt: off
|
# fmt: off
|
||||||
output_file: Path = Arg(..., help="File to save the config to or - for stdout (will only output config and no additional logging info)", allow_dash=True),
|
output_file: Path = Arg(..., help="File to save the config to or - for stdout (will only output config and no additional logging info)", allow_dash=True),
|
||||||
lang: str = Opt("en", "--lang", "-l", help="Two-letter code of the language to use"),
|
lang: str = Opt(InitValues.lang, "--lang", "-l", help="Two-letter code of the language to use"),
|
||||||
pipeline: str = Opt("tagger,parser,ner", "--pipeline", "-p", help="Comma-separated names of trainable pipeline components to include (without 'tok2vec' or 'transformer')"),
|
pipeline: str = Opt(",".join(InitValues.pipeline), "--pipeline", "-p", help="Comma-separated names of trainable pipeline components to include (without 'tok2vec' or 'transformer')"),
|
||||||
optimize: Optimizations = Opt(Optimizations.efficiency.value, "--optimize", "-o", help="Whether to optimize for efficiency (faster inference, smaller model, lower memory consumption) or higher accuracy (potentially larger and slower model). This will impact the choice of architecture, pretrained weights and related hyperparameters."),
|
optimize: Optimizations = Opt(InitValues.optimize, "--optimize", "-o", help="Whether to optimize for efficiency (faster inference, smaller model, lower memory consumption) or higher accuracy (potentially larger and slower model). This will impact the choice of architecture, pretrained weights and related hyperparameters."),
|
||||||
gpu: bool = Opt(False, "--gpu", "-G", help="Whether the model can run on GPU. This will impact the choice of architecture, pretrained weights and related hyperparameters."),
|
gpu: bool = Opt(InitValues.gpu, "--gpu", "-G", help="Whether the model can run on GPU. This will impact the choice of architecture, pretrained weights and related hyperparameters."),
|
||||||
pretraining: bool = Opt(False, "--pretraining", "-pt", help="Include config for pretraining (with 'spacy pretrain')"),
|
pretraining: bool = Opt(InitValues.pretraining, "--pretraining", "-pt", help="Include config for pretraining (with 'spacy pretrain')"),
|
||||||
force_overwrite: bool = Opt(False, "--force", "-F", help="Force overwriting the output file"),
|
force_overwrite: bool = Opt(InitValues.force_overwrite, "--force", "-F", help="Force overwriting the output file"),
|
||||||
# fmt: on
|
# fmt: on
|
||||||
):
|
):
|
||||||
"""
|
"""
|
||||||
|
@ -133,11 +148,11 @@ def fill_config(
|
||||||
|
|
||||||
def init_config(
|
def init_config(
|
||||||
*,
|
*,
|
||||||
lang: str,
|
lang: str = InitValues.lang,
|
||||||
pipeline: List[str],
|
pipeline: List[str] = InitValues.pipeline,
|
||||||
optimize: str,
|
optimize: str = InitValues.optimize,
|
||||||
gpu: bool,
|
gpu: bool = InitValues.gpu,
|
||||||
pretraining: bool = False,
|
pretraining: bool = InitValues.pretraining,
|
||||||
silent: bool = True,
|
silent: bool = True,
|
||||||
) -> Config:
|
) -> Config:
|
||||||
msg = Printer(no_print=silent)
|
msg = Printer(no_print=silent)
|
||||||
|
|
|
@ -61,7 +61,7 @@ def pretrain_cli(
|
||||||
# TODO: What's the solution here? How do we handle optional blocks?
|
# TODO: What's the solution here? How do we handle optional blocks?
|
||||||
msg.fail("The [pretraining] block in your config is empty", exits=1)
|
msg.fail("The [pretraining] block in your config is empty", exits=1)
|
||||||
if not output_dir.exists():
|
if not output_dir.exists():
|
||||||
output_dir.mkdir()
|
output_dir.mkdir(parents=True)
|
||||||
msg.good(f"Created output directory: {output_dir}")
|
msg.good(f"Created output directory: {output_dir}")
|
||||||
# Save non-interpolated config
|
# Save non-interpolated config
|
||||||
raw_config.to_disk(output_dir / "config.cfg")
|
raw_config.to_disk(output_dir / "config.cfg")
|
||||||
|
|
|
@ -12,6 +12,9 @@ from .._util import project_cli, Arg, Opt, PROJECT_FILE, load_project_config
|
||||||
from .._util import get_checksum, download_file, git_checkout, get_git_version
|
from .._util import get_checksum, download_file, git_checkout, get_git_version
|
||||||
from .._util import SimpleFrozenDict, parse_config_overrides
|
from .._util import SimpleFrozenDict, parse_config_overrides
|
||||||
|
|
||||||
|
# Whether assets are extra if `extra` is not set.
|
||||||
|
EXTRA_DEFAULT = False
|
||||||
|
|
||||||
|
|
||||||
@project_cli.command(
|
@project_cli.command(
|
||||||
"assets",
|
"assets",
|
||||||
|
@ -21,7 +24,8 @@ def project_assets_cli(
|
||||||
# fmt: off
|
# fmt: off
|
||||||
ctx: typer.Context, # This is only used to read additional arguments
|
ctx: typer.Context, # This is only used to read additional arguments
|
||||||
project_dir: Path = Arg(Path.cwd(), help="Path to cloned project. Defaults to current working directory.", exists=True, file_okay=False),
|
project_dir: Path = Arg(Path.cwd(), help="Path to cloned project. Defaults to current working directory.", exists=True, file_okay=False),
|
||||||
sparse_checkout: bool = Opt(False, "--sparse", "-S", help="Use sparse checkout for assets provided via Git, to only check out and clone the files needed. Requires Git v22.2+.")
|
sparse_checkout: bool = Opt(False, "--sparse", "-S", help="Use sparse checkout for assets provided via Git, to only check out and clone the files needed. Requires Git v22.2+."),
|
||||||
|
extra: bool = Opt(False, "--extra", "-e", help="Download all assets, including those marked as 'extra'.")
|
||||||
# fmt: on
|
# fmt: on
|
||||||
):
|
):
|
||||||
"""Fetch project assets like datasets and pretrained weights. Assets are
|
"""Fetch project assets like datasets and pretrained weights. Assets are
|
||||||
|
@ -32,7 +36,12 @@ def project_assets_cli(
|
||||||
DOCS: https://spacy.io/api/cli#project-assets
|
DOCS: https://spacy.io/api/cli#project-assets
|
||||||
"""
|
"""
|
||||||
overrides = parse_config_overrides(ctx.args)
|
overrides = parse_config_overrides(ctx.args)
|
||||||
project_assets(project_dir, overrides=overrides, sparse_checkout=sparse_checkout)
|
project_assets(
|
||||||
|
project_dir,
|
||||||
|
overrides=overrides,
|
||||||
|
sparse_checkout=sparse_checkout,
|
||||||
|
extra=extra,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def project_assets(
|
def project_assets(
|
||||||
|
@ -40,17 +49,29 @@ def project_assets(
|
||||||
*,
|
*,
|
||||||
overrides: Dict[str, Any] = SimpleFrozenDict(),
|
overrides: Dict[str, Any] = SimpleFrozenDict(),
|
||||||
sparse_checkout: bool = False,
|
sparse_checkout: bool = False,
|
||||||
|
extra: bool = False,
|
||||||
) -> None:
|
) -> None:
|
||||||
"""Fetch assets for a project using DVC if possible.
|
"""Fetch assets for a project using DVC if possible.
|
||||||
|
|
||||||
project_dir (Path): Path to project directory.
|
project_dir (Path): Path to project directory.
|
||||||
|
sparse_checkout (bool): Use sparse checkout for assets provided via Git, to only check out and clone the files
|
||||||
|
needed.
|
||||||
|
extra (bool): Whether to download all assets, including those marked as 'extra'.
|
||||||
"""
|
"""
|
||||||
project_path = ensure_path(project_dir)
|
project_path = ensure_path(project_dir)
|
||||||
config = load_project_config(project_path, overrides=overrides)
|
config = load_project_config(project_path, overrides=overrides)
|
||||||
assets = config.get("assets", {})
|
assets = [
|
||||||
|
asset
|
||||||
|
for asset in config.get("assets", [])
|
||||||
|
if extra or not asset.get("extra", EXTRA_DEFAULT)
|
||||||
|
]
|
||||||
if not assets:
|
if not assets:
|
||||||
msg.warn(f"No assets specified in {PROJECT_FILE}", exits=0)
|
msg.warn(
|
||||||
|
f"No assets specified in {PROJECT_FILE} (if assets are marked as extra, download them with --extra)",
|
||||||
|
exits=0,
|
||||||
|
)
|
||||||
msg.info(f"Fetching {len(assets)} asset(s)")
|
msg.info(f"Fetching {len(assets)} asset(s)")
|
||||||
|
|
||||||
for asset in assets:
|
for asset in assets:
|
||||||
dest = (project_dir / asset["dest"]).resolve()
|
dest = (project_dir / asset["dest"]).resolve()
|
||||||
checksum = asset.get("checksum")
|
checksum = asset.get("checksum")
|
||||||
|
|
|
@ -7,11 +7,11 @@ import re
|
||||||
from ... import about
|
from ... import about
|
||||||
from ...util import ensure_path
|
from ...util import ensure_path
|
||||||
from .._util import project_cli, Arg, Opt, COMMAND, PROJECT_FILE
|
from .._util import project_cli, Arg, Opt, COMMAND, PROJECT_FILE
|
||||||
from .._util import git_checkout, get_git_version
|
from .._util import git_checkout, get_git_version, git_repo_branch_exists
|
||||||
|
|
||||||
DEFAULT_REPO = about.__projects__
|
DEFAULT_REPO = about.__projects__
|
||||||
DEFAULT_PROJECTS_BRANCH = about.__projects_branch__
|
DEFAULT_PROJECTS_BRANCH = about.__projects_branch__
|
||||||
DEFAULT_BRANCH = "master"
|
DEFAULT_BRANCHES = ["main", "master"]
|
||||||
|
|
||||||
|
|
||||||
@project_cli.command("clone")
|
@project_cli.command("clone")
|
||||||
|
@ -20,7 +20,7 @@ def project_clone_cli(
|
||||||
name: str = Arg(..., help="The name of the template to clone"),
|
name: str = Arg(..., help="The name of the template to clone"),
|
||||||
dest: Optional[Path] = Arg(None, help="Where to clone the project. Defaults to current working directory", exists=False),
|
dest: Optional[Path] = Arg(None, help="Where to clone the project. Defaults to current working directory", exists=False),
|
||||||
repo: str = Opt(DEFAULT_REPO, "--repo", "-r", help="The repository to clone from"),
|
repo: str = Opt(DEFAULT_REPO, "--repo", "-r", help="The repository to clone from"),
|
||||||
branch: Optional[str] = Opt(None, "--branch", "-b", help="The branch to clone from"),
|
branch: Optional[str] = Opt(None, "--branch", "-b", help=f"The branch to clone from. If not provided, will attempt {', '.join(DEFAULT_BRANCHES)}"),
|
||||||
sparse_checkout: bool = Opt(False, "--sparse", "-S", help="Use sparse Git checkout to only check out and clone the files needed. Requires Git v22.2+.")
|
sparse_checkout: bool = Opt(False, "--sparse", "-S", help="Use sparse Git checkout to only check out and clone the files needed. Requires Git v22.2+.")
|
||||||
# fmt: on
|
# fmt: on
|
||||||
):
|
):
|
||||||
|
@ -33,9 +33,25 @@ def project_clone_cli(
|
||||||
"""
|
"""
|
||||||
if dest is None:
|
if dest is None:
|
||||||
dest = Path.cwd() / Path(name).parts[-1]
|
dest = Path.cwd() / Path(name).parts[-1]
|
||||||
|
if repo == DEFAULT_REPO and branch is None:
|
||||||
|
branch = DEFAULT_PROJECTS_BRANCH
|
||||||
|
|
||||||
if branch is None:
|
if branch is None:
|
||||||
# If it's a user repo, we want to default to other branch
|
for default_branch in DEFAULT_BRANCHES:
|
||||||
branch = DEFAULT_PROJECTS_BRANCH if repo == DEFAULT_REPO else DEFAULT_BRANCH
|
if git_repo_branch_exists(repo, default_branch):
|
||||||
|
branch = default_branch
|
||||||
|
break
|
||||||
|
if branch is None:
|
||||||
|
default_branches_msg = ", ".join(f"'{b}'" for b in DEFAULT_BRANCHES)
|
||||||
|
msg.fail(
|
||||||
|
"No branch provided and attempted default "
|
||||||
|
f"branches {default_branches_msg} do not exist.",
|
||||||
|
exits=1,
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
if not git_repo_branch_exists(repo, branch):
|
||||||
|
msg.fail(f"repo: {repo} (branch: {branch}) does not exist.", exits=1)
|
||||||
|
assert isinstance(branch, str)
|
||||||
project_clone(name, dest, repo=repo, branch=branch, sparse_checkout=sparse_checkout)
|
project_clone(name, dest, repo=repo, branch=branch, sparse_checkout=sparse_checkout)
|
||||||
|
|
||||||
|
|
||||||
|
@ -61,9 +77,9 @@ def project_clone(
|
||||||
try:
|
try:
|
||||||
git_checkout(repo, name, dest, branch=branch, sparse=sparse_checkout)
|
git_checkout(repo, name, dest, branch=branch, sparse=sparse_checkout)
|
||||||
except subprocess.CalledProcessError:
|
except subprocess.CalledProcessError:
|
||||||
err = f"Could not clone '{name}' from repo '{repo_name}'"
|
err = f"Could not clone '{name}' from repo '{repo_name}' (branch '{branch}')"
|
||||||
msg.fail(err, exits=1)
|
msg.fail(err, exits=1)
|
||||||
msg.good(f"Cloned '{name}' from {repo_name}", project_dir)
|
msg.good(f"Cloned '{name}' from '{repo_name}' (branch '{branch}')", project_dir)
|
||||||
if not (project_dir / PROJECT_FILE).exists():
|
if not (project_dir / PROJECT_FILE).exists():
|
||||||
msg.warn(f"No {PROJECT_FILE} found in directory")
|
msg.warn(f"No {PROJECT_FILE} found in directory")
|
||||||
else:
|
else:
|
||||||
|
|
|
@ -64,8 +64,11 @@ class SpanRenderer:
|
||||||
# Set up how the text and labels will be rendered
|
# Set up how the text and labels will be rendered
|
||||||
self.direction = DEFAULT_DIR
|
self.direction = DEFAULT_DIR
|
||||||
self.lang = DEFAULT_LANG
|
self.lang = DEFAULT_LANG
|
||||||
|
# These values are in px
|
||||||
self.top_offset = options.get("top_offset", 40)
|
self.top_offset = options.get("top_offset", 40)
|
||||||
self.top_offset_step = options.get("top_offset_step", 17)
|
# This is how far under the top offset the span labels appear
|
||||||
|
self.span_label_offset = options.get("span_label_offset", 20)
|
||||||
|
self.offset_step = options.get("top_offset_step", 17)
|
||||||
|
|
||||||
# Set up which templates will be used
|
# Set up which templates will be used
|
||||||
template = options.get("template")
|
template = options.get("template")
|
||||||
|
@ -127,26 +130,56 @@ class SpanRenderer:
|
||||||
title (str / None): Document title set in Doc.user_data['title'].
|
title (str / None): Document title set in Doc.user_data['title'].
|
||||||
"""
|
"""
|
||||||
per_token_info = []
|
per_token_info = []
|
||||||
|
# we must sort so that we can correctly describe when spans need to "stack"
|
||||||
|
# which is determined by their start token, then span length (longer spans on top),
|
||||||
|
# then break any remaining ties with the span label
|
||||||
|
spans = sorted(
|
||||||
|
spans,
|
||||||
|
key=lambda s: (
|
||||||
|
s["start_token"],
|
||||||
|
-(s["end_token"] - s["start_token"]),
|
||||||
|
s["label"],
|
||||||
|
),
|
||||||
|
)
|
||||||
|
for s in spans:
|
||||||
|
# this is the vertical 'slot' that the span will be rendered in
|
||||||
|
# vertical_position = span_label_offset + (offset_step * (slot - 1))
|
||||||
|
s["render_slot"] = 0
|
||||||
for idx, token in enumerate(tokens):
|
for idx, token in enumerate(tokens):
|
||||||
# Identify if a token belongs to a Span (and which) and if it's a
|
# Identify if a token belongs to a Span (and which) and if it's a
|
||||||
# start token of said Span. We'll use this for the final HTML render
|
# start token of said Span. We'll use this for the final HTML render
|
||||||
token_markup: Dict[str, Any] = {}
|
token_markup: Dict[str, Any] = {}
|
||||||
token_markup["text"] = token
|
token_markup["text"] = token
|
||||||
|
concurrent_spans = 0
|
||||||
entities = []
|
entities = []
|
||||||
for span in spans:
|
for span in spans:
|
||||||
ent = {}
|
ent = {}
|
||||||
if span["start_token"] <= idx < span["end_token"]:
|
if span["start_token"] <= idx < span["end_token"]:
|
||||||
|
concurrent_spans += 1
|
||||||
|
span_start = idx == span["start_token"]
|
||||||
ent["label"] = span["label"]
|
ent["label"] = span["label"]
|
||||||
ent["is_start"] = True if idx == span["start_token"] else False
|
ent["is_start"] = span_start
|
||||||
|
if span_start:
|
||||||
|
# When the span starts, we need to know how many other
|
||||||
|
# spans are on the 'span stack' and will be rendered.
|
||||||
|
# This value becomes the vertical render slot for this entire span
|
||||||
|
span["render_slot"] = concurrent_spans
|
||||||
|
ent["render_slot"] = span["render_slot"]
|
||||||
kb_id = span.get("kb_id", "")
|
kb_id = span.get("kb_id", "")
|
||||||
kb_url = span.get("kb_url", "#")
|
kb_url = span.get("kb_url", "#")
|
||||||
ent["kb_link"] = (
|
ent["kb_link"] = (
|
||||||
TPL_KB_LINK.format(kb_id=kb_id, kb_url=kb_url) if kb_id else ""
|
TPL_KB_LINK.format(kb_id=kb_id, kb_url=kb_url) if kb_id else ""
|
||||||
)
|
)
|
||||||
entities.append(ent)
|
entities.append(ent)
|
||||||
|
else:
|
||||||
|
# We don't specifically need to do this since we loop
|
||||||
|
# over tokens and spans sorted by their start_token,
|
||||||
|
# so we'll never use a span again after the last token it appears in,
|
||||||
|
# but if we were to use these spans again we'd want to make sure
|
||||||
|
# this value was reset correctly.
|
||||||
|
span["render_slot"] = 0
|
||||||
token_markup["entities"] = entities
|
token_markup["entities"] = entities
|
||||||
per_token_info.append(token_markup)
|
per_token_info.append(token_markup)
|
||||||
|
|
||||||
markup = self._render_markup(per_token_info)
|
markup = self._render_markup(per_token_info)
|
||||||
markup = TPL_SPANS.format(content=markup, dir=self.direction)
|
markup = TPL_SPANS.format(content=markup, dir=self.direction)
|
||||||
if title:
|
if title:
|
||||||
|
@ -157,12 +190,24 @@ class SpanRenderer:
|
||||||
"""Render the markup from per-token information"""
|
"""Render the markup from per-token information"""
|
||||||
markup = ""
|
markup = ""
|
||||||
for token in per_token_info:
|
for token in per_token_info:
|
||||||
entities = sorted(token["entities"], key=lambda d: d["label"])
|
entities = sorted(token["entities"], key=lambda d: d["render_slot"])
|
||||||
if entities:
|
# Whitespace tokens disrupt the vertical space (no line height) so that the
|
||||||
|
# span indicators get misaligned. We don't render them as individual
|
||||||
|
# tokens anyway, so we'll just not display a span indicator either.
|
||||||
|
is_whitespace = token["text"].strip() == ""
|
||||||
|
if entities and not is_whitespace:
|
||||||
slices = self._get_span_slices(token["entities"])
|
slices = self._get_span_slices(token["entities"])
|
||||||
starts = self._get_span_starts(token["entities"])
|
starts = self._get_span_starts(token["entities"])
|
||||||
|
total_height = (
|
||||||
|
self.top_offset
|
||||||
|
+ self.span_label_offset
|
||||||
|
+ (self.offset_step * (len(entities) - 1))
|
||||||
|
)
|
||||||
markup += self.span_template.format(
|
markup += self.span_template.format(
|
||||||
text=token["text"], span_slices=slices, span_starts=starts
|
text=token["text"],
|
||||||
|
span_slices=slices,
|
||||||
|
span_starts=starts,
|
||||||
|
total_height=total_height,
|
||||||
)
|
)
|
||||||
else:
|
else:
|
||||||
markup += escape_html(token["text"] + " ")
|
markup += escape_html(token["text"] + " ")
|
||||||
|
@ -171,10 +216,18 @@ class SpanRenderer:
|
||||||
def _get_span_slices(self, entities: List[Dict]) -> str:
|
def _get_span_slices(self, entities: List[Dict]) -> str:
|
||||||
"""Get the rendered markup of all Span slices"""
|
"""Get the rendered markup of all Span slices"""
|
||||||
span_slices = []
|
span_slices = []
|
||||||
for entity, step in zip(entities, itertools.count(step=self.top_offset_step)):
|
for entity in entities:
|
||||||
|
# rather than iterate over multiples of offset_step, we use entity['render_slot']
|
||||||
|
# to determine the vertical position, since that tells where
|
||||||
|
# the span starts vertically so we can extend it horizontally,
|
||||||
|
# past other spans that might have already ended
|
||||||
color = self.colors.get(entity["label"].upper(), self.default_color)
|
color = self.colors.get(entity["label"].upper(), self.default_color)
|
||||||
|
top_offset = self.top_offset + (
|
||||||
|
self.offset_step * (entity["render_slot"] - 1)
|
||||||
|
)
|
||||||
span_slice = self.span_slice_template.format(
|
span_slice = self.span_slice_template.format(
|
||||||
bg=color, top_offset=self.top_offset + step
|
bg=color,
|
||||||
|
top_offset=top_offset,
|
||||||
)
|
)
|
||||||
span_slices.append(span_slice)
|
span_slices.append(span_slice)
|
||||||
return "".join(span_slices)
|
return "".join(span_slices)
|
||||||
|
@ -182,12 +235,15 @@ class SpanRenderer:
|
||||||
def _get_span_starts(self, entities: List[Dict]) -> str:
|
def _get_span_starts(self, entities: List[Dict]) -> str:
|
||||||
"""Get the rendered markup of all Span start tokens"""
|
"""Get the rendered markup of all Span start tokens"""
|
||||||
span_starts = []
|
span_starts = []
|
||||||
for entity, step in zip(entities, itertools.count(step=self.top_offset_step)):
|
for entity in entities:
|
||||||
color = self.colors.get(entity["label"].upper(), self.default_color)
|
color = self.colors.get(entity["label"].upper(), self.default_color)
|
||||||
|
top_offset = self.top_offset + (
|
||||||
|
self.offset_step * (entity["render_slot"] - 1)
|
||||||
|
)
|
||||||
span_start = (
|
span_start = (
|
||||||
self.span_start_template.format(
|
self.span_start_template.format(
|
||||||
bg=color,
|
bg=color,
|
||||||
top_offset=self.top_offset + step,
|
top_offset=top_offset,
|
||||||
label=entity["label"],
|
label=entity["label"],
|
||||||
kb_link=entity["kb_link"],
|
kb_link=entity["kb_link"],
|
||||||
)
|
)
|
||||||
|
|
|
@ -67,7 +67,7 @@ TPL_SPANS = """
|
||||||
"""
|
"""
|
||||||
|
|
||||||
TPL_SPAN = """
|
TPL_SPAN = """
|
||||||
<span style="font-weight: bold; display: inline-block; position: relative;">
|
<span style="font-weight: bold; display: inline-block; position: relative; height: {total_height}px;">
|
||||||
{text}
|
{text}
|
||||||
{span_slices}
|
{span_slices}
|
||||||
{span_starts}
|
{span_starts}
|
||||||
|
|
|
@ -1,4 +1,5 @@
|
||||||
import warnings
|
import warnings
|
||||||
|
from .compat import Literal
|
||||||
|
|
||||||
|
|
||||||
class ErrorsWithCodes(type):
|
class ErrorsWithCodes(type):
|
||||||
|
@ -26,7 +27,10 @@ def setup_default_warnings():
|
||||||
filter_warning("once", error_msg="[W114]")
|
filter_warning("once", error_msg="[W114]")
|
||||||
|
|
||||||
|
|
||||||
def filter_warning(action: str, error_msg: str):
|
def filter_warning(
|
||||||
|
action: Literal["default", "error", "ignore", "always", "module", "once"],
|
||||||
|
error_msg: str,
|
||||||
|
):
|
||||||
"""Customize how spaCy should handle a certain warning.
|
"""Customize how spaCy should handle a certain warning.
|
||||||
|
|
||||||
error_msg (str): e.g. "W006", or a full error message
|
error_msg (str): e.g. "W006", or a full error message
|
||||||
|
@ -199,6 +203,15 @@ class Warnings(metaclass=ErrorsWithCodes):
|
||||||
W118 = ("Term '{term}' not found in glossary. It may however be explained in documentation "
|
W118 = ("Term '{term}' not found in glossary. It may however be explained in documentation "
|
||||||
"for the corpora used to train the language. Please check "
|
"for the corpora used to train the language. Please check "
|
||||||
"`nlp.meta[\"sources\"]` for any relevant links.")
|
"`nlp.meta[\"sources\"]` for any relevant links.")
|
||||||
|
W119 = ("Overriding pipe name in `config` is not supported. Ignoring override '{name_in_config}'.")
|
||||||
|
W120 = ("Unable to load all spans in Doc.spans: more than one span group "
|
||||||
|
"with the name '{group_name}' was found in the saved spans data. "
|
||||||
|
"Only the last span group will be loaded under "
|
||||||
|
"Doc.spans['{group_name}']. Skipping span group with values: "
|
||||||
|
"{group_values}")
|
||||||
|
W121 = ("Attempting to trace non-existent method '{method}' in pipe '{pipe}'")
|
||||||
|
W122 = ("Couldn't trace method '{method}' in pipe '{pipe}'. This can happen if the pipe class "
|
||||||
|
"is a Cython extension type.")
|
||||||
|
|
||||||
|
|
||||||
class Errors(metaclass=ErrorsWithCodes):
|
class Errors(metaclass=ErrorsWithCodes):
|
||||||
|
@ -444,10 +457,10 @@ class Errors(metaclass=ErrorsWithCodes):
|
||||||
"same, but found '{nlp}' and '{vocab}' respectively.")
|
"same, but found '{nlp}' and '{vocab}' respectively.")
|
||||||
E152 = ("The attribute {attr} is not supported for token patterns. "
|
E152 = ("The attribute {attr} is not supported for token patterns. "
|
||||||
"Please use the option `validate=True` with the Matcher, PhraseMatcher, "
|
"Please use the option `validate=True` with the Matcher, PhraseMatcher, "
|
||||||
"or EntityRuler for more details.")
|
"EntityRuler or AttributeRuler for more details.")
|
||||||
E153 = ("The value type {vtype} is not supported for token patterns. "
|
E153 = ("The value type {vtype} is not supported for token patterns. "
|
||||||
"Please use the option validate=True with Matcher, PhraseMatcher, "
|
"Please use the option validate=True with Matcher, PhraseMatcher, "
|
||||||
"or EntityRuler for more details.")
|
"EntityRuler or AttributeRuler for more details.")
|
||||||
E154 = ("One of the attributes or values is not supported for token "
|
E154 = ("One of the attributes or values is not supported for token "
|
||||||
"patterns. Please use the option `validate=True` with the Matcher, "
|
"patterns. Please use the option `validate=True` with the Matcher, "
|
||||||
"PhraseMatcher, or EntityRuler for more details.")
|
"PhraseMatcher, or EntityRuler for more details.")
|
||||||
|
@ -527,6 +540,8 @@ class Errors(metaclass=ErrorsWithCodes):
|
||||||
E202 = ("Unsupported {name} mode '{mode}'. Supported modes: {modes}.")
|
E202 = ("Unsupported {name} mode '{mode}'. Supported modes: {modes}.")
|
||||||
|
|
||||||
# New errors added in v3.x
|
# New errors added in v3.x
|
||||||
|
E854 = ("Unable to set doc.ents. Check that the 'ents_filter' does not "
|
||||||
|
"permit overlapping spans.")
|
||||||
E855 = ("Invalid {obj}: {obj} is not from the same doc.")
|
E855 = ("Invalid {obj}: {obj} is not from the same doc.")
|
||||||
E856 = ("Error accessing span at position {i}: out of bounds in span group "
|
E856 = ("Error accessing span at position {i}: out of bounds in span group "
|
||||||
"of length {length}.")
|
"of length {length}.")
|
||||||
|
@ -898,8 +913,8 @@ class Errors(metaclass=ErrorsWithCodes):
|
||||||
E1022 = ("Words must be of type str or int, but input is of type '{wtype}'")
|
E1022 = ("Words must be of type str or int, but input is of type '{wtype}'")
|
||||||
E1023 = ("Couldn't read EntityRuler from the {path}. This file doesn't "
|
E1023 = ("Couldn't read EntityRuler from the {path}. This file doesn't "
|
||||||
"exist.")
|
"exist.")
|
||||||
E1024 = ("A pattern with ID \"{ent_id}\" is not present in EntityRuler "
|
E1024 = ("A pattern with {attr_type} '{label}' is not present in "
|
||||||
"patterns.")
|
"'{component}' patterns.")
|
||||||
E1025 = ("Cannot intify the value '{value}' as an IOB string. The only "
|
E1025 = ("Cannot intify the value '{value}' as an IOB string. The only "
|
||||||
"supported values are: 'I', 'O', 'B' and ''")
|
"supported values are: 'I', 'O', 'B' and ''")
|
||||||
E1026 = ("Edit tree has an invalid format:\n{errors}")
|
E1026 = ("Edit tree has an invalid format:\n{errors}")
|
||||||
|
@ -913,6 +928,17 @@ class Errors(metaclass=ErrorsWithCodes):
|
||||||
E1034 = ("Node index {i} out of bounds ({length})")
|
E1034 = ("Node index {i} out of bounds ({length})")
|
||||||
E1035 = ("Token index {i} out of bounds ({length})")
|
E1035 = ("Token index {i} out of bounds ({length})")
|
||||||
E1036 = ("Cannot index into NoneNode")
|
E1036 = ("Cannot index into NoneNode")
|
||||||
|
E1037 = ("Invalid attribute value '{attr}'.")
|
||||||
|
E1038 = ("Invalid JSON input: {message}")
|
||||||
|
E1039 = ("The {obj} start or end annotations (start: {start}, end: {end}) "
|
||||||
|
"could not be aligned to token boundaries.")
|
||||||
|
E1040 = ("Doc.from_json requires all tokens to have the same attributes. "
|
||||||
|
"Some tokens do not contain annotation for: {partial_attrs}")
|
||||||
|
E1041 = ("Expected a string, Doc, or bytes as input, but got: {type}")
|
||||||
|
E1042 = ("Function was called with `{arg1}`={arg1_values} and "
|
||||||
|
"`{arg2}`={arg2_values} but these arguments are conflicting.")
|
||||||
|
E1043 = ("Expected None or a value in range [{range_start}, {range_end}] for entity linker threshold, but got "
|
||||||
|
"{value}.")
|
||||||
|
|
||||||
|
|
||||||
# Deprecated model shortcuts, only used in errors and warnings
|
# Deprecated model shortcuts, only used in errors and warnings
|
||||||
|
|
|
@ -273,6 +273,7 @@ GLOSSARY = {
|
||||||
"relcl": "relative clause modifier",
|
"relcl": "relative clause modifier",
|
||||||
"reparandum": "overridden disfluency",
|
"reparandum": "overridden disfluency",
|
||||||
"root": "root",
|
"root": "root",
|
||||||
|
"ROOT": "root",
|
||||||
"vocative": "vocative",
|
"vocative": "vocative",
|
||||||
"xcomp": "open clausal complement",
|
"xcomp": "open clausal complement",
|
||||||
# Dependency labels (German)
|
# Dependency labels (German)
|
||||||
|
|
22
spacy/kb.pyx
22
spacy/kb.pyx
|
@ -93,14 +93,14 @@ cdef class KnowledgeBase:
|
||||||
self.vocab = vocab
|
self.vocab = vocab
|
||||||
self._create_empty_vectors(dummy_hash=self.vocab.strings[""])
|
self._create_empty_vectors(dummy_hash=self.vocab.strings[""])
|
||||||
|
|
||||||
def initialize_entities(self, int64_t nr_entities):
|
def _initialize_entities(self, int64_t nr_entities):
|
||||||
self._entry_index = PreshMap(nr_entities + 1)
|
self._entry_index = PreshMap(nr_entities + 1)
|
||||||
self._entries = entry_vec(nr_entities + 1)
|
self._entries = entry_vec(nr_entities + 1)
|
||||||
|
|
||||||
def initialize_vectors(self, int64_t nr_entities):
|
def _initialize_vectors(self, int64_t nr_entities):
|
||||||
self._vectors_table = float_matrix(nr_entities + 1)
|
self._vectors_table = float_matrix(nr_entities + 1)
|
||||||
|
|
||||||
def initialize_aliases(self, int64_t nr_aliases):
|
def _initialize_aliases(self, int64_t nr_aliases):
|
||||||
self._alias_index = PreshMap(nr_aliases + 1)
|
self._alias_index = PreshMap(nr_aliases + 1)
|
||||||
self._aliases_table = alias_vec(nr_aliases + 1)
|
self._aliases_table = alias_vec(nr_aliases + 1)
|
||||||
|
|
||||||
|
@ -155,8 +155,8 @@ cdef class KnowledgeBase:
|
||||||
raise ValueError(Errors.E140)
|
raise ValueError(Errors.E140)
|
||||||
|
|
||||||
nr_entities = len(set(entity_list))
|
nr_entities = len(set(entity_list))
|
||||||
self.initialize_entities(nr_entities)
|
self._initialize_entities(nr_entities)
|
||||||
self.initialize_vectors(nr_entities)
|
self._initialize_vectors(nr_entities)
|
||||||
|
|
||||||
i = 0
|
i = 0
|
||||||
cdef KBEntryC entry
|
cdef KBEntryC entry
|
||||||
|
@ -388,9 +388,9 @@ cdef class KnowledgeBase:
|
||||||
nr_entities = header[0]
|
nr_entities = header[0]
|
||||||
nr_aliases = header[1]
|
nr_aliases = header[1]
|
||||||
entity_vector_length = header[2]
|
entity_vector_length = header[2]
|
||||||
self.initialize_entities(nr_entities)
|
self._initialize_entities(nr_entities)
|
||||||
self.initialize_vectors(nr_entities)
|
self._initialize_vectors(nr_entities)
|
||||||
self.initialize_aliases(nr_aliases)
|
self._initialize_aliases(nr_aliases)
|
||||||
self.entity_vector_length = entity_vector_length
|
self.entity_vector_length = entity_vector_length
|
||||||
|
|
||||||
def deserialize_vectors(b):
|
def deserialize_vectors(b):
|
||||||
|
@ -512,8 +512,8 @@ cdef class KnowledgeBase:
|
||||||
cdef int64_t entity_vector_length
|
cdef int64_t entity_vector_length
|
||||||
reader.read_header(&nr_entities, &entity_vector_length)
|
reader.read_header(&nr_entities, &entity_vector_length)
|
||||||
|
|
||||||
self.initialize_entities(nr_entities)
|
self._initialize_entities(nr_entities)
|
||||||
self.initialize_vectors(nr_entities)
|
self._initialize_vectors(nr_entities)
|
||||||
self.entity_vector_length = entity_vector_length
|
self.entity_vector_length = entity_vector_length
|
||||||
|
|
||||||
# STEP 1: load entity vectors
|
# STEP 1: load entity vectors
|
||||||
|
@ -552,7 +552,7 @@ cdef class KnowledgeBase:
|
||||||
# STEP 3: load aliases
|
# STEP 3: load aliases
|
||||||
cdef int64_t nr_aliases
|
cdef int64_t nr_aliases
|
||||||
reader.read_alias_length(&nr_aliases)
|
reader.read_alias_length(&nr_aliases)
|
||||||
self.initialize_aliases(nr_aliases)
|
self._initialize_aliases(nr_aliases)
|
||||||
|
|
||||||
cdef int64_t nr_candidates
|
cdef int64_t nr_candidates
|
||||||
cdef vector[int64_t] entry_indices
|
cdef vector[int64_t] entry_indices
|
||||||
|
|
|
@ -2,7 +2,8 @@ from .stop_words import STOP_WORDS
|
||||||
from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS
|
from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS
|
||||||
from .lex_attrs import LEX_ATTRS
|
from .lex_attrs import LEX_ATTRS
|
||||||
from ..tokenizer_exceptions import BASE_EXCEPTIONS
|
from ..tokenizer_exceptions import BASE_EXCEPTIONS
|
||||||
|
from ..punctuation import COMBINING_DIACRITICS_TOKENIZER_INFIXES
|
||||||
|
from ..punctuation import COMBINING_DIACRITICS_TOKENIZER_SUFFIXES
|
||||||
from ...language import Language, BaseDefaults
|
from ...language import Language, BaseDefaults
|
||||||
from ...attrs import LANG
|
from ...attrs import LANG
|
||||||
from ...util import update_exc
|
from ...util import update_exc
|
||||||
|
@ -16,6 +17,8 @@ class BulgarianDefaults(BaseDefaults):
|
||||||
|
|
||||||
stop_words = STOP_WORDS
|
stop_words = STOP_WORDS
|
||||||
tokenizer_exceptions = update_exc(BASE_EXCEPTIONS, TOKENIZER_EXCEPTIONS)
|
tokenizer_exceptions = update_exc(BASE_EXCEPTIONS, TOKENIZER_EXCEPTIONS)
|
||||||
|
suffixes = COMBINING_DIACRITICS_TOKENIZER_SUFFIXES
|
||||||
|
infixes = COMBINING_DIACRITICS_TOKENIZER_INFIXES
|
||||||
|
|
||||||
|
|
||||||
class Bulgarian(Language):
|
class Bulgarian(Language):
|
||||||
|
|
|
@ -258,6 +258,10 @@ ALPHA = group_chars(
|
||||||
ALPHA_LOWER = group_chars(_lower + _uncased)
|
ALPHA_LOWER = group_chars(_lower + _uncased)
|
||||||
ALPHA_UPPER = group_chars(_upper + _uncased)
|
ALPHA_UPPER = group_chars(_upper + _uncased)
|
||||||
|
|
||||||
|
_combining_diacritics = r"\u0300-\u036f"
|
||||||
|
|
||||||
|
COMBINING_DIACRITICS = _combining_diacritics
|
||||||
|
|
||||||
_units = (
|
_units = (
|
||||||
"km km² km³ m m² m³ dm dm² dm³ cm cm² cm³ mm mm² mm³ ha µm nm yd in ft "
|
"km km² km³ m m² m³ dm dm² dm³ cm cm² cm³ mm mm² mm³ ha µm nm yd in ft "
|
||||||
"kg g mg µg t lb oz m/s km/h kmh mph hPa Pa mbar mb MB kb KB gb GB tb "
|
"kg g mg µg t lb oz m/s km/h kmh mph hPa Pa mbar mb MB kb KB gb GB tb "
|
||||||
|
|
|
@ -35,7 +35,7 @@ for pron in ["i"]:
|
||||||
|
|
||||||
_exc[orth + "m"] = [
|
_exc[orth + "m"] = [
|
||||||
{ORTH: orth, NORM: pron},
|
{ORTH: orth, NORM: pron},
|
||||||
{ORTH: "m", "tenspect": 1, "number": 1},
|
{ORTH: "m"},
|
||||||
]
|
]
|
||||||
|
|
||||||
_exc[orth + "'ma"] = [
|
_exc[orth + "'ma"] = [
|
||||||
|
@ -139,26 +139,27 @@ for pron in ["he", "she", "it"]:
|
||||||
|
|
||||||
# W-words, relative pronouns, prepositions etc.
|
# W-words, relative pronouns, prepositions etc.
|
||||||
|
|
||||||
for word in [
|
for word, morph in [
|
||||||
"who",
|
("who", None),
|
||||||
"what",
|
("what", None),
|
||||||
"when",
|
("when", None),
|
||||||
"where",
|
("where", None),
|
||||||
"why",
|
("why", None),
|
||||||
"how",
|
("how", None),
|
||||||
"there",
|
("there", None),
|
||||||
"that",
|
("that", "Number=Sing|Person=3"),
|
||||||
"this",
|
("this", "Number=Sing|Person=3"),
|
||||||
"these",
|
("these", "Number=Plur|Person=3"),
|
||||||
"those",
|
("those", "Number=Plur|Person=3"),
|
||||||
]:
|
]:
|
||||||
for orth in [word, word.title()]:
|
for orth in [word, word.title()]:
|
||||||
_exc[orth + "'s"] = [
|
if morph != "Number=Plur|Person=3":
|
||||||
{ORTH: orth, NORM: word},
|
_exc[orth + "'s"] = [
|
||||||
{ORTH: "'s", NORM: "'s"},
|
{ORTH: orth, NORM: word},
|
||||||
]
|
{ORTH: "'s", NORM: "'s"},
|
||||||
|
]
|
||||||
|
|
||||||
_exc[orth + "s"] = [{ORTH: orth, NORM: word}, {ORTH: "s"}]
|
_exc[orth + "s"] = [{ORTH: orth, NORM: word}, {ORTH: "s"}]
|
||||||
|
|
||||||
_exc[orth + "'ll"] = [
|
_exc[orth + "'ll"] = [
|
||||||
{ORTH: orth, NORM: word},
|
{ORTH: orth, NORM: word},
|
||||||
|
@ -182,25 +183,26 @@ for word in [
|
||||||
{ORTH: "ve", NORM: "have"},
|
{ORTH: "ve", NORM: "have"},
|
||||||
]
|
]
|
||||||
|
|
||||||
_exc[orth + "'re"] = [
|
if morph != "Number=Sing|Person=3":
|
||||||
{ORTH: orth, NORM: word},
|
_exc[orth + "'re"] = [
|
||||||
{ORTH: "'re", NORM: "are"},
|
{ORTH: orth, NORM: word},
|
||||||
]
|
{ORTH: "'re", NORM: "are"},
|
||||||
|
]
|
||||||
|
|
||||||
_exc[orth + "re"] = [
|
_exc[orth + "re"] = [
|
||||||
{ORTH: orth, NORM: word},
|
{ORTH: orth, NORM: word},
|
||||||
{ORTH: "re", NORM: "are"},
|
{ORTH: "re", NORM: "are"},
|
||||||
]
|
]
|
||||||
|
|
||||||
_exc[orth + "'ve"] = [
|
_exc[orth + "'ve"] = [
|
||||||
{ORTH: orth, NORM: word},
|
{ORTH: orth, NORM: word},
|
||||||
{ORTH: "'ve"},
|
{ORTH: "'ve"},
|
||||||
]
|
]
|
||||||
|
|
||||||
_exc[orth + "ve"] = [
|
_exc[orth + "ve"] = [
|
||||||
{ORTH: orth},
|
{ORTH: orth},
|
||||||
{ORTH: "ve", NORM: "have"},
|
{ORTH: "ve", NORM: "have"},
|
||||||
]
|
]
|
||||||
|
|
||||||
_exc[orth + "'d"] = [
|
_exc[orth + "'d"] = [
|
||||||
{ORTH: orth, NORM: word},
|
{ORTH: orth, NORM: word},
|
||||||
|
|
|
@ -1,5 +1,5 @@
|
||||||
from .char_classes import LIST_PUNCT, LIST_ELLIPSES, LIST_QUOTES, LIST_CURRENCY
|
from .char_classes import LIST_PUNCT, LIST_ELLIPSES, LIST_QUOTES, LIST_CURRENCY
|
||||||
from .char_classes import LIST_ICONS, HYPHENS, CURRENCY, UNITS
|
from .char_classes import LIST_ICONS, HYPHENS, CURRENCY, UNITS, COMBINING_DIACRITICS
|
||||||
from .char_classes import CONCAT_QUOTES, ALPHA_LOWER, ALPHA_UPPER, ALPHA, PUNCT
|
from .char_classes import CONCAT_QUOTES, ALPHA_LOWER, ALPHA_UPPER, ALPHA, PUNCT
|
||||||
|
|
||||||
|
|
||||||
|
@ -44,3 +44,23 @@ TOKENIZER_INFIXES = (
|
||||||
r"(?<=[{a}0-9])[:<>=/](?=[{a}])".format(a=ALPHA),
|
r"(?<=[{a}0-9])[:<>=/](?=[{a}])".format(a=ALPHA),
|
||||||
]
|
]
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# Some languages e.g. written with the Cyrillic alphabet permit the use of diacritics
|
||||||
|
# to mark stressed syllables in words where stress is distinctive. Such languages
|
||||||
|
# should use the COMBINING_DIACRITICS... suffix and infix regex lists in
|
||||||
|
# place of the standard ones.
|
||||||
|
COMBINING_DIACRITICS_TOKENIZER_SUFFIXES = list(TOKENIZER_SUFFIXES) + [
|
||||||
|
r"(?<=[{a}][{d}])\.".format(a=ALPHA, d=COMBINING_DIACRITICS),
|
||||||
|
]
|
||||||
|
|
||||||
|
COMBINING_DIACRITICS_TOKENIZER_INFIXES = list(TOKENIZER_INFIXES) + [
|
||||||
|
r"(?<=[{al}][{d}])\.(?=[{au}{q}])".format(
|
||||||
|
al=ALPHA_LOWER, au=ALPHA_UPPER, q=CONCAT_QUOTES, d=COMBINING_DIACRITICS
|
||||||
|
),
|
||||||
|
r"(?<=[{a}][{d}]),(?=[{a}])".format(a=ALPHA, d=COMBINING_DIACRITICS),
|
||||||
|
r"(?<=[{a}][{d}])(?:{h})(?=[{a}])".format(
|
||||||
|
a=ALPHA, d=COMBINING_DIACRITICS, h=HYPHENS
|
||||||
|
),
|
||||||
|
r"(?<=[{a}][{d}])[:<>=/](?=[{a}])".format(a=ALPHA, d=COMBINING_DIACRITICS),
|
||||||
|
]
|
||||||
|
|
|
@ -5,6 +5,8 @@ from .stop_words import STOP_WORDS
|
||||||
from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS
|
from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS
|
||||||
from .lex_attrs import LEX_ATTRS
|
from .lex_attrs import LEX_ATTRS
|
||||||
from .lemmatizer import RussianLemmatizer
|
from .lemmatizer import RussianLemmatizer
|
||||||
|
from ..punctuation import COMBINING_DIACRITICS_TOKENIZER_INFIXES
|
||||||
|
from ..punctuation import COMBINING_DIACRITICS_TOKENIZER_SUFFIXES
|
||||||
from ...language import Language, BaseDefaults
|
from ...language import Language, BaseDefaults
|
||||||
|
|
||||||
|
|
||||||
|
@ -12,6 +14,8 @@ class RussianDefaults(BaseDefaults):
|
||||||
tokenizer_exceptions = TOKENIZER_EXCEPTIONS
|
tokenizer_exceptions = TOKENIZER_EXCEPTIONS
|
||||||
lex_attr_getters = LEX_ATTRS
|
lex_attr_getters = LEX_ATTRS
|
||||||
stop_words = STOP_WORDS
|
stop_words = STOP_WORDS
|
||||||
|
suffixes = COMBINING_DIACRITICS_TOKENIZER_SUFFIXES
|
||||||
|
infixes = COMBINING_DIACRITICS_TOKENIZER_INFIXES
|
||||||
|
|
||||||
|
|
||||||
class Russian(Language):
|
class Russian(Language):
|
||||||
|
|
|
@ -6,6 +6,8 @@ from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS
|
||||||
from .stop_words import STOP_WORDS
|
from .stop_words import STOP_WORDS
|
||||||
from .lex_attrs import LEX_ATTRS
|
from .lex_attrs import LEX_ATTRS
|
||||||
from .lemmatizer import UkrainianLemmatizer
|
from .lemmatizer import UkrainianLemmatizer
|
||||||
|
from ..punctuation import COMBINING_DIACRITICS_TOKENIZER_INFIXES
|
||||||
|
from ..punctuation import COMBINING_DIACRITICS_TOKENIZER_SUFFIXES
|
||||||
from ...language import Language, BaseDefaults
|
from ...language import Language, BaseDefaults
|
||||||
|
|
||||||
|
|
||||||
|
@ -13,6 +15,8 @@ class UkrainianDefaults(BaseDefaults):
|
||||||
tokenizer_exceptions = TOKENIZER_EXCEPTIONS
|
tokenizer_exceptions = TOKENIZER_EXCEPTIONS
|
||||||
lex_attr_getters = LEX_ATTRS
|
lex_attr_getters = LEX_ATTRS
|
||||||
stop_words = STOP_WORDS
|
stop_words = STOP_WORDS
|
||||||
|
suffixes = COMBINING_DIACRITICS_TOKENIZER_SUFFIXES
|
||||||
|
infixes = COMBINING_DIACRITICS_TOKENIZER_INFIXES
|
||||||
|
|
||||||
|
|
||||||
class Ukrainian(Language):
|
class Ukrainian(Language):
|
||||||
|
|
|
@ -1,4 +1,4 @@
|
||||||
from typing import Iterator, Optional, Any, Dict, Callable, Iterable
|
from typing import Iterator, Optional, Any, Dict, Callable, Iterable, Collection
|
||||||
from typing import Union, Tuple, List, Set, Pattern, Sequence
|
from typing import Union, Tuple, List, Set, Pattern, Sequence
|
||||||
from typing import NoReturn, TYPE_CHECKING, TypeVar, cast, overload
|
from typing import NoReturn, TYPE_CHECKING, TypeVar, cast, overload
|
||||||
|
|
||||||
|
@ -774,6 +774,9 @@ class Language:
|
||||||
name = name if name is not None else factory_name
|
name = name if name is not None else factory_name
|
||||||
if name in self.component_names:
|
if name in self.component_names:
|
||||||
raise ValueError(Errors.E007.format(name=name, opts=self.component_names))
|
raise ValueError(Errors.E007.format(name=name, opts=self.component_names))
|
||||||
|
# Overriding pipe name in the config is not supported and will be ignored.
|
||||||
|
if "name" in config:
|
||||||
|
warnings.warn(Warnings.W119.format(name_in_config=config.pop("name")))
|
||||||
if source is not None:
|
if source is not None:
|
||||||
# We're loading the component from a model. After loading the
|
# We're loading the component from a model. After loading the
|
||||||
# component, we know its real factory name
|
# component, we know its real factory name
|
||||||
|
@ -1087,16 +1090,21 @@ class Language:
|
||||||
)
|
)
|
||||||
return self.tokenizer(text)
|
return self.tokenizer(text)
|
||||||
|
|
||||||
def _ensure_doc(self, doc_like: Union[str, Doc]) -> Doc:
|
def _ensure_doc(self, doc_like: Union[str, Doc, bytes]) -> Doc:
|
||||||
"""Create a Doc if need be, or raise an error if the input is not a Doc or a string."""
|
"""Create a Doc if need be, or raise an error if the input is not
|
||||||
|
a Doc, string, or a byte array (generated by Doc.to_bytes())."""
|
||||||
if isinstance(doc_like, Doc):
|
if isinstance(doc_like, Doc):
|
||||||
return doc_like
|
return doc_like
|
||||||
if isinstance(doc_like, str):
|
if isinstance(doc_like, str):
|
||||||
return self.make_doc(doc_like)
|
return self.make_doc(doc_like)
|
||||||
raise ValueError(Errors.E866.format(type=type(doc_like)))
|
if isinstance(doc_like, bytes):
|
||||||
|
return Doc(self.vocab).from_bytes(doc_like)
|
||||||
|
raise ValueError(Errors.E1041.format(type=type(doc_like)))
|
||||||
|
|
||||||
def _ensure_doc_with_context(self, doc_like: Union[str, Doc], context: Any) -> Doc:
|
def _ensure_doc_with_context(
|
||||||
"""Create a Doc if need be and add as_tuples context, or raise an error if the input is not a Doc or a string."""
|
self, doc_like: Union[str, Doc, bytes], context: _AnyContext
|
||||||
|
) -> Doc:
|
||||||
|
"""Call _ensure_doc to generate a Doc and set its context object."""
|
||||||
doc = self._ensure_doc(doc_like)
|
doc = self._ensure_doc(doc_like)
|
||||||
doc._context = context
|
doc._context = context
|
||||||
return doc
|
return doc
|
||||||
|
@ -1516,7 +1524,6 @@ class Language:
|
||||||
|
|
||||||
DOCS: https://spacy.io/api/language#pipe
|
DOCS: https://spacy.io/api/language#pipe
|
||||||
"""
|
"""
|
||||||
# Handle texts with context as tuples
|
|
||||||
if as_tuples:
|
if as_tuples:
|
||||||
texts = cast(Iterable[Tuple[Union[str, Doc], _AnyContext]], texts)
|
texts = cast(Iterable[Tuple[Union[str, Doc], _AnyContext]], texts)
|
||||||
docs_with_contexts = (
|
docs_with_contexts = (
|
||||||
|
@ -1594,8 +1601,21 @@ class Language:
|
||||||
n_process: int,
|
n_process: int,
|
||||||
batch_size: int,
|
batch_size: int,
|
||||||
) -> Iterator[Doc]:
|
) -> Iterator[Doc]:
|
||||||
|
def prepare_input(
|
||||||
|
texts: Iterable[Union[str, Doc]]
|
||||||
|
) -> Iterable[Tuple[Union[str, bytes], _AnyContext]]:
|
||||||
|
# Serialize Doc inputs to bytes to avoid incurring pickling
|
||||||
|
# overhead when they are passed to child processes. Also yield
|
||||||
|
# any context objects they might have separately (as they are not serialized).
|
||||||
|
for doc_like in texts:
|
||||||
|
if isinstance(doc_like, Doc):
|
||||||
|
yield (doc_like.to_bytes(), cast(_AnyContext, doc_like._context))
|
||||||
|
else:
|
||||||
|
yield (doc_like, cast(_AnyContext, None))
|
||||||
|
|
||||||
|
serialized_texts_with_ctx = prepare_input(texts) # type: ignore
|
||||||
# raw_texts is used later to stop iteration.
|
# raw_texts is used later to stop iteration.
|
||||||
texts, raw_texts = itertools.tee(texts)
|
texts, raw_texts = itertools.tee(serialized_texts_with_ctx) # type: ignore
|
||||||
# for sending texts to worker
|
# for sending texts to worker
|
||||||
texts_q: List[mp.Queue] = [mp.Queue() for _ in range(n_process)]
|
texts_q: List[mp.Queue] = [mp.Queue() for _ in range(n_process)]
|
||||||
# for receiving byte-encoded docs from worker
|
# for receiving byte-encoded docs from worker
|
||||||
|
@ -1615,7 +1635,13 @@ class Language:
|
||||||
procs = [
|
procs = [
|
||||||
mp.Process(
|
mp.Process(
|
||||||
target=_apply_pipes,
|
target=_apply_pipes,
|
||||||
args=(self._ensure_doc, pipes, rch, sch, Underscore.get_state()),
|
args=(
|
||||||
|
self._ensure_doc_with_context,
|
||||||
|
pipes,
|
||||||
|
rch,
|
||||||
|
sch,
|
||||||
|
Underscore.get_state(),
|
||||||
|
),
|
||||||
)
|
)
|
||||||
for rch, sch in zip(texts_q, bytedocs_send_ch)
|
for rch, sch in zip(texts_q, bytedocs_send_ch)
|
||||||
]
|
]
|
||||||
|
@ -1628,12 +1654,12 @@ class Language:
|
||||||
recv.recv() for recv in cycle(bytedocs_recv_ch)
|
recv.recv() for recv in cycle(bytedocs_recv_ch)
|
||||||
)
|
)
|
||||||
try:
|
try:
|
||||||
for i, (_, (byte_doc, byte_context, byte_error)) in enumerate(
|
for i, (_, (byte_doc, context, byte_error)) in enumerate(
|
||||||
zip(raw_texts, byte_tuples), 1
|
zip(raw_texts, byte_tuples), 1
|
||||||
):
|
):
|
||||||
if byte_doc is not None:
|
if byte_doc is not None:
|
||||||
doc = Doc(self.vocab).from_bytes(byte_doc)
|
doc = Doc(self.vocab).from_bytes(byte_doc)
|
||||||
doc._context = byte_context
|
doc._context = context
|
||||||
yield doc
|
yield doc
|
||||||
elif byte_error is not None:
|
elif byte_error is not None:
|
||||||
error = srsly.msgpack_loads(byte_error)
|
error = srsly.msgpack_loads(byte_error)
|
||||||
|
@ -1668,6 +1694,7 @@ class Language:
|
||||||
*,
|
*,
|
||||||
vocab: Union[Vocab, bool] = True,
|
vocab: Union[Vocab, bool] = True,
|
||||||
disable: Iterable[str] = SimpleFrozenList(),
|
disable: Iterable[str] = SimpleFrozenList(),
|
||||||
|
enable: Iterable[str] = SimpleFrozenList(),
|
||||||
exclude: Iterable[str] = SimpleFrozenList(),
|
exclude: Iterable[str] = SimpleFrozenList(),
|
||||||
meta: Dict[str, Any] = SimpleFrozenDict(),
|
meta: Dict[str, Any] = SimpleFrozenDict(),
|
||||||
auto_fill: bool = True,
|
auto_fill: bool = True,
|
||||||
|
@ -1682,6 +1709,8 @@ class Language:
|
||||||
disable (Iterable[str]): Names of pipeline components to disable.
|
disable (Iterable[str]): Names of pipeline components to disable.
|
||||||
Disabled pipes will be loaded but they won't be run unless you
|
Disabled pipes will be loaded but they won't be run unless you
|
||||||
explicitly enable them by calling nlp.enable_pipe.
|
explicitly enable them by calling nlp.enable_pipe.
|
||||||
|
enable (Iterable[str]): Names of pipeline components to enable. All other
|
||||||
|
pipes will be disabled (and can be enabled using `nlp.enable_pipe`).
|
||||||
exclude (Iterable[str]): Names of pipeline components to exclude.
|
exclude (Iterable[str]): Names of pipeline components to exclude.
|
||||||
Excluded components won't be loaded.
|
Excluded components won't be loaded.
|
||||||
meta (Dict[str, Any]): Meta overrides for nlp.meta.
|
meta (Dict[str, Any]): Meta overrides for nlp.meta.
|
||||||
|
@ -1835,8 +1864,15 @@ class Language:
|
||||||
# Restore the original vocab after sourcing if necessary
|
# Restore the original vocab after sourcing if necessary
|
||||||
if vocab_b is not None:
|
if vocab_b is not None:
|
||||||
nlp.vocab.from_bytes(vocab_b)
|
nlp.vocab.from_bytes(vocab_b)
|
||||||
disabled_pipes = [*config["nlp"]["disabled"], *disable]
|
|
||||||
|
# Resolve disabled/enabled settings.
|
||||||
|
disabled_pipes = cls._resolve_component_status(
|
||||||
|
[*config["nlp"]["disabled"], *disable],
|
||||||
|
[*config["nlp"].get("enabled", []), *enable],
|
||||||
|
config["nlp"]["pipeline"],
|
||||||
|
)
|
||||||
nlp._disabled = set(p for p in disabled_pipes if p not in exclude)
|
nlp._disabled = set(p for p in disabled_pipes if p not in exclude)
|
||||||
|
|
||||||
nlp.batch_size = config["nlp"]["batch_size"]
|
nlp.batch_size = config["nlp"]["batch_size"]
|
||||||
nlp.config = filled if auto_fill else config
|
nlp.config = filled if auto_fill else config
|
||||||
if after_pipeline_creation is not None:
|
if after_pipeline_creation is not None:
|
||||||
|
@ -1988,6 +2024,42 @@ class Language:
|
||||||
serializers["vocab"] = lambda p: self.vocab.to_disk(p, exclude=exclude)
|
serializers["vocab"] = lambda p: self.vocab.to_disk(p, exclude=exclude)
|
||||||
util.to_disk(path, serializers, exclude)
|
util.to_disk(path, serializers, exclude)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _resolve_component_status(
|
||||||
|
disable: Iterable[str], enable: Iterable[str], pipe_names: Collection[str]
|
||||||
|
) -> Tuple[str, ...]:
|
||||||
|
"""Derives whether (1) `disable` and `enable` values are consistent and (2)
|
||||||
|
resolves those to a single set of disabled components. Raises an error in
|
||||||
|
case of inconsistency.
|
||||||
|
|
||||||
|
disable (Iterable[str]): Names of components or serialization fields to disable.
|
||||||
|
enable (Iterable[str]): Names of pipeline components to enable.
|
||||||
|
pipe_names (Iterable[str]): Names of all pipeline components.
|
||||||
|
|
||||||
|
RETURNS (Tuple[str, ...]): Names of components to exclude from pipeline w.r.t.
|
||||||
|
specified includes and excludes.
|
||||||
|
"""
|
||||||
|
|
||||||
|
if disable is not None and isinstance(disable, str):
|
||||||
|
disable = [disable]
|
||||||
|
to_disable = disable
|
||||||
|
|
||||||
|
if enable:
|
||||||
|
to_disable = [
|
||||||
|
pipe_name for pipe_name in pipe_names if pipe_name not in enable
|
||||||
|
]
|
||||||
|
if disable and disable != to_disable:
|
||||||
|
raise ValueError(
|
||||||
|
Errors.E1042.format(
|
||||||
|
arg1="enable",
|
||||||
|
arg2="disable",
|
||||||
|
arg1_values=enable,
|
||||||
|
arg2_values=disable,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
return tuple(to_disable)
|
||||||
|
|
||||||
def from_disk(
|
def from_disk(
|
||||||
self,
|
self,
|
||||||
path: Union[str, Path],
|
path: Union[str, Path],
|
||||||
|
@ -2160,7 +2232,7 @@ def _copy_examples(examples: Iterable[Example]) -> List[Example]:
|
||||||
|
|
||||||
|
|
||||||
def _apply_pipes(
|
def _apply_pipes(
|
||||||
ensure_doc: Callable[[Union[str, Doc]], Doc],
|
ensure_doc: Callable[[Union[str, Doc, bytes], _AnyContext], Doc],
|
||||||
pipes: Iterable[Callable[..., Iterator[Doc]]],
|
pipes: Iterable[Callable[..., Iterator[Doc]]],
|
||||||
receiver,
|
receiver,
|
||||||
sender,
|
sender,
|
||||||
|
@ -2181,17 +2253,19 @@ def _apply_pipes(
|
||||||
Underscore.load_state(underscore_state)
|
Underscore.load_state(underscore_state)
|
||||||
while True:
|
while True:
|
||||||
try:
|
try:
|
||||||
texts = receiver.get()
|
texts_with_ctx = receiver.get()
|
||||||
docs = (ensure_doc(text) for text in texts)
|
docs = (
|
||||||
|
ensure_doc(doc_like, context) for doc_like, context in texts_with_ctx
|
||||||
|
)
|
||||||
for pipe in pipes:
|
for pipe in pipes:
|
||||||
docs = pipe(docs) # type: ignore[arg-type, assignment]
|
docs = pipe(docs) # type: ignore[arg-type, assignment]
|
||||||
# Connection does not accept unpickable objects, so send list.
|
# Connection does not accept unpickable objects, so send list.
|
||||||
byte_docs = [(doc.to_bytes(), doc._context, None) for doc in docs]
|
byte_docs = [(doc.to_bytes(), doc._context, None) for doc in docs]
|
||||||
padding = [(None, None, None)] * (len(texts) - len(byte_docs))
|
padding = [(None, None, None)] * (len(texts_with_ctx) - len(byte_docs))
|
||||||
sender.send(byte_docs + padding) # type: ignore[operator]
|
sender.send(byte_docs + padding) # type: ignore[operator]
|
||||||
except Exception:
|
except Exception:
|
||||||
error_msg = [(None, None, srsly.msgpack_dumps(traceback.format_exc()))]
|
error_msg = [(None, None, srsly.msgpack_dumps(traceback.format_exc()))]
|
||||||
padding = [(None, None, None)] * (len(texts) - 1)
|
padding = [(None, None, None)] * (len(texts_with_ctx) - 1)
|
||||||
sender.send(error_msg + padding)
|
sender.send(error_msg + padding)
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -85,7 +85,7 @@ class Table(OrderedDict):
|
||||||
value: The value to set.
|
value: The value to set.
|
||||||
"""
|
"""
|
||||||
key = get_string_id(key)
|
key = get_string_id(key)
|
||||||
OrderedDict.__setitem__(self, key, value)
|
OrderedDict.__setitem__(self, key, value) # type:ignore[assignment]
|
||||||
self.bloom.add(key)
|
self.bloom.add(key)
|
||||||
|
|
||||||
def set(self, key: Union[str, int], value: Any) -> None:
|
def set(self, key: Union[str, int], value: Any) -> None:
|
||||||
|
@ -104,7 +104,7 @@ class Table(OrderedDict):
|
||||||
RETURNS: The value.
|
RETURNS: The value.
|
||||||
"""
|
"""
|
||||||
key = get_string_id(key)
|
key = get_string_id(key)
|
||||||
return OrderedDict.__getitem__(self, key)
|
return OrderedDict.__getitem__(self, key) # type:ignore[index]
|
||||||
|
|
||||||
def get(self, key: Union[str, int], default: Optional[Any] = None) -> Any:
|
def get(self, key: Union[str, int], default: Optional[Any] = None) -> Any:
|
||||||
"""Get the value for a given key. String keys will be hashed.
|
"""Get the value for a given key. String keys will be hashed.
|
||||||
|
@ -114,7 +114,7 @@ class Table(OrderedDict):
|
||||||
RETURNS: The value.
|
RETURNS: The value.
|
||||||
"""
|
"""
|
||||||
key = get_string_id(key)
|
key = get_string_id(key)
|
||||||
return OrderedDict.get(self, key, default)
|
return OrderedDict.get(self, key, default) # type:ignore[arg-type]
|
||||||
|
|
||||||
def __contains__(self, key: Union[str, int]) -> bool: # type: ignore[override]
|
def __contains__(self, key: Union[str, int]) -> bool: # type: ignore[override]
|
||||||
"""Check whether a key is in the table. String keys will be hashed.
|
"""Check whether a key is in the table. String keys will be hashed.
|
||||||
|
|
|
@ -82,6 +82,10 @@ cdef class DependencyMatcher:
|
||||||
"$-": self._imm_left_sib,
|
"$-": self._imm_left_sib,
|
||||||
"$++": self._right_sib,
|
"$++": self._right_sib,
|
||||||
"$--": self._left_sib,
|
"$--": self._left_sib,
|
||||||
|
">++": self._right_child,
|
||||||
|
">--": self._left_child,
|
||||||
|
"<++": self._right_parent,
|
||||||
|
"<--": self._left_parent,
|
||||||
}
|
}
|
||||||
|
|
||||||
def __reduce__(self):
|
def __reduce__(self):
|
||||||
|
@ -423,6 +427,22 @@ cdef class DependencyMatcher:
|
||||||
def _left_sib(self, doc, node):
|
def _left_sib(self, doc, node):
|
||||||
return [doc[child.i] for child in doc[node].head.children if child.i < node]
|
return [doc[child.i] for child in doc[node].head.children if child.i < node]
|
||||||
|
|
||||||
|
def _right_child(self, doc, node):
|
||||||
|
return [doc[child.i] for child in doc[node].children if child.i > node]
|
||||||
|
|
||||||
|
def _left_child(self, doc, node):
|
||||||
|
return [doc[child.i] for child in doc[node].children if child.i < node]
|
||||||
|
|
||||||
|
def _right_parent(self, doc, node):
|
||||||
|
if doc[node].head.i > node:
|
||||||
|
return [doc[node].head]
|
||||||
|
return []
|
||||||
|
|
||||||
|
def _left_parent(self, doc, node):
|
||||||
|
if doc[node].head.i < node:
|
||||||
|
return [doc[node].head]
|
||||||
|
return []
|
||||||
|
|
||||||
def _normalize_key(self, key):
|
def _normalize_key(self, key):
|
||||||
if isinstance(key, str):
|
if isinstance(key, str):
|
||||||
return self.vocab.strings.add(key)
|
return self.vocab.strings.add(key)
|
||||||
|
|
|
@ -86,10 +86,14 @@ cdef class Matcher:
|
||||||
is a dictionary mapping attribute IDs to values, and optionally a
|
is a dictionary mapping attribute IDs to values, and optionally a
|
||||||
quantifier operator under the key "op". The available quantifiers are:
|
quantifier operator under the key "op". The available quantifiers are:
|
||||||
|
|
||||||
'!': Negate the pattern, by requiring it to match exactly 0 times.
|
'!': Negate the pattern, by requiring it to match exactly 0 times.
|
||||||
'?': Make the pattern optional, by allowing it to match 0 or 1 times.
|
'?': Make the pattern optional, by allowing it to match 0 or 1 times.
|
||||||
'+': Require the pattern to match 1 or more times.
|
'+': Require the pattern to match 1 or more times.
|
||||||
'*': Allow the pattern to zero or more times.
|
'*': Allow the pattern to zero or more times.
|
||||||
|
'{n}': Require the pattern to match exactly _n_ times.
|
||||||
|
'{n,m}': Require the pattern to match at least _n_ but not more than _m_ times.
|
||||||
|
'{n,}': Require the pattern to match at least _n_ times.
|
||||||
|
'{,m}': Require the pattern to match at most _m_ times.
|
||||||
|
|
||||||
The + and * operators return all possible matches (not just the greedy
|
The + and * operators return all possible matches (not just the greedy
|
||||||
ones). However, the "greedy" argument can filter the final matches
|
ones). However, the "greedy" argument can filter the final matches
|
||||||
|
@ -786,6 +790,7 @@ def _preprocess_pattern(token_specs, vocab, extensions_table, extra_predicates):
|
||||||
def _get_attr_values(spec, string_store):
|
def _get_attr_values(spec, string_store):
|
||||||
attr_values = []
|
attr_values = []
|
||||||
for attr, value in spec.items():
|
for attr, value in spec.items():
|
||||||
|
input_attr = attr
|
||||||
if isinstance(attr, str):
|
if isinstance(attr, str):
|
||||||
attr = attr.upper()
|
attr = attr.upper()
|
||||||
if attr == '_':
|
if attr == '_':
|
||||||
|
@ -814,7 +819,7 @@ def _get_attr_values(spec, string_store):
|
||||||
attr_values.append((attr, value))
|
attr_values.append((attr, value))
|
||||||
else:
|
else:
|
||||||
# should be caught in validation
|
# should be caught in validation
|
||||||
raise ValueError(Errors.E152.format(attr=attr))
|
raise ValueError(Errors.E152.format(attr=input_attr))
|
||||||
return attr_values
|
return attr_values
|
||||||
|
|
||||||
|
|
||||||
|
@ -1003,8 +1008,29 @@ def _get_operators(spec):
|
||||||
return (ONE,)
|
return (ONE,)
|
||||||
elif spec["OP"] in lookup:
|
elif spec["OP"] in lookup:
|
||||||
return lookup[spec["OP"]]
|
return lookup[spec["OP"]]
|
||||||
|
#Min_max {n,m}
|
||||||
|
elif spec["OP"].startswith("{") and spec["OP"].endswith("}"):
|
||||||
|
# {n} --> {n,n} exactly n ONE,(n)
|
||||||
|
# {n,m}--> {n,m} min of n, max of m ONE,(n),ZERO_ONE,(m)
|
||||||
|
# {,m} --> {0,m} min of zero, max of m ZERO_ONE,(m)
|
||||||
|
# {n,} --> {n,∞} min of n, max of inf ONE,(n),ZERO_PLUS
|
||||||
|
|
||||||
|
min_max = spec["OP"][1:-1]
|
||||||
|
min_max = min_max if "," in min_max else f"{min_max},{min_max}"
|
||||||
|
n, m = min_max.split(",")
|
||||||
|
|
||||||
|
#1. Either n or m is a blank string and the other is numeric -->isdigit
|
||||||
|
#2. Both are numeric and n <= m
|
||||||
|
if (not n.isdecimal() and not m.isdecimal()) or (n.isdecimal() and m.isdecimal() and int(n) > int(m)):
|
||||||
|
keys = ", ".join(lookup.keys()) + ", {n}, {n,m}, {n,}, {,m} where n and m are integers and n <= m "
|
||||||
|
raise ValueError(Errors.E011.format(op=spec["OP"], opts=keys))
|
||||||
|
|
||||||
|
# if n is empty string, zero would be used
|
||||||
|
head = tuple(ONE for __ in range(int(n or 0)))
|
||||||
|
tail = tuple(ZERO_ONE for __ in range(int(m) - int(n or 0))) if m else (ZERO_PLUS,)
|
||||||
|
return head + tail
|
||||||
else:
|
else:
|
||||||
keys = ", ".join(lookup.keys())
|
keys = ", ".join(lookup.keys()) + ", {n}, {n,m}, {n,}, {,m} where n and m are integers and n <= m "
|
||||||
raise ValueError(Errors.E011.format(op=spec["OP"], opts=keys))
|
raise ValueError(Errors.E011.format(op=spec["OP"], opts=keys))
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -118,6 +118,8 @@ cdef class PhraseMatcher:
|
||||||
# if token is not found, break out of the loop
|
# if token is not found, break out of the loop
|
||||||
current_node = NULL
|
current_node = NULL
|
||||||
break
|
break
|
||||||
|
path_nodes.push_back(current_node)
|
||||||
|
path_keys.push_back(self._terminal_hash)
|
||||||
# remove the tokens from trie node if there are no other
|
# remove the tokens from trie node if there are no other
|
||||||
# keywords with them
|
# keywords with them
|
||||||
result = map_get(current_node, self._terminal_hash)
|
result = map_get(current_node, self._terminal_hash)
|
||||||
|
|
|
@ -22,9 +22,15 @@ def forward(model, X, is_train):
|
||||||
nP = model.get_dim("nP")
|
nP = model.get_dim("nP")
|
||||||
nI = model.get_dim("nI")
|
nI = model.get_dim("nI")
|
||||||
W = model.get_param("W")
|
W = model.get_param("W")
|
||||||
Yf = model.ops.gemm(X, W.reshape((nF * nO * nP, nI)), trans2=True)
|
# Preallocate array for layer output, including padding.
|
||||||
|
Yf = model.ops.alloc2f(X.shape[0] + 1, nF * nO * nP, zeros=False)
|
||||||
|
model.ops.gemm(X, W.reshape((nF * nO * nP, nI)), trans2=True, out=Yf[1:])
|
||||||
Yf = Yf.reshape((Yf.shape[0], nF, nO, nP))
|
Yf = Yf.reshape((Yf.shape[0], nF, nO, nP))
|
||||||
Yf = model.ops.xp.vstack((model.get_param("pad"), Yf))
|
|
||||||
|
# Set padding. Padding has shape (1, nF, nO, nP). Unfortunately, we cannot
|
||||||
|
# change its shape to (nF, nO, nP) without breaking existing models. So
|
||||||
|
# we'll squeeze the first dimension here.
|
||||||
|
Yf[0] = model.ops.xp.squeeze(model.get_param("pad"), 0)
|
||||||
|
|
||||||
def backward(dY_ids):
|
def backward(dY_ids):
|
||||||
# This backprop is particularly tricky, because we get back a different
|
# This backprop is particularly tricky, because we get back a different
|
||||||
|
|
|
@ -1,9 +1,14 @@
|
||||||
from functools import partial
|
from typing import Type, Callable, Dict, TYPE_CHECKING, List, Optional, Set
|
||||||
from typing import Type, Callable, TYPE_CHECKING
|
import functools
|
||||||
|
import inspect
|
||||||
|
import types
|
||||||
|
import warnings
|
||||||
|
|
||||||
from thinc.layers import with_nvtx_range
|
from thinc.layers import with_nvtx_range
|
||||||
from thinc.model import Model, wrap_model_recursive
|
from thinc.model import Model, wrap_model_recursive
|
||||||
|
from thinc.util import use_nvtx_range
|
||||||
|
|
||||||
|
from ..errors import Warnings
|
||||||
from ..util import registry
|
from ..util import registry
|
||||||
|
|
||||||
if TYPE_CHECKING:
|
if TYPE_CHECKING:
|
||||||
|
@ -11,29 +16,106 @@ if TYPE_CHECKING:
|
||||||
from ..language import Language # noqa: F401
|
from ..language import Language # noqa: F401
|
||||||
|
|
||||||
|
|
||||||
@registry.callbacks("spacy.models_with_nvtx_range.v1")
|
DEFAULT_NVTX_ANNOTATABLE_PIPE_METHODS = [
|
||||||
def create_models_with_nvtx_range(
|
"pipe",
|
||||||
forward_color: int = -1, backprop_color: int = -1
|
"predict",
|
||||||
) -> Callable[["Language"], "Language"]:
|
"set_annotations",
|
||||||
def models_with_nvtx_range(nlp):
|
"update",
|
||||||
pipes = [
|
"rehearse",
|
||||||
pipe
|
"get_loss",
|
||||||
for _, pipe in nlp.components
|
"initialize",
|
||||||
if hasattr(pipe, "is_trainable") and pipe.is_trainable
|
"begin_update",
|
||||||
]
|
"finish_update",
|
||||||
|
"update",
|
||||||
|
]
|
||||||
|
|
||||||
# We need process all models jointly to avoid wrapping callbacks twice.
|
|
||||||
models = Model(
|
|
||||||
"wrap_with_nvtx_range",
|
|
||||||
forward=lambda model, X, is_train: ...,
|
|
||||||
layers=[pipe.model for pipe in pipes],
|
|
||||||
)
|
|
||||||
|
|
||||||
for node in models.walk():
|
def models_with_nvtx_range(nlp, forward_color: int, backprop_color: int):
|
||||||
|
pipes = [
|
||||||
|
pipe
|
||||||
|
for _, pipe in nlp.components
|
||||||
|
if hasattr(pipe, "is_trainable") and pipe.is_trainable
|
||||||
|
]
|
||||||
|
|
||||||
|
seen_models: Set[int] = set()
|
||||||
|
for pipe in pipes:
|
||||||
|
for node in pipe.model.walk():
|
||||||
|
if id(node) in seen_models:
|
||||||
|
continue
|
||||||
|
seen_models.add(id(node))
|
||||||
with_nvtx_range(
|
with_nvtx_range(
|
||||||
node, forward_color=forward_color, backprop_color=backprop_color
|
node, forward_color=forward_color, backprop_color=backprop_color
|
||||||
)
|
)
|
||||||
|
|
||||||
|
return nlp
|
||||||
|
|
||||||
|
|
||||||
|
@registry.callbacks("spacy.models_with_nvtx_range.v1")
|
||||||
|
def create_models_with_nvtx_range(
|
||||||
|
forward_color: int = -1, backprop_color: int = -1
|
||||||
|
) -> Callable[["Language"], "Language"]:
|
||||||
|
return functools.partial(
|
||||||
|
models_with_nvtx_range,
|
||||||
|
forward_color=forward_color,
|
||||||
|
backprop_color=backprop_color,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def nvtx_range_wrapper_for_pipe_method(self, func, *args, **kwargs):
|
||||||
|
if isinstance(func, functools.partial):
|
||||||
|
return func(*args, **kwargs)
|
||||||
|
else:
|
||||||
|
with use_nvtx_range(f"{self.name} {func.__name__}"):
|
||||||
|
return func(*args, **kwargs)
|
||||||
|
|
||||||
|
|
||||||
|
def pipes_with_nvtx_range(
|
||||||
|
nlp, additional_pipe_functions: Optional[Dict[str, List[str]]]
|
||||||
|
):
|
||||||
|
for _, pipe in nlp.components:
|
||||||
|
if additional_pipe_functions:
|
||||||
|
extra_funcs = additional_pipe_functions.get(pipe.name, [])
|
||||||
|
else:
|
||||||
|
extra_funcs = []
|
||||||
|
|
||||||
|
for name in DEFAULT_NVTX_ANNOTATABLE_PIPE_METHODS + extra_funcs:
|
||||||
|
func = getattr(pipe, name, None)
|
||||||
|
if func is None:
|
||||||
|
if name in extra_funcs:
|
||||||
|
warnings.warn(Warnings.W121.format(method=name, pipe=pipe.name))
|
||||||
|
continue
|
||||||
|
|
||||||
|
wrapped_func = functools.partial(
|
||||||
|
types.MethodType(nvtx_range_wrapper_for_pipe_method, pipe), func
|
||||||
|
)
|
||||||
|
|
||||||
|
# Try to preserve the original function signature.
|
||||||
|
try:
|
||||||
|
wrapped_func.__signature__ = inspect.signature(func) # type: ignore
|
||||||
|
except:
|
||||||
|
pass
|
||||||
|
|
||||||
|
try:
|
||||||
|
setattr(
|
||||||
|
pipe,
|
||||||
|
name,
|
||||||
|
wrapped_func,
|
||||||
|
)
|
||||||
|
except AttributeError:
|
||||||
|
warnings.warn(Warnings.W122.format(method=name, pipe=pipe.name))
|
||||||
|
|
||||||
|
return nlp
|
||||||
|
|
||||||
|
|
||||||
|
@registry.callbacks("spacy.models_and_pipes_with_nvtx_range.v1")
|
||||||
|
def create_models_and_pipes_with_nvtx_range(
|
||||||
|
forward_color: int = -1,
|
||||||
|
backprop_color: int = -1,
|
||||||
|
additional_pipe_functions: Optional[Dict[str, List[str]]] = None,
|
||||||
|
) -> Callable[["Language"], "Language"]:
|
||||||
|
def inner(nlp):
|
||||||
|
nlp = models_with_nvtx_range(nlp, forward_color, backprop_color)
|
||||||
|
nlp = pipes_with_nvtx_range(nlp, additional_pipe_functions)
|
||||||
return nlp
|
return nlp
|
||||||
|
|
||||||
return models_with_nvtx_range
|
return inner
|
||||||
|
|
|
@ -23,7 +23,7 @@ def build_nel_encoder(
|
||||||
((tok2vec >> list2ragged()) & build_span_maker())
|
((tok2vec >> list2ragged()) & build_span_maker())
|
||||||
>> extract_spans()
|
>> extract_spans()
|
||||||
>> reduce_mean()
|
>> reduce_mean()
|
||||||
>> residual(Maxout(nO=token_width, nI=token_width, nP=2, dropout=0.0)) # type: ignore[arg-type]
|
>> residual(Maxout(nO=token_width, nI=token_width, nP=2, dropout=0.0)) # type: ignore
|
||||||
>> output_layer
|
>> output_layer
|
||||||
)
|
)
|
||||||
model.set_ref("output_layer", output_layer)
|
model.set_ref("output_layer", output_layer)
|
||||||
|
|
|
@ -72,7 +72,7 @@ def build_tb_parser_model(
|
||||||
t2v_width = tok2vec.get_dim("nO") if tok2vec.has_dim("nO") else None
|
t2v_width = tok2vec.get_dim("nO") if tok2vec.has_dim("nO") else None
|
||||||
tok2vec = chain(
|
tok2vec = chain(
|
||||||
tok2vec,
|
tok2vec,
|
||||||
cast(Model[List["Floats2d"], Floats2d], list2array()),
|
list2array(),
|
||||||
Linear(hidden_width, t2v_width),
|
Linear(hidden_width, t2v_width),
|
||||||
)
|
)
|
||||||
tok2vec.set_dim("nO", hidden_width)
|
tok2vec.set_dim("nO", hidden_width)
|
||||||
|
|
|
@ -1,5 +1,5 @@
|
||||||
|
from typing import Optional, List, cast
|
||||||
from functools import partial
|
from functools import partial
|
||||||
from typing import Optional, List
|
|
||||||
|
|
||||||
from thinc.types import Floats2d
|
from thinc.types import Floats2d
|
||||||
from thinc.api import Model, reduce_mean, Linear, list2ragged, Logistic
|
from thinc.api import Model, reduce_mean, Linear, list2ragged, Logistic
|
||||||
|
@ -59,7 +59,8 @@ def build_simple_cnn_text_classifier(
|
||||||
resizable_layer=resizable_layer,
|
resizable_layer=resizable_layer,
|
||||||
)
|
)
|
||||||
model.set_ref("tok2vec", tok2vec)
|
model.set_ref("tok2vec", tok2vec)
|
||||||
model.set_dim("nO", nO) # type: ignore # TODO: remove type ignore once Thinc has been updated
|
if nO is not None:
|
||||||
|
model.set_dim("nO", cast(int, nO))
|
||||||
model.attrs["multi_label"] = not exclusive_classes
|
model.attrs["multi_label"] = not exclusive_classes
|
||||||
return model
|
return model
|
||||||
|
|
||||||
|
@ -85,7 +86,7 @@ def build_bow_text_classifier(
|
||||||
if not no_output_layer:
|
if not no_output_layer:
|
||||||
fill_defaults["b"] = NEG_VALUE
|
fill_defaults["b"] = NEG_VALUE
|
||||||
output_layer = softmax_activation() if exclusive_classes else Logistic()
|
output_layer = softmax_activation() if exclusive_classes else Logistic()
|
||||||
resizable_layer = resizable( # type: ignore[var-annotated]
|
resizable_layer: Model[Floats2d, Floats2d] = resizable(
|
||||||
sparse_linear,
|
sparse_linear,
|
||||||
resize_layer=partial(resize_linear_weighted, fill_defaults=fill_defaults),
|
resize_layer=partial(resize_linear_weighted, fill_defaults=fill_defaults),
|
||||||
)
|
)
|
||||||
|
@ -93,7 +94,8 @@ def build_bow_text_classifier(
|
||||||
model = with_cpu(model, model.ops)
|
model = with_cpu(model, model.ops)
|
||||||
if output_layer:
|
if output_layer:
|
||||||
model = model >> with_cpu(output_layer, output_layer.ops)
|
model = model >> with_cpu(output_layer, output_layer.ops)
|
||||||
model.set_dim("nO", nO) # type: ignore[arg-type]
|
if nO is not None:
|
||||||
|
model.set_dim("nO", cast(int, nO))
|
||||||
model.set_ref("output_layer", sparse_linear)
|
model.set_ref("output_layer", sparse_linear)
|
||||||
model.attrs["multi_label"] = not exclusive_classes
|
model.attrs["multi_label"] = not exclusive_classes
|
||||||
model.attrs["resize_output"] = partial(
|
model.attrs["resize_output"] = partial(
|
||||||
|
@ -129,8 +131,8 @@ def build_text_classifier_v2(
|
||||||
output_layer = Linear(nO=nO, nI=nO_double) >> Logistic()
|
output_layer = Linear(nO=nO, nI=nO_double) >> Logistic()
|
||||||
model = (linear_model | cnn_model) >> output_layer
|
model = (linear_model | cnn_model) >> output_layer
|
||||||
model.set_ref("tok2vec", tok2vec)
|
model.set_ref("tok2vec", tok2vec)
|
||||||
if model.has_dim("nO") is not False:
|
if model.has_dim("nO") is not False and nO is not None:
|
||||||
model.set_dim("nO", nO) # type: ignore[arg-type]
|
model.set_dim("nO", cast(int, nO))
|
||||||
model.set_ref("output_layer", linear_model.get_ref("output_layer"))
|
model.set_ref("output_layer", linear_model.get_ref("output_layer"))
|
||||||
model.set_ref("attention_layer", attention_layer)
|
model.set_ref("attention_layer", attention_layer)
|
||||||
model.set_ref("maxout_layer", maxout_layer)
|
model.set_ref("maxout_layer", maxout_layer)
|
||||||
|
@ -164,7 +166,7 @@ def build_text_classifier_lowdata(
|
||||||
>> list2ragged()
|
>> list2ragged()
|
||||||
>> ParametricAttention(width)
|
>> ParametricAttention(width)
|
||||||
>> reduce_sum()
|
>> reduce_sum()
|
||||||
>> residual(Relu(width, width)) ** 2 # type: ignore[arg-type]
|
>> residual(Relu(width, width)) ** 2
|
||||||
>> Linear(nO, width)
|
>> Linear(nO, width)
|
||||||
)
|
)
|
||||||
if dropout:
|
if dropout:
|
||||||
|
|
|
@ -1,5 +1,5 @@
|
||||||
from typing import Optional, List, Union, cast
|
from typing import Optional, List, Union, cast
|
||||||
from thinc.types import Floats2d, Ints2d, Ragged
|
from thinc.types import Floats2d, Ints2d, Ragged, Ints1d
|
||||||
from thinc.api import chain, clone, concatenate, with_array, with_padded
|
from thinc.api import chain, clone, concatenate, with_array, with_padded
|
||||||
from thinc.api import Model, noop, list2ragged, ragged2list, HashEmbed
|
from thinc.api import Model, noop, list2ragged, ragged2list, HashEmbed
|
||||||
from thinc.api import expand_window, residual, Maxout, Mish, PyTorchLSTM
|
from thinc.api import expand_window, residual, Maxout, Mish, PyTorchLSTM
|
||||||
|
@ -159,7 +159,7 @@ def MultiHashEmbed(
|
||||||
embeddings = [make_hash_embed(i) for i in range(len(attrs))]
|
embeddings = [make_hash_embed(i) for i in range(len(attrs))]
|
||||||
concat_size = width * (len(embeddings) + include_static_vectors)
|
concat_size = width * (len(embeddings) + include_static_vectors)
|
||||||
max_out: Model[Ragged, Ragged] = with_array(
|
max_out: Model[Ragged, Ragged] = with_array(
|
||||||
Maxout(width, concat_size, nP=3, dropout=0.0, normalize=True) # type: ignore
|
Maxout(width, concat_size, nP=3, dropout=0.0, normalize=True)
|
||||||
)
|
)
|
||||||
if include_static_vectors:
|
if include_static_vectors:
|
||||||
feature_extractor: Model[List[Doc], Ragged] = chain(
|
feature_extractor: Model[List[Doc], Ragged] = chain(
|
||||||
|
@ -173,7 +173,7 @@ def MultiHashEmbed(
|
||||||
StaticVectors(width, dropout=0.0),
|
StaticVectors(width, dropout=0.0),
|
||||||
),
|
),
|
||||||
max_out,
|
max_out,
|
||||||
cast(Model[Ragged, List[Floats2d]], ragged2list()),
|
ragged2list(),
|
||||||
)
|
)
|
||||||
else:
|
else:
|
||||||
model = chain(
|
model = chain(
|
||||||
|
@ -181,7 +181,7 @@ def MultiHashEmbed(
|
||||||
cast(Model[List[Ints2d], Ragged], list2ragged()),
|
cast(Model[List[Ints2d], Ragged], list2ragged()),
|
||||||
with_array(concatenate(*embeddings)),
|
with_array(concatenate(*embeddings)),
|
||||||
max_out,
|
max_out,
|
||||||
cast(Model[Ragged, List[Floats2d]], ragged2list()),
|
ragged2list(),
|
||||||
)
|
)
|
||||||
return model
|
return model
|
||||||
|
|
||||||
|
@ -232,12 +232,12 @@ def CharacterEmbed(
|
||||||
feature_extractor: Model[List[Doc], Ragged] = chain(
|
feature_extractor: Model[List[Doc], Ragged] = chain(
|
||||||
FeatureExtractor([feature]),
|
FeatureExtractor([feature]),
|
||||||
cast(Model[List[Ints2d], Ragged], list2ragged()),
|
cast(Model[List[Ints2d], Ragged], list2ragged()),
|
||||||
with_array(HashEmbed(nO=width, nV=rows, column=0, seed=5)), # type: ignore
|
with_array(HashEmbed(nO=width, nV=rows, column=0, seed=5)), # type: ignore[misc]
|
||||||
)
|
)
|
||||||
max_out: Model[Ragged, Ragged]
|
max_out: Model[Ragged, Ragged]
|
||||||
if include_static_vectors:
|
if include_static_vectors:
|
||||||
max_out = with_array(
|
max_out = with_array(
|
||||||
Maxout(width, nM * nC + (2 * width), nP=3, normalize=True, dropout=0.0) # type: ignore
|
Maxout(width, nM * nC + (2 * width), nP=3, normalize=True, dropout=0.0)
|
||||||
)
|
)
|
||||||
model = chain(
|
model = chain(
|
||||||
concatenate(
|
concatenate(
|
||||||
|
@ -246,11 +246,11 @@ def CharacterEmbed(
|
||||||
StaticVectors(width, dropout=0.0),
|
StaticVectors(width, dropout=0.0),
|
||||||
),
|
),
|
||||||
max_out,
|
max_out,
|
||||||
cast(Model[Ragged, List[Floats2d]], ragged2list()),
|
ragged2list(),
|
||||||
)
|
)
|
||||||
else:
|
else:
|
||||||
max_out = with_array(
|
max_out = with_array(
|
||||||
Maxout(width, nM * nC + width, nP=3, normalize=True, dropout=0.0) # type: ignore
|
Maxout(width, nM * nC + width, nP=3, normalize=True, dropout=0.0)
|
||||||
)
|
)
|
||||||
model = chain(
|
model = chain(
|
||||||
concatenate(
|
concatenate(
|
||||||
|
@ -258,7 +258,7 @@ def CharacterEmbed(
|
||||||
feature_extractor,
|
feature_extractor,
|
||||||
),
|
),
|
||||||
max_out,
|
max_out,
|
||||||
cast(Model[Ragged, List[Floats2d]], ragged2list()),
|
ragged2list(),
|
||||||
)
|
)
|
||||||
return model
|
return model
|
||||||
|
|
||||||
|
@ -289,10 +289,10 @@ def MaxoutWindowEncoder(
|
||||||
normalize=True,
|
normalize=True,
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
model = clone(residual(cnn), depth) # type: ignore[arg-type]
|
model = clone(residual(cnn), depth)
|
||||||
model.set_dim("nO", width)
|
model.set_dim("nO", width)
|
||||||
receptive_field = window_size * depth
|
receptive_field = window_size * depth
|
||||||
return with_array(model, pad=receptive_field) # type: ignore[arg-type]
|
return with_array(model, pad=receptive_field)
|
||||||
|
|
||||||
|
|
||||||
@registry.architectures("spacy.MishWindowEncoder.v2")
|
@registry.architectures("spacy.MishWindowEncoder.v2")
|
||||||
|
@ -313,9 +313,9 @@ def MishWindowEncoder(
|
||||||
expand_window(window_size=window_size),
|
expand_window(window_size=window_size),
|
||||||
Mish(nO=width, nI=width * ((window_size * 2) + 1), dropout=0.0, normalize=True),
|
Mish(nO=width, nI=width * ((window_size * 2) + 1), dropout=0.0, normalize=True),
|
||||||
)
|
)
|
||||||
model = clone(residual(cnn), depth) # type: ignore[arg-type]
|
model = clone(residual(cnn), depth)
|
||||||
model.set_dim("nO", width)
|
model.set_dim("nO", width)
|
||||||
return with_array(model) # type: ignore[arg-type]
|
return with_array(model)
|
||||||
|
|
||||||
|
|
||||||
@registry.architectures("spacy.TorchBiLSTMEncoder.v1")
|
@registry.architectures("spacy.TorchBiLSTMEncoder.v1")
|
||||||
|
|
|
@ -1,4 +1,5 @@
|
||||||
from libc.string cimport memset, memcpy
|
from libc.string cimport memset, memcpy
|
||||||
|
from thinc.backends.cblas cimport CBlas
|
||||||
from ..typedefs cimport weight_t, hash_t
|
from ..typedefs cimport weight_t, hash_t
|
||||||
from ..pipeline._parser_internals._state cimport StateC
|
from ..pipeline._parser_internals._state cimport StateC
|
||||||
|
|
||||||
|
@ -38,7 +39,7 @@ cdef ActivationsC alloc_activations(SizesC n) nogil
|
||||||
|
|
||||||
cdef void free_activations(const ActivationsC* A) nogil
|
cdef void free_activations(const ActivationsC* A) nogil
|
||||||
|
|
||||||
cdef void predict_states(ActivationsC* A, StateC** states,
|
cdef void predict_states(CBlas cblas, ActivationsC* A, StateC** states,
|
||||||
const WeightsC* W, SizesC n) nogil
|
const WeightsC* W, SizesC n) nogil
|
||||||
|
|
||||||
cdef int arg_max_if_valid(const weight_t* scores, const int* is_valid, int n) nogil
|
cdef int arg_max_if_valid(const weight_t* scores, const int* is_valid, int n) nogil
|
||||||
|
|
|
@ -4,11 +4,11 @@ from libc.math cimport exp
|
||||||
from libc.string cimport memset, memcpy
|
from libc.string cimport memset, memcpy
|
||||||
from libc.stdlib cimport calloc, free, realloc
|
from libc.stdlib cimport calloc, free, realloc
|
||||||
from thinc.backends.linalg cimport Vec, VecVec
|
from thinc.backends.linalg cimport Vec, VecVec
|
||||||
cimport blis.cy
|
from thinc.backends.cblas cimport saxpy, sgemm
|
||||||
|
|
||||||
import numpy
|
import numpy
|
||||||
import numpy.random
|
import numpy.random
|
||||||
from thinc.api import Model, CupyOps, NumpyOps
|
from thinc.api import Model, CupyOps, NumpyOps, get_ops
|
||||||
|
|
||||||
from .. import util
|
from .. import util
|
||||||
from ..errors import Errors
|
from ..errors import Errors
|
||||||
|
@ -91,7 +91,7 @@ cdef void resize_activations(ActivationsC* A, SizesC n) nogil:
|
||||||
A._curr_size = n.states
|
A._curr_size = n.states
|
||||||
|
|
||||||
|
|
||||||
cdef void predict_states(ActivationsC* A, StateC** states,
|
cdef void predict_states(CBlas cblas, ActivationsC* A, StateC** states,
|
||||||
const WeightsC* W, SizesC n) nogil:
|
const WeightsC* W, SizesC n) nogil:
|
||||||
cdef double one = 1.0
|
cdef double one = 1.0
|
||||||
resize_activations(A, n)
|
resize_activations(A, n)
|
||||||
|
@ -99,7 +99,7 @@ cdef void predict_states(ActivationsC* A, StateC** states,
|
||||||
states[i].set_context_tokens(&A.token_ids[i*n.feats], n.feats)
|
states[i].set_context_tokens(&A.token_ids[i*n.feats], n.feats)
|
||||||
memset(A.unmaxed, 0, n.states * n.hiddens * n.pieces * sizeof(float))
|
memset(A.unmaxed, 0, n.states * n.hiddens * n.pieces * sizeof(float))
|
||||||
memset(A.hiddens, 0, n.states * n.hiddens * sizeof(float))
|
memset(A.hiddens, 0, n.states * n.hiddens * sizeof(float))
|
||||||
sum_state_features(A.unmaxed,
|
sum_state_features(cblas, A.unmaxed,
|
||||||
W.feat_weights, A.token_ids, n.states, n.feats, n.hiddens * n.pieces)
|
W.feat_weights, A.token_ids, n.states, n.feats, n.hiddens * n.pieces)
|
||||||
for i in range(n.states):
|
for i in range(n.states):
|
||||||
VecVec.add_i(&A.unmaxed[i*n.hiddens*n.pieces],
|
VecVec.add_i(&A.unmaxed[i*n.hiddens*n.pieces],
|
||||||
|
@ -113,12 +113,10 @@ cdef void predict_states(ActivationsC* A, StateC** states,
|
||||||
memcpy(A.scores, A.hiddens, n.states * n.classes * sizeof(float))
|
memcpy(A.scores, A.hiddens, n.states * n.classes * sizeof(float))
|
||||||
else:
|
else:
|
||||||
# Compute hidden-to-output
|
# Compute hidden-to-output
|
||||||
blis.cy.gemm(blis.cy.NO_TRANSPOSE, blis.cy.TRANSPOSE,
|
sgemm(cblas)(False, True, n.states, n.classes, n.hiddens,
|
||||||
n.states, n.classes, n.hiddens, one,
|
1.0, <const float *>A.hiddens, n.hiddens,
|
||||||
<float*>A.hiddens, n.hiddens, 1,
|
<const float *>W.hidden_weights, n.hiddens,
|
||||||
<float*>W.hidden_weights, n.hiddens, 1,
|
0.0, A.scores, n.classes)
|
||||||
one,
|
|
||||||
<float*>A.scores, n.classes, 1)
|
|
||||||
# Add bias
|
# Add bias
|
||||||
for i in range(n.states):
|
for i in range(n.states):
|
||||||
VecVec.add_i(&A.scores[i*n.classes],
|
VecVec.add_i(&A.scores[i*n.classes],
|
||||||
|
@ -135,7 +133,7 @@ cdef void predict_states(ActivationsC* A, StateC** states,
|
||||||
A.scores[i*n.classes+j] = min_
|
A.scores[i*n.classes+j] = min_
|
||||||
|
|
||||||
|
|
||||||
cdef void sum_state_features(float* output,
|
cdef void sum_state_features(CBlas cblas, float* output,
|
||||||
const float* cached, const int* token_ids, int B, int F, int O) nogil:
|
const float* cached, const int* token_ids, int B, int F, int O) nogil:
|
||||||
cdef int idx, b, f, i
|
cdef int idx, b, f, i
|
||||||
cdef const float* feature
|
cdef const float* feature
|
||||||
|
@ -150,9 +148,7 @@ cdef void sum_state_features(float* output,
|
||||||
else:
|
else:
|
||||||
idx = token_ids[f] * id_stride + f*O
|
idx = token_ids[f] * id_stride + f*O
|
||||||
feature = &cached[idx]
|
feature = &cached[idx]
|
||||||
blis.cy.axpyv(blis.cy.NO_CONJUGATE, O, one,
|
saxpy(cblas)(O, one, <const float*>feature, 1, &output[b*O], 1)
|
||||||
<float*>feature, 1,
|
|
||||||
&output[b*O], 1)
|
|
||||||
token_ids += F
|
token_ids += F
|
||||||
|
|
||||||
|
|
||||||
|
@ -443,9 +439,15 @@ cdef class precompute_hiddens:
|
||||||
# - Output from backward on GPU
|
# - Output from backward on GPU
|
||||||
bp_hiddens = self._bp_hiddens
|
bp_hiddens = self._bp_hiddens
|
||||||
|
|
||||||
|
cdef CBlas cblas
|
||||||
|
if isinstance(self.ops, CupyOps):
|
||||||
|
cblas = NUMPY_OPS.cblas()
|
||||||
|
else:
|
||||||
|
cblas = self.ops.cblas()
|
||||||
|
|
||||||
feat_weights = self.get_feat_weights()
|
feat_weights = self.get_feat_weights()
|
||||||
cdef int[:, ::1] ids = token_ids
|
cdef int[:, ::1] ids = token_ids
|
||||||
sum_state_features(<float*>state_vector.data,
|
sum_state_features(cblas, <float*>state_vector.data,
|
||||||
feat_weights, &ids[0,0],
|
feat_weights, &ids[0,0],
|
||||||
token_ids.shape[0], self.nF, self.nO*self.nP)
|
token_ids.shape[0], self.nF, self.nO*self.nP)
|
||||||
state_vector += self.bias
|
state_vector += self.bias
|
||||||
|
|
|
@ -40,17 +40,15 @@ def forward(
|
||||||
if not token_count:
|
if not token_count:
|
||||||
return _handle_empty(model.ops, model.get_dim("nO"))
|
return _handle_empty(model.ops, model.get_dim("nO"))
|
||||||
key_attr: int = model.attrs["key_attr"]
|
key_attr: int = model.attrs["key_attr"]
|
||||||
keys: Ints1d = model.ops.flatten(
|
keys = model.ops.flatten([cast(Ints1d, doc.to_array(key_attr)) for doc in docs])
|
||||||
cast(Sequence, [doc.to_array(key_attr) for doc in docs])
|
|
||||||
)
|
|
||||||
vocab: Vocab = docs[0].vocab
|
vocab: Vocab = docs[0].vocab
|
||||||
W = cast(Floats2d, model.ops.as_contig(model.get_param("W")))
|
W = cast(Floats2d, model.ops.as_contig(model.get_param("W")))
|
||||||
if vocab.vectors.mode == Mode.default:
|
if vocab.vectors.mode == Mode.default:
|
||||||
V = cast(Floats2d, model.ops.asarray(vocab.vectors.data))
|
V = model.ops.asarray(vocab.vectors.data)
|
||||||
rows = vocab.vectors.find(keys=keys)
|
rows = vocab.vectors.find(keys=keys)
|
||||||
V = model.ops.as_contig(V[rows])
|
V = model.ops.as_contig(V[rows])
|
||||||
elif vocab.vectors.mode == Mode.floret:
|
elif vocab.vectors.mode == Mode.floret:
|
||||||
V = cast(Floats2d, vocab.vectors.get_batch(keys))
|
V = vocab.vectors.get_batch(keys)
|
||||||
V = model.ops.as_contig(V)
|
V = model.ops.as_contig(V)
|
||||||
else:
|
else:
|
||||||
raise RuntimeError(Errors.E896)
|
raise RuntimeError(Errors.E896)
|
||||||
|
@ -62,9 +60,7 @@ def forward(
|
||||||
# Convert negative indices to 0-vectors
|
# Convert negative indices to 0-vectors
|
||||||
# TODO: more options for UNK tokens
|
# TODO: more options for UNK tokens
|
||||||
vectors_data[rows < 0] = 0
|
vectors_data[rows < 0] = 0
|
||||||
output = Ragged(
|
output = Ragged(vectors_data, model.ops.asarray1i([len(doc) for doc in docs]))
|
||||||
vectors_data, model.ops.asarray([len(doc) for doc in docs], dtype="i") # type: ignore
|
|
||||||
)
|
|
||||||
mask = None
|
mask = None
|
||||||
if is_train:
|
if is_train:
|
||||||
mask = _get_drop_mask(model.ops, W.shape[0], model.attrs.get("dropout_rate"))
|
mask = _get_drop_mask(model.ops, W.shape[0], model.attrs.get("dropout_rate"))
|
||||||
|
@ -77,7 +73,9 @@ def forward(
|
||||||
model.inc_grad(
|
model.inc_grad(
|
||||||
"W",
|
"W",
|
||||||
model.ops.gemm(
|
model.ops.gemm(
|
||||||
cast(Floats2d, d_output.data), model.ops.as_contig(V), trans1=True
|
cast(Floats2d, d_output.data),
|
||||||
|
cast(Floats2d, model.ops.as_contig(V)),
|
||||||
|
trans1=True,
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
return []
|
return []
|
||||||
|
|
|
@ -13,6 +13,7 @@ from .sentencizer import Sentencizer
|
||||||
from .tagger import Tagger
|
from .tagger import Tagger
|
||||||
from .textcat import TextCategorizer
|
from .textcat import TextCategorizer
|
||||||
from .spancat import SpanCategorizer
|
from .spancat import SpanCategorizer
|
||||||
|
from .span_ruler import SpanRuler
|
||||||
from .textcat_multilabel import MultiLabel_TextCategorizer
|
from .textcat_multilabel import MultiLabel_TextCategorizer
|
||||||
from .tok2vec import Tok2Vec
|
from .tok2vec import Tok2Vec
|
||||||
from .functions import merge_entities, merge_noun_chunks, merge_subtokens
|
from .functions import merge_entities, merge_noun_chunks, merge_subtokens
|
||||||
|
@ -30,6 +31,7 @@ __all__ = [
|
||||||
"SentenceRecognizer",
|
"SentenceRecognizer",
|
||||||
"Sentencizer",
|
"Sentencizer",
|
||||||
"SpanCategorizer",
|
"SpanCategorizer",
|
||||||
|
"SpanRuler",
|
||||||
"Tagger",
|
"Tagger",
|
||||||
"TextCategorizer",
|
"TextCategorizer",
|
||||||
"Tok2Vec",
|
"Tok2Vec",
|
||||||
|
|
|
@ -10,6 +10,7 @@ from ...strings cimport hash_string
|
||||||
from ...structs cimport TokenC
|
from ...structs cimport TokenC
|
||||||
from ...tokens.doc cimport Doc, set_children_from_heads
|
from ...tokens.doc cimport Doc, set_children_from_heads
|
||||||
from ...tokens.token cimport MISSING_DEP
|
from ...tokens.token cimport MISSING_DEP
|
||||||
|
from ...training import split_bilu_label
|
||||||
from ...training.example cimport Example
|
from ...training.example cimport Example
|
||||||
from .stateclass cimport StateClass
|
from .stateclass cimport StateClass
|
||||||
from ._state cimport StateC, ArcC
|
from ._state cimport StateC, ArcC
|
||||||
|
@ -687,7 +688,7 @@ cdef class ArcEager(TransitionSystem):
|
||||||
return self.c[name_or_id]
|
return self.c[name_or_id]
|
||||||
name = name_or_id
|
name = name_or_id
|
||||||
if '-' in name:
|
if '-' in name:
|
||||||
move_str, label_str = name.split('-', 1)
|
move_str, label_str = split_bilu_label(name)
|
||||||
label = self.strings[label_str]
|
label = self.strings[label_str]
|
||||||
else:
|
else:
|
||||||
move_str = name
|
move_str = name
|
||||||
|
|
|
@ -13,6 +13,7 @@ from ...typedefs cimport weight_t, attr_t
|
||||||
from ...lexeme cimport Lexeme
|
from ...lexeme cimport Lexeme
|
||||||
from ...attrs cimport IS_SPACE
|
from ...attrs cimport IS_SPACE
|
||||||
from ...structs cimport TokenC, SpanC
|
from ...structs cimport TokenC, SpanC
|
||||||
|
from ...training import split_bilu_label
|
||||||
from ...training.example cimport Example
|
from ...training.example cimport Example
|
||||||
from .stateclass cimport StateClass
|
from .stateclass cimport StateClass
|
||||||
from ._state cimport StateC
|
from ._state cimport StateC
|
||||||
|
@ -182,7 +183,7 @@ cdef class BiluoPushDown(TransitionSystem):
|
||||||
if name == '-' or name == '' or name is None:
|
if name == '-' or name == '' or name is None:
|
||||||
return Transition(clas=0, move=MISSING, label=0, score=0)
|
return Transition(clas=0, move=MISSING, label=0, score=0)
|
||||||
elif '-' in name:
|
elif '-' in name:
|
||||||
move_str, label_str = name.split('-', 1)
|
move_str, label_str = split_bilu_label(name)
|
||||||
# Deprecated, hacky way to denote 'not this entity'
|
# Deprecated, hacky way to denote 'not this entity'
|
||||||
if label_str.startswith('!'):
|
if label_str.startswith('!'):
|
||||||
raise ValueError(Errors.E869.format(label=name))
|
raise ValueError(Errors.E869.format(label=name))
|
||||||
|
|
11
spacy/pipeline/_parser_internals/nonproj.hh
Normal file
11
spacy/pipeline/_parser_internals/nonproj.hh
Normal file
|
@ -0,0 +1,11 @@
|
||||||
|
#ifndef NONPROJ_HH
|
||||||
|
#define NONPROJ_HH
|
||||||
|
|
||||||
|
#include <stdexcept>
|
||||||
|
#include <string>
|
||||||
|
|
||||||
|
void raise_domain_error(std::string const &msg) {
|
||||||
|
throw std::domain_error(msg);
|
||||||
|
}
|
||||||
|
|
||||||
|
#endif // NONPROJ_HH
|
|
@ -0,0 +1,4 @@
|
||||||
|
from libcpp.string cimport string
|
||||||
|
|
||||||
|
cdef extern from "nonproj.hh":
|
||||||
|
cdef void raise_domain_error(const string& msg) nogil except +
|
|
@ -4,10 +4,13 @@ for doing pseudo-projective parsing implementation uses the HEAD decoration
|
||||||
scheme.
|
scheme.
|
||||||
"""
|
"""
|
||||||
from copy import copy
|
from copy import copy
|
||||||
|
from cython.operator cimport preincrement as incr, dereference as deref
|
||||||
from libc.limits cimport INT_MAX
|
from libc.limits cimport INT_MAX
|
||||||
from libc.stdlib cimport abs
|
from libc.stdlib cimport abs
|
||||||
from libcpp cimport bool
|
from libcpp cimport bool
|
||||||
|
from libcpp.string cimport string, to_string
|
||||||
from libcpp.vector cimport vector
|
from libcpp.vector cimport vector
|
||||||
|
from libcpp.unordered_set cimport unordered_set
|
||||||
|
|
||||||
from ...tokens.doc cimport Doc, set_children_from_heads
|
from ...tokens.doc cimport Doc, set_children_from_heads
|
||||||
|
|
||||||
|
@ -49,7 +52,7 @@ def is_nonproj_arc(tokenid, heads):
|
||||||
return _is_nonproj_arc(tokenid, c_heads)
|
return _is_nonproj_arc(tokenid, c_heads)
|
||||||
|
|
||||||
|
|
||||||
cdef bool _is_nonproj_arc(int tokenid, const vector[int]& heads) nogil:
|
cdef bool _is_nonproj_arc(int tokenid, const vector[int]& heads) nogil except *:
|
||||||
# definition (e.g. Havelka 2007): an arc h -> d, h < d is non-projective
|
# definition (e.g. Havelka 2007): an arc h -> d, h < d is non-projective
|
||||||
# if there is a token k, h < k < d such that h is not
|
# if there is a token k, h < k < d such that h is not
|
||||||
# an ancestor of k. Same for h -> d, h > d
|
# an ancestor of k. Same for h -> d, h > d
|
||||||
|
@ -58,32 +61,56 @@ cdef bool _is_nonproj_arc(int tokenid, const vector[int]& heads) nogil:
|
||||||
return False
|
return False
|
||||||
elif head < 0: # unattached tokens cannot be non-projective
|
elif head < 0: # unattached tokens cannot be non-projective
|
||||||
return False
|
return False
|
||||||
|
|
||||||
cdef int start, end
|
cdef int start, end
|
||||||
if head < tokenid:
|
if head < tokenid:
|
||||||
start, end = (head+1, tokenid)
|
start, end = (head+1, tokenid)
|
||||||
else:
|
else:
|
||||||
start, end = (tokenid+1, head)
|
start, end = (tokenid+1, head)
|
||||||
for k in range(start, end):
|
for k in range(start, end):
|
||||||
if _has_head_as_ancestor(k, head, heads):
|
if not _has_head_as_ancestor(k, head, heads):
|
||||||
continue
|
|
||||||
else: # head not in ancestors: d -> h is non-projective
|
|
||||||
return True
|
return True
|
||||||
return False
|
return False
|
||||||
|
|
||||||
|
|
||||||
cdef bool _has_head_as_ancestor(int tokenid, int head, const vector[int]& heads) nogil:
|
cdef bool _has_head_as_ancestor(int tokenid, int head, const vector[int]& heads) nogil except *:
|
||||||
ancestor = tokenid
|
ancestor = tokenid
|
||||||
cnt = 0
|
cdef unordered_set[int] seen_tokens
|
||||||
while cnt < heads.size():
|
seen_tokens.insert(ancestor)
|
||||||
|
while True:
|
||||||
|
# Reached the head or a disconnected node
|
||||||
if heads[ancestor] == head or heads[ancestor] < 0:
|
if heads[ancestor] == head or heads[ancestor] < 0:
|
||||||
return True
|
return True
|
||||||
|
# Reached the root
|
||||||
|
if heads[ancestor] == ancestor:
|
||||||
|
return False
|
||||||
ancestor = heads[ancestor]
|
ancestor = heads[ancestor]
|
||||||
cnt += 1
|
result = seen_tokens.insert(ancestor)
|
||||||
|
# Found cycle
|
||||||
|
if not result.second:
|
||||||
|
raise_domain_error(heads_to_string(heads))
|
||||||
|
|
||||||
return False
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
cdef string heads_to_string(const vector[int]& heads) nogil:
|
||||||
|
cdef vector[int].const_iterator citer
|
||||||
|
cdef string cycle_str
|
||||||
|
|
||||||
|
cycle_str.append("Found cycle in dependency graph: [")
|
||||||
|
|
||||||
|
# FIXME: Rewrite using ostringstream when available in Cython.
|
||||||
|
citer = heads.const_begin()
|
||||||
|
while citer != heads.const_end():
|
||||||
|
if citer != heads.const_begin():
|
||||||
|
cycle_str.append(", ")
|
||||||
|
cycle_str.append(to_string(deref(citer)))
|
||||||
|
incr(citer)
|
||||||
|
cycle_str.append("]")
|
||||||
|
|
||||||
|
return cycle_str
|
||||||
|
|
||||||
|
|
||||||
def is_nonproj_tree(heads):
|
def is_nonproj_tree(heads):
|
||||||
cdef vector[int] c_heads = _heads_to_c(heads)
|
cdef vector[int] c_heads = _heads_to_c(heads)
|
||||||
# a tree is non-projective if at least one arc is non-projective
|
# a tree is non-projective if at least one arc is non-projective
|
||||||
|
@ -176,11 +203,12 @@ def get_smallest_nonproj_arc_slow(heads):
|
||||||
return _get_smallest_nonproj_arc(c_heads)
|
return _get_smallest_nonproj_arc(c_heads)
|
||||||
|
|
||||||
|
|
||||||
cdef int _get_smallest_nonproj_arc(const vector[int]& heads) nogil:
|
cdef int _get_smallest_nonproj_arc(const vector[int]& heads) nogil except -2:
|
||||||
# return the smallest non-proj arc or None
|
# return the smallest non-proj arc or None
|
||||||
# where size is defined as the distance between dep and head
|
# where size is defined as the distance between dep and head
|
||||||
# and ties are broken left to right
|
# and ties are broken left to right
|
||||||
cdef int smallest_size = INT_MAX
|
cdef int smallest_size = INT_MAX
|
||||||
|
# -1 means its already projective.
|
||||||
cdef int smallest_np_arc = -1
|
cdef int smallest_np_arc = -1
|
||||||
cdef int size
|
cdef int size
|
||||||
cdef int tokenid
|
cdef int tokenid
|
||||||
|
|
|
@ -12,6 +12,7 @@ from ..language import Language
|
||||||
from ._parser_internals import nonproj
|
from ._parser_internals import nonproj
|
||||||
from ._parser_internals.nonproj import DELIMITER
|
from ._parser_internals.nonproj import DELIMITER
|
||||||
from ..scorer import Scorer
|
from ..scorer import Scorer
|
||||||
|
from ..training import remove_bilu_prefix
|
||||||
from ..util import registry
|
from ..util import registry
|
||||||
|
|
||||||
|
|
||||||
|
@ -314,7 +315,7 @@ cdef class DependencyParser(Parser):
|
||||||
# Get the labels from the model by looking at the available moves
|
# Get the labels from the model by looking at the available moves
|
||||||
for move in self.move_names:
|
for move in self.move_names:
|
||||||
if "-" in move:
|
if "-" in move:
|
||||||
label = move.split("-")[1]
|
label = remove_bilu_prefix(move)
|
||||||
if DELIMITER in label:
|
if DELIMITER in label:
|
||||||
label = label.split(DELIMITER)[1]
|
label = label.split(DELIMITER)[1]
|
||||||
labels.add(label)
|
labels.add(label)
|
||||||
|
|
|
@ -138,7 +138,7 @@ class EditTreeLemmatizer(TrainablePipe):
|
||||||
|
|
||||||
truths.append(eg_truths)
|
truths.append(eg_truths)
|
||||||
|
|
||||||
d_scores, loss = loss_func(scores, truths) # type: ignore
|
d_scores, loss = loss_func(scores, truths)
|
||||||
if self.model.ops.xp.isnan(loss):
|
if self.model.ops.xp.isnan(loss):
|
||||||
raise ValueError(Errors.E910.format(name=self.name))
|
raise ValueError(Errors.E910.format(name=self.name))
|
||||||
|
|
||||||
|
|
|
@ -56,6 +56,7 @@ DEFAULT_NEL_MODEL = Config().from_str(default_model_config)["model"]
|
||||||
"overwrite": True,
|
"overwrite": True,
|
||||||
"scorer": {"@scorers": "spacy.entity_linker_scorer.v1"},
|
"scorer": {"@scorers": "spacy.entity_linker_scorer.v1"},
|
||||||
"use_gold_ents": True,
|
"use_gold_ents": True,
|
||||||
|
"threshold": None,
|
||||||
},
|
},
|
||||||
default_score_weights={
|
default_score_weights={
|
||||||
"nel_micro_f": 1.0,
|
"nel_micro_f": 1.0,
|
||||||
|
@ -77,6 +78,7 @@ def make_entity_linker(
|
||||||
overwrite: bool,
|
overwrite: bool,
|
||||||
scorer: Optional[Callable],
|
scorer: Optional[Callable],
|
||||||
use_gold_ents: bool,
|
use_gold_ents: bool,
|
||||||
|
threshold: Optional[float] = None,
|
||||||
):
|
):
|
||||||
"""Construct an EntityLinker component.
|
"""Construct an EntityLinker component.
|
||||||
|
|
||||||
|
@ -91,6 +93,10 @@ def make_entity_linker(
|
||||||
get_candidates (Callable[[KnowledgeBase, "Span"], Iterable[Candidate]]): Function that
|
get_candidates (Callable[[KnowledgeBase, "Span"], Iterable[Candidate]]): Function that
|
||||||
produces a list of candidates, given a certain knowledge base and a textual mention.
|
produces a list of candidates, given a certain knowledge base and a textual mention.
|
||||||
scorer (Optional[Callable]): The scoring method.
|
scorer (Optional[Callable]): The scoring method.
|
||||||
|
use_gold_ents (bool): Whether to copy entities from gold docs or not. If false, another
|
||||||
|
component must provide entity annotations.
|
||||||
|
threshold (Optional[float]): Confidence threshold for entity predictions. If confidence is below the threshold,
|
||||||
|
prediction is discarded. If None, predictions are not filtered by any threshold.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
if not model.attrs.get("include_span_maker", False):
|
if not model.attrs.get("include_span_maker", False):
|
||||||
|
@ -121,6 +127,7 @@ def make_entity_linker(
|
||||||
overwrite=overwrite,
|
overwrite=overwrite,
|
||||||
scorer=scorer,
|
scorer=scorer,
|
||||||
use_gold_ents=use_gold_ents,
|
use_gold_ents=use_gold_ents,
|
||||||
|
threshold=threshold,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@ -156,6 +163,7 @@ class EntityLinker(TrainablePipe):
|
||||||
overwrite: bool = BACKWARD_OVERWRITE,
|
overwrite: bool = BACKWARD_OVERWRITE,
|
||||||
scorer: Optional[Callable] = entity_linker_score,
|
scorer: Optional[Callable] = entity_linker_score,
|
||||||
use_gold_ents: bool,
|
use_gold_ents: bool,
|
||||||
|
threshold: Optional[float] = None,
|
||||||
) -> None:
|
) -> None:
|
||||||
"""Initialize an entity linker.
|
"""Initialize an entity linker.
|
||||||
|
|
||||||
|
@ -174,9 +182,20 @@ class EntityLinker(TrainablePipe):
|
||||||
Scorer.score_links.
|
Scorer.score_links.
|
||||||
use_gold_ents (bool): Whether to copy entities from gold docs or not. If false, another
|
use_gold_ents (bool): Whether to copy entities from gold docs or not. If false, another
|
||||||
component must provide entity annotations.
|
component must provide entity annotations.
|
||||||
|
threshold (Optional[float]): Confidence threshold for entity predictions. If confidence is below the
|
||||||
|
threshold, prediction is discarded. If None, predictions are not filtered by any threshold.
|
||||||
DOCS: https://spacy.io/api/entitylinker#init
|
DOCS: https://spacy.io/api/entitylinker#init
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
if threshold is not None and not (0 <= threshold <= 1):
|
||||||
|
raise ValueError(
|
||||||
|
Errors.E1043.format(
|
||||||
|
range_start=0,
|
||||||
|
range_end=1,
|
||||||
|
value=threshold,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
self.vocab = vocab
|
self.vocab = vocab
|
||||||
self.model = model
|
self.model = model
|
||||||
self.name = name
|
self.name = name
|
||||||
|
@ -192,6 +211,7 @@ class EntityLinker(TrainablePipe):
|
||||||
self.kb = empty_kb(entity_vector_length)(self.vocab)
|
self.kb = empty_kb(entity_vector_length)(self.vocab)
|
||||||
self.scorer = scorer
|
self.scorer = scorer
|
||||||
self.use_gold_ents = use_gold_ents
|
self.use_gold_ents = use_gold_ents
|
||||||
|
self.threshold = threshold
|
||||||
|
|
||||||
def set_kb(self, kb_loader: Callable[[Vocab], KnowledgeBase]):
|
def set_kb(self, kb_loader: Callable[[Vocab], KnowledgeBase]):
|
||||||
"""Define the KB of this pipe by providing a function that will
|
"""Define the KB of this pipe by providing a function that will
|
||||||
|
@ -234,10 +254,11 @@ class EntityLinker(TrainablePipe):
|
||||||
nO = self.kb.entity_vector_length
|
nO = self.kb.entity_vector_length
|
||||||
doc_sample = []
|
doc_sample = []
|
||||||
vector_sample = []
|
vector_sample = []
|
||||||
for example in islice(get_examples(), 10):
|
for eg in islice(get_examples(), 10):
|
||||||
doc = example.x
|
doc = eg.x
|
||||||
if self.use_gold_ents:
|
if self.use_gold_ents:
|
||||||
doc.ents = example.y.ents
|
ents, _ = eg.get_aligned_ents_and_ner()
|
||||||
|
doc.ents = ents
|
||||||
doc_sample.append(doc)
|
doc_sample.append(doc)
|
||||||
vector_sample.append(self.model.ops.alloc1f(nO))
|
vector_sample.append(self.model.ops.alloc1f(nO))
|
||||||
assert len(doc_sample) > 0, Errors.E923.format(name=self.name)
|
assert len(doc_sample) > 0, Errors.E923.format(name=self.name)
|
||||||
|
@ -312,7 +333,8 @@ class EntityLinker(TrainablePipe):
|
||||||
|
|
||||||
for doc, ex in zip(docs, examples):
|
for doc, ex in zip(docs, examples):
|
||||||
if self.use_gold_ents:
|
if self.use_gold_ents:
|
||||||
doc.ents = ex.reference.ents
|
ents, _ = ex.get_aligned_ents_and_ner()
|
||||||
|
doc.ents = ents
|
||||||
else:
|
else:
|
||||||
# only keep matching ents
|
# only keep matching ents
|
||||||
doc.ents = ex.get_matching_ents()
|
doc.ents = ex.get_matching_ents()
|
||||||
|
@ -345,7 +367,7 @@ class EntityLinker(TrainablePipe):
|
||||||
for eg in examples:
|
for eg in examples:
|
||||||
kb_ids = eg.get_aligned("ENT_KB_ID", as_string=True)
|
kb_ids = eg.get_aligned("ENT_KB_ID", as_string=True)
|
||||||
|
|
||||||
for ent in eg.reference.ents:
|
for ent in eg.get_matching_ents():
|
||||||
kb_id = kb_ids[ent.start]
|
kb_id = kb_ids[ent.start]
|
||||||
if kb_id:
|
if kb_id:
|
||||||
entity_encoding = self.kb.get_vector(kb_id)
|
entity_encoding = self.kb.get_vector(kb_id)
|
||||||
|
@ -353,22 +375,25 @@ class EntityLinker(TrainablePipe):
|
||||||
keep_ents.append(eidx)
|
keep_ents.append(eidx)
|
||||||
|
|
||||||
eidx += 1
|
eidx += 1
|
||||||
entity_encodings = self.model.ops.asarray(entity_encodings, dtype="float32")
|
entity_encodings = self.model.ops.asarray2f(entity_encodings, dtype="float32")
|
||||||
selected_encodings = sentence_encodings[keep_ents]
|
selected_encodings = sentence_encodings[keep_ents]
|
||||||
|
|
||||||
# If the entity encodings list is empty, then
|
# if there are no matches, short circuit
|
||||||
|
if not keep_ents:
|
||||||
|
out = self.model.ops.alloc2f(*sentence_encodings.shape)
|
||||||
|
return 0, out
|
||||||
|
|
||||||
if selected_encodings.shape != entity_encodings.shape:
|
if selected_encodings.shape != entity_encodings.shape:
|
||||||
err = Errors.E147.format(
|
err = Errors.E147.format(
|
||||||
method="get_loss", msg="gold entities do not match up"
|
method="get_loss", msg="gold entities do not match up"
|
||||||
)
|
)
|
||||||
raise RuntimeError(err)
|
raise RuntimeError(err)
|
||||||
# TODO: fix typing issue here
|
gradients = self.distance.get_grad(selected_encodings, entity_encodings)
|
||||||
gradients = self.distance.get_grad(selected_encodings, entity_encodings) # type: ignore
|
|
||||||
# to match the input size, we need to give a zero gradient for items not in the kb
|
# to match the input size, we need to give a zero gradient for items not in the kb
|
||||||
out = self.model.ops.alloc2f(*sentence_encodings.shape)
|
out = self.model.ops.alloc2f(*sentence_encodings.shape)
|
||||||
out[keep_ents] = gradients
|
out[keep_ents] = gradients
|
||||||
|
|
||||||
loss = self.distance.get_loss(selected_encodings, entity_encodings) # type: ignore
|
loss = self.distance.get_loss(selected_encodings, entity_encodings)
|
||||||
loss = loss / len(entity_encodings)
|
loss = loss / len(entity_encodings)
|
||||||
return float(loss), out
|
return float(loss), out
|
||||||
|
|
||||||
|
@ -385,18 +410,21 @@ class EntityLinker(TrainablePipe):
|
||||||
self.validate_kb()
|
self.validate_kb()
|
||||||
entity_count = 0
|
entity_count = 0
|
||||||
final_kb_ids: List[str] = []
|
final_kb_ids: List[str] = []
|
||||||
|
xp = self.model.ops.xp
|
||||||
if not docs:
|
if not docs:
|
||||||
return final_kb_ids
|
return final_kb_ids
|
||||||
if isinstance(docs, Doc):
|
if isinstance(docs, Doc):
|
||||||
docs = [docs]
|
docs = [docs]
|
||||||
for i, doc in enumerate(docs):
|
for i, doc in enumerate(docs):
|
||||||
|
if len(doc) == 0:
|
||||||
|
continue
|
||||||
sentences = [s for s in doc.sents]
|
sentences = [s for s in doc.sents]
|
||||||
if len(doc) > 0:
|
# Looping through each entity (TODO: rewrite)
|
||||||
# Looping through each entity (TODO: rewrite)
|
for ent in doc.ents:
|
||||||
for ent in doc.ents:
|
sent_index = sentences.index(ent.sent)
|
||||||
sent = ent.sent
|
assert sent_index >= 0
|
||||||
sent_index = sentences.index(sent)
|
|
||||||
assert sent_index >= 0
|
if self.incl_context:
|
||||||
# get n_neighbour sentences, clipped to the length of the document
|
# get n_neighbour sentences, clipped to the length of the document
|
||||||
start_sentence = max(0, sent_index - self.n_sents)
|
start_sentence = max(0, sent_index - self.n_sents)
|
||||||
end_sentence = min(len(sentences) - 1, sent_index + self.n_sents)
|
end_sentence = min(len(sentences) - 1, sent_index + self.n_sents)
|
||||||
|
@ -404,55 +432,53 @@ class EntityLinker(TrainablePipe):
|
||||||
end_token = sentences[end_sentence].end
|
end_token = sentences[end_sentence].end
|
||||||
sent_doc = doc[start_token:end_token].as_doc()
|
sent_doc = doc[start_token:end_token].as_doc()
|
||||||
# currently, the context is the same for each entity in a sentence (should be refined)
|
# currently, the context is the same for each entity in a sentence (should be refined)
|
||||||
xp = self.model.ops.xp
|
sentence_encoding = self.model.predict([sent_doc])[0]
|
||||||
if self.incl_context:
|
sentence_encoding_t = sentence_encoding.T
|
||||||
sentence_encoding = self.model.predict([sent_doc])[0]
|
sentence_norm = xp.linalg.norm(sentence_encoding_t)
|
||||||
sentence_encoding_t = sentence_encoding.T
|
entity_count += 1
|
||||||
sentence_norm = xp.linalg.norm(sentence_encoding_t)
|
if ent.label_ in self.labels_discard:
|
||||||
entity_count += 1
|
# ignoring this entity - setting to NIL
|
||||||
if ent.label_ in self.labels_discard:
|
final_kb_ids.append(self.NIL)
|
||||||
# ignoring this entity - setting to NIL
|
else:
|
||||||
|
candidates = list(self.get_candidates(self.kb, ent))
|
||||||
|
if not candidates:
|
||||||
|
# no prediction possible for this entity - setting to NIL
|
||||||
final_kb_ids.append(self.NIL)
|
final_kb_ids.append(self.NIL)
|
||||||
|
elif len(candidates) == 1 and self.threshold is None:
|
||||||
|
# shortcut for efficiency reasons: take the 1 candidate
|
||||||
|
final_kb_ids.append(candidates[0].entity_)
|
||||||
else:
|
else:
|
||||||
candidates = list(self.get_candidates(self.kb, ent))
|
random.shuffle(candidates)
|
||||||
if not candidates:
|
# set all prior probabilities to 0 if incl_prior=False
|
||||||
# no prediction possible for this entity - setting to NIL
|
prior_probs = xp.asarray([c.prior_prob for c in candidates])
|
||||||
final_kb_ids.append(self.NIL)
|
if not self.incl_prior:
|
||||||
elif len(candidates) == 1:
|
prior_probs = xp.asarray([0.0 for _ in candidates])
|
||||||
# shortcut for efficiency reasons: take the 1 candidate
|
scores = prior_probs
|
||||||
# TODO: thresholding
|
# add in similarity from the context
|
||||||
final_kb_ids.append(candidates[0].entity_)
|
if self.incl_context:
|
||||||
else:
|
entity_encodings = xp.asarray(
|
||||||
random.shuffle(candidates)
|
[c.entity_vector for c in candidates]
|
||||||
# set all prior probabilities to 0 if incl_prior=False
|
)
|
||||||
prior_probs = xp.asarray([c.prior_prob for c in candidates])
|
entity_norm = xp.linalg.norm(entity_encodings, axis=1)
|
||||||
if not self.incl_prior:
|
if len(entity_encodings) != len(prior_probs):
|
||||||
prior_probs = xp.asarray([0.0 for _ in candidates])
|
raise RuntimeError(
|
||||||
scores = prior_probs
|
Errors.E147.format(
|
||||||
# add in similarity from the context
|
method="predict",
|
||||||
if self.incl_context:
|
msg="vectors not of equal length",
|
||||||
entity_encodings = xp.asarray(
|
|
||||||
[c.entity_vector for c in candidates]
|
|
||||||
)
|
|
||||||
entity_norm = xp.linalg.norm(entity_encodings, axis=1)
|
|
||||||
if len(entity_encodings) != len(prior_probs):
|
|
||||||
raise RuntimeError(
|
|
||||||
Errors.E147.format(
|
|
||||||
method="predict",
|
|
||||||
msg="vectors not of equal length",
|
|
||||||
)
|
|
||||||
)
|
)
|
||||||
# cosine similarity
|
|
||||||
sims = xp.dot(entity_encodings, sentence_encoding_t) / (
|
|
||||||
sentence_norm * entity_norm
|
|
||||||
)
|
)
|
||||||
if sims.shape != prior_probs.shape:
|
# cosine similarity
|
||||||
raise ValueError(Errors.E161)
|
sims = xp.dot(entity_encodings, sentence_encoding_t) / (
|
||||||
scores = prior_probs + sims - (prior_probs * sims)
|
sentence_norm * entity_norm
|
||||||
# TODO: thresholding
|
)
|
||||||
best_index = scores.argmax().item()
|
if sims.shape != prior_probs.shape:
|
||||||
best_candidate = candidates[best_index]
|
raise ValueError(Errors.E161)
|
||||||
final_kb_ids.append(best_candidate.entity_)
|
scores = prior_probs + sims - (prior_probs * sims)
|
||||||
|
final_kb_ids.append(
|
||||||
|
candidates[scores.argmax().item()].entity_
|
||||||
|
if self.threshold is None or scores.max() >= self.threshold
|
||||||
|
else EntityLinker.NIL
|
||||||
|
)
|
||||||
if not (len(final_kb_ids) == entity_count):
|
if not (len(final_kb_ids) == entity_count):
|
||||||
err = Errors.E147.format(
|
err = Errors.E147.format(
|
||||||
method="predict", msg="result variables not of equal length"
|
method="predict", msg="result variables not of equal length"
|
||||||
|
|
|
@ -159,10 +159,8 @@ class EntityRuler(Pipe):
|
||||||
self._require_patterns()
|
self._require_patterns()
|
||||||
with warnings.catch_warnings():
|
with warnings.catch_warnings():
|
||||||
warnings.filterwarnings("ignore", message="\\[W036")
|
warnings.filterwarnings("ignore", message="\\[W036")
|
||||||
matches = cast(
|
matches = list(self.matcher(doc)) + list(self.phrase_matcher(doc))
|
||||||
List[Tuple[int, int, int]],
|
|
||||||
list(self.matcher(doc)) + list(self.phrase_matcher(doc)),
|
|
||||||
)
|
|
||||||
final_matches = set(
|
final_matches = set(
|
||||||
[(m_id, start, end) for m_id, start, end in matches if start != end]
|
[(m_id, start, end) for m_id, start, end in matches if start != end]
|
||||||
)
|
)
|
||||||
|
@ -182,10 +180,7 @@ class EntityRuler(Pipe):
|
||||||
if start not in seen_tokens and end - 1 not in seen_tokens:
|
if start not in seen_tokens and end - 1 not in seen_tokens:
|
||||||
if match_id in self._ent_ids:
|
if match_id in self._ent_ids:
|
||||||
label, ent_id = self._ent_ids[match_id]
|
label, ent_id = self._ent_ids[match_id]
|
||||||
span = Span(doc, start, end, label=label)
|
span = Span(doc, start, end, label=label, span_id=ent_id)
|
||||||
if ent_id:
|
|
||||||
for token in span:
|
|
||||||
token.ent_id_ = ent_id
|
|
||||||
else:
|
else:
|
||||||
span = Span(doc, start, end, label=match_id)
|
span = Span(doc, start, end, label=match_id)
|
||||||
new_entities.append(span)
|
new_entities.append(span)
|
||||||
|
@ -359,7 +354,9 @@ class EntityRuler(Pipe):
|
||||||
(label, eid) for (label, eid) in self._ent_ids.values() if eid == ent_id
|
(label, eid) for (label, eid) in self._ent_ids.values() if eid == ent_id
|
||||||
]
|
]
|
||||||
if not label_id_pairs:
|
if not label_id_pairs:
|
||||||
raise ValueError(Errors.E1024.format(ent_id=ent_id))
|
raise ValueError(
|
||||||
|
Errors.E1024.format(attr_type="ID", label=ent_id, component=self.name)
|
||||||
|
)
|
||||||
created_labels = [
|
created_labels = [
|
||||||
self._create_label(label, eid) for (label, eid) in label_id_pairs
|
self._create_label(label, eid) for (label, eid) in label_id_pairs
|
||||||
]
|
]
|
||||||
|
|
|
@ -7,7 +7,7 @@ from pathlib import Path
|
||||||
from itertools import islice
|
from itertools import islice
|
||||||
import srsly
|
import srsly
|
||||||
import random
|
import random
|
||||||
from thinc.api import CosineDistance, Model, Optimizer, Config
|
from thinc.api import CosineDistance, Model, Optimizer
|
||||||
from thinc.api import set_dropout_rate
|
from thinc.api import set_dropout_rate
|
||||||
import warnings
|
import warnings
|
||||||
|
|
||||||
|
@ -20,7 +20,7 @@ from ...language import Language
|
||||||
from ...vocab import Vocab
|
from ...vocab import Vocab
|
||||||
from ...training import Example, validate_examples, validate_get_examples
|
from ...training import Example, validate_examples, validate_get_examples
|
||||||
from ...errors import Errors, Warnings
|
from ...errors import Errors, Warnings
|
||||||
from ...util import SimpleFrozenList, registry
|
from ...util import SimpleFrozenList
|
||||||
from ... import util
|
from ... import util
|
||||||
from ...scorer import Scorer
|
from ...scorer import Scorer
|
||||||
|
|
||||||
|
@ -70,7 +70,6 @@ class EntityLinker_v1(TrainablePipe):
|
||||||
produces a list of candidates, given a certain knowledge base and a textual mention.
|
produces a list of candidates, given a certain knowledge base and a textual mention.
|
||||||
scorer (Optional[Callable]): The scoring method. Defaults to
|
scorer (Optional[Callable]): The scoring method. Defaults to
|
||||||
Scorer.score_links.
|
Scorer.score_links.
|
||||||
|
|
||||||
DOCS: https://spacy.io/api/entitylinker#init
|
DOCS: https://spacy.io/api/entitylinker#init
|
||||||
"""
|
"""
|
||||||
self.vocab = vocab
|
self.vocab = vocab
|
||||||
|
@ -213,15 +212,14 @@ class EntityLinker_v1(TrainablePipe):
|
||||||
if kb_id:
|
if kb_id:
|
||||||
entity_encoding = self.kb.get_vector(kb_id)
|
entity_encoding = self.kb.get_vector(kb_id)
|
||||||
entity_encodings.append(entity_encoding)
|
entity_encodings.append(entity_encoding)
|
||||||
entity_encodings = self.model.ops.asarray(entity_encodings, dtype="float32")
|
entity_encodings = self.model.ops.asarray2f(entity_encodings)
|
||||||
if sentence_encodings.shape != entity_encodings.shape:
|
if sentence_encodings.shape != entity_encodings.shape:
|
||||||
err = Errors.E147.format(
|
err = Errors.E147.format(
|
||||||
method="get_loss", msg="gold entities do not match up"
|
method="get_loss", msg="gold entities do not match up"
|
||||||
)
|
)
|
||||||
raise RuntimeError(err)
|
raise RuntimeError(err)
|
||||||
# TODO: fix typing issue here
|
gradients = self.distance.get_grad(sentence_encodings, entity_encodings)
|
||||||
gradients = self.distance.get_grad(sentence_encodings, entity_encodings) # type: ignore
|
loss = self.distance.get_loss(sentence_encodings, entity_encodings)
|
||||||
loss = self.distance.get_loss(sentence_encodings, entity_encodings) # type: ignore
|
|
||||||
loss = loss / len(entity_encodings)
|
loss = loss / len(entity_encodings)
|
||||||
return float(loss), gradients
|
return float(loss), gradients
|
||||||
|
|
||||||
|
@ -273,7 +271,6 @@ class EntityLinker_v1(TrainablePipe):
|
||||||
final_kb_ids.append(self.NIL)
|
final_kb_ids.append(self.NIL)
|
||||||
elif len(candidates) == 1:
|
elif len(candidates) == 1:
|
||||||
# shortcut for efficiency reasons: take the 1 candidate
|
# shortcut for efficiency reasons: take the 1 candidate
|
||||||
# TODO: thresholding
|
|
||||||
final_kb_ids.append(candidates[0].entity_)
|
final_kb_ids.append(candidates[0].entity_)
|
||||||
else:
|
else:
|
||||||
random.shuffle(candidates)
|
random.shuffle(candidates)
|
||||||
|
@ -302,7 +299,6 @@ class EntityLinker_v1(TrainablePipe):
|
||||||
if sims.shape != prior_probs.shape:
|
if sims.shape != prior_probs.shape:
|
||||||
raise ValueError(Errors.E161)
|
raise ValueError(Errors.E161)
|
||||||
scores = prior_probs + sims - (prior_probs * sims)
|
scores = prior_probs + sims - (prior_probs * sims)
|
||||||
# TODO: thresholding
|
|
||||||
best_index = scores.argmax().item()
|
best_index = scores.argmax().item()
|
||||||
best_candidate = candidates[best_index]
|
best_candidate = candidates[best_index]
|
||||||
final_kb_ids.append(best_candidate.entity_)
|
final_kb_ids.append(best_candidate.entity_)
|
||||||
|
|
|
@ -6,10 +6,10 @@ from thinc.api import Model, Config
|
||||||
from ._parser_internals.transition_system import TransitionSystem
|
from ._parser_internals.transition_system import TransitionSystem
|
||||||
from .transition_parser cimport Parser
|
from .transition_parser cimport Parser
|
||||||
from ._parser_internals.ner cimport BiluoPushDown
|
from ._parser_internals.ner cimport BiluoPushDown
|
||||||
|
|
||||||
from ..language import Language
|
from ..language import Language
|
||||||
from ..scorer import get_ner_prf, PRFScore
|
from ..scorer import get_ner_prf, PRFScore
|
||||||
from ..util import registry
|
from ..util import registry
|
||||||
|
from ..training import remove_bilu_prefix
|
||||||
|
|
||||||
|
|
||||||
default_model_config = """
|
default_model_config = """
|
||||||
|
@ -242,7 +242,7 @@ cdef class EntityRecognizer(Parser):
|
||||||
def labels(self):
|
def labels(self):
|
||||||
# Get the labels from the model by looking at the available moves, e.g.
|
# Get the labels from the model by looking at the available moves, e.g.
|
||||||
# B-PERSON, I-PERSON, L-PERSON, U-PERSON
|
# B-PERSON, I-PERSON, L-PERSON, U-PERSON
|
||||||
labels = set(move.split("-")[1] for move in self.move_names
|
labels = set(remove_bilu_prefix(move) for move in self.move_names
|
||||||
if move[0] in ("B", "I", "L", "U"))
|
if move[0] in ("B", "I", "L", "U"))
|
||||||
return tuple(sorted(labels))
|
return tuple(sorted(labels))
|
||||||
|
|
||||||
|
|
|
@ -31,7 +31,7 @@ cdef class Pipe:
|
||||||
and returned. This usually happens under the hood when the nlp object
|
and returned. This usually happens under the hood when the nlp object
|
||||||
is called on a text and all components are applied to the Doc.
|
is called on a text and all components are applied to the Doc.
|
||||||
|
|
||||||
docs (Doc): The Doc to process.
|
doc (Doc): The Doc to process.
|
||||||
RETURNS (Doc): The processed Doc.
|
RETURNS (Doc): The processed Doc.
|
||||||
|
|
||||||
DOCS: https://spacy.io/api/pipe#call
|
DOCS: https://spacy.io/api/pipe#call
|
||||||
|
|
569
spacy/pipeline/span_ruler.py
Normal file
569
spacy/pipeline/span_ruler.py
Normal file
|
@ -0,0 +1,569 @@
|
||||||
|
from typing import Optional, Union, List, Dict, Tuple, Iterable, Any, Callable
|
||||||
|
from typing import Sequence, Set, cast
|
||||||
|
import warnings
|
||||||
|
from functools import partial
|
||||||
|
from pathlib import Path
|
||||||
|
import srsly
|
||||||
|
|
||||||
|
from .pipe import Pipe
|
||||||
|
from ..training import Example
|
||||||
|
from ..language import Language
|
||||||
|
from ..errors import Errors, Warnings
|
||||||
|
from ..util import ensure_path, SimpleFrozenList, registry
|
||||||
|
from ..tokens import Doc, Span
|
||||||
|
from ..scorer import Scorer
|
||||||
|
from ..matcher import Matcher, PhraseMatcher
|
||||||
|
from .. import util
|
||||||
|
|
||||||
|
PatternType = Dict[str, Union[str, List[Dict[str, Any]]]]
|
||||||
|
DEFAULT_SPANS_KEY = "ruler"
|
||||||
|
|
||||||
|
|
||||||
|
@Language.factory(
|
||||||
|
"future_entity_ruler",
|
||||||
|
assigns=["doc.ents"],
|
||||||
|
default_config={
|
||||||
|
"phrase_matcher_attr": None,
|
||||||
|
"validate": False,
|
||||||
|
"overwrite_ents": False,
|
||||||
|
"scorer": {"@scorers": "spacy.entity_ruler_scorer.v1"},
|
||||||
|
"ent_id_sep": "__unused__",
|
||||||
|
},
|
||||||
|
default_score_weights={
|
||||||
|
"ents_f": 1.0,
|
||||||
|
"ents_p": 0.0,
|
||||||
|
"ents_r": 0.0,
|
||||||
|
"ents_per_type": None,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
def make_entity_ruler(
|
||||||
|
nlp: Language,
|
||||||
|
name: str,
|
||||||
|
phrase_matcher_attr: Optional[Union[int, str]],
|
||||||
|
validate: bool,
|
||||||
|
overwrite_ents: bool,
|
||||||
|
scorer: Optional[Callable],
|
||||||
|
ent_id_sep: str,
|
||||||
|
):
|
||||||
|
if overwrite_ents:
|
||||||
|
ents_filter = prioritize_new_ents_filter
|
||||||
|
else:
|
||||||
|
ents_filter = prioritize_existing_ents_filter
|
||||||
|
return SpanRuler(
|
||||||
|
nlp,
|
||||||
|
name,
|
||||||
|
spans_key=None,
|
||||||
|
spans_filter=None,
|
||||||
|
annotate_ents=True,
|
||||||
|
ents_filter=ents_filter,
|
||||||
|
phrase_matcher_attr=phrase_matcher_attr,
|
||||||
|
validate=validate,
|
||||||
|
overwrite=False,
|
||||||
|
scorer=scorer,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@Language.factory(
|
||||||
|
"span_ruler",
|
||||||
|
assigns=["doc.spans"],
|
||||||
|
default_config={
|
||||||
|
"spans_key": DEFAULT_SPANS_KEY,
|
||||||
|
"spans_filter": None,
|
||||||
|
"annotate_ents": False,
|
||||||
|
"ents_filter": {"@misc": "spacy.first_longest_spans_filter.v1"},
|
||||||
|
"phrase_matcher_attr": None,
|
||||||
|
"validate": False,
|
||||||
|
"overwrite": True,
|
||||||
|
"scorer": {
|
||||||
|
"@scorers": "spacy.overlapping_labeled_spans_scorer.v1",
|
||||||
|
"spans_key": DEFAULT_SPANS_KEY,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
default_score_weights={
|
||||||
|
f"spans_{DEFAULT_SPANS_KEY}_f": 1.0,
|
||||||
|
f"spans_{DEFAULT_SPANS_KEY}_p": 0.0,
|
||||||
|
f"spans_{DEFAULT_SPANS_KEY}_r": 0.0,
|
||||||
|
f"spans_{DEFAULT_SPANS_KEY}_per_type": None,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
def make_span_ruler(
|
||||||
|
nlp: Language,
|
||||||
|
name: str,
|
||||||
|
spans_key: Optional[str],
|
||||||
|
spans_filter: Optional[Callable[[Iterable[Span], Iterable[Span]], Iterable[Span]]],
|
||||||
|
annotate_ents: bool,
|
||||||
|
ents_filter: Callable[[Iterable[Span], Iterable[Span]], Iterable[Span]],
|
||||||
|
phrase_matcher_attr: Optional[Union[int, str]],
|
||||||
|
validate: bool,
|
||||||
|
overwrite: bool,
|
||||||
|
scorer: Optional[Callable],
|
||||||
|
):
|
||||||
|
return SpanRuler(
|
||||||
|
nlp,
|
||||||
|
name,
|
||||||
|
spans_key=spans_key,
|
||||||
|
spans_filter=spans_filter,
|
||||||
|
annotate_ents=annotate_ents,
|
||||||
|
ents_filter=ents_filter,
|
||||||
|
phrase_matcher_attr=phrase_matcher_attr,
|
||||||
|
validate=validate,
|
||||||
|
overwrite=overwrite,
|
||||||
|
scorer=scorer,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def prioritize_new_ents_filter(
|
||||||
|
entities: Iterable[Span], spans: Iterable[Span]
|
||||||
|
) -> List[Span]:
|
||||||
|
"""Merge entities and spans into one list without overlaps by allowing
|
||||||
|
spans to overwrite any entities that they overlap with. Intended to
|
||||||
|
replicate the overwrite_ents=True behavior from the EntityRuler.
|
||||||
|
|
||||||
|
entities (Iterable[Span]): The entities, already filtered for overlaps.
|
||||||
|
spans (Iterable[Span]): The spans to merge, may contain overlaps.
|
||||||
|
RETURNS (List[Span]): Filtered list of non-overlapping spans.
|
||||||
|
"""
|
||||||
|
get_sort_key = lambda span: (span.end - span.start, -span.start)
|
||||||
|
spans = sorted(spans, key=get_sort_key, reverse=True)
|
||||||
|
entities = list(entities)
|
||||||
|
new_entities = []
|
||||||
|
seen_tokens: Set[int] = set()
|
||||||
|
for span in spans:
|
||||||
|
start = span.start
|
||||||
|
end = span.end
|
||||||
|
if all(token.i not in seen_tokens for token in span):
|
||||||
|
new_entities.append(span)
|
||||||
|
entities = [e for e in entities if not (e.start < end and e.end > start)]
|
||||||
|
seen_tokens.update(range(start, end))
|
||||||
|
return entities + new_entities
|
||||||
|
|
||||||
|
|
||||||
|
@registry.misc("spacy.prioritize_new_ents_filter.v1")
|
||||||
|
def make_prioritize_new_ents_filter():
|
||||||
|
return prioritize_new_ents_filter
|
||||||
|
|
||||||
|
|
||||||
|
def prioritize_existing_ents_filter(
|
||||||
|
entities: Iterable[Span], spans: Iterable[Span]
|
||||||
|
) -> List[Span]:
|
||||||
|
"""Merge entities and spans into one list without overlaps by prioritizing
|
||||||
|
existing entities. Intended to replicate the overwrite_ents=False behavior
|
||||||
|
from the EntityRuler.
|
||||||
|
|
||||||
|
entities (Iterable[Span]): The entities, already filtered for overlaps.
|
||||||
|
spans (Iterable[Span]): The spans to merge, may contain overlaps.
|
||||||
|
RETURNS (List[Span]): Filtered list of non-overlapping spans.
|
||||||
|
"""
|
||||||
|
get_sort_key = lambda span: (span.end - span.start, -span.start)
|
||||||
|
spans = sorted(spans, key=get_sort_key, reverse=True)
|
||||||
|
entities = list(entities)
|
||||||
|
new_entities = []
|
||||||
|
seen_tokens: Set[int] = set()
|
||||||
|
seen_tokens.update(*(range(ent.start, ent.end) for ent in entities))
|
||||||
|
for span in spans:
|
||||||
|
start = span.start
|
||||||
|
end = span.end
|
||||||
|
if all(token.i not in seen_tokens for token in span):
|
||||||
|
new_entities.append(span)
|
||||||
|
seen_tokens.update(range(start, end))
|
||||||
|
return entities + new_entities
|
||||||
|
|
||||||
|
|
||||||
|
@registry.misc("spacy.prioritize_existing_ents_filter.v1")
|
||||||
|
def make_preverse_existing_ents_filter():
|
||||||
|
return prioritize_existing_ents_filter
|
||||||
|
|
||||||
|
|
||||||
|
def overlapping_labeled_spans_score(
|
||||||
|
examples: Iterable[Example], *, spans_key=DEFAULT_SPANS_KEY, **kwargs
|
||||||
|
) -> Dict[str, Any]:
|
||||||
|
kwargs = dict(kwargs)
|
||||||
|
attr_prefix = f"spans_"
|
||||||
|
kwargs.setdefault("attr", f"{attr_prefix}{spans_key}")
|
||||||
|
kwargs.setdefault("allow_overlap", True)
|
||||||
|
kwargs.setdefault("labeled", True)
|
||||||
|
kwargs.setdefault(
|
||||||
|
"getter", lambda doc, key: doc.spans.get(key[len(attr_prefix) :], [])
|
||||||
|
)
|
||||||
|
kwargs.setdefault("has_annotation", lambda doc: spans_key in doc.spans)
|
||||||
|
return Scorer.score_spans(examples, **kwargs)
|
||||||
|
|
||||||
|
|
||||||
|
@registry.scorers("spacy.overlapping_labeled_spans_scorer.v1")
|
||||||
|
def make_overlapping_labeled_spans_scorer(spans_key: str = DEFAULT_SPANS_KEY):
|
||||||
|
return partial(overlapping_labeled_spans_score, spans_key=spans_key)
|
||||||
|
|
||||||
|
|
||||||
|
class SpanRuler(Pipe):
|
||||||
|
"""The SpanRuler lets you add spans to the `Doc.spans` using token-based
|
||||||
|
rules or exact phrase matches.
|
||||||
|
|
||||||
|
DOCS: https://spacy.io/api/spanruler
|
||||||
|
USAGE: https://spacy.io/usage/rule-based-matching#spanruler
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
nlp: Language,
|
||||||
|
name: str = "span_ruler",
|
||||||
|
*,
|
||||||
|
spans_key: Optional[str] = DEFAULT_SPANS_KEY,
|
||||||
|
spans_filter: Optional[
|
||||||
|
Callable[[Iterable[Span], Iterable[Span]], Iterable[Span]]
|
||||||
|
] = None,
|
||||||
|
annotate_ents: bool = False,
|
||||||
|
ents_filter: Callable[
|
||||||
|
[Iterable[Span], Iterable[Span]], Iterable[Span]
|
||||||
|
] = util.filter_chain_spans,
|
||||||
|
phrase_matcher_attr: Optional[Union[int, str]] = None,
|
||||||
|
validate: bool = False,
|
||||||
|
overwrite: bool = False,
|
||||||
|
scorer: Optional[Callable] = partial(
|
||||||
|
overlapping_labeled_spans_score, spans_key=DEFAULT_SPANS_KEY
|
||||||
|
),
|
||||||
|
) -> None:
|
||||||
|
"""Initialize the span ruler. If patterns are supplied here, they
|
||||||
|
need to be a list of dictionaries with a `"label"` and `"pattern"`
|
||||||
|
key. A pattern can either be a token pattern (list) or a phrase pattern
|
||||||
|
(string). For example: `{'label': 'ORG', 'pattern': 'Apple'}`.
|
||||||
|
|
||||||
|
nlp (Language): The shared nlp object to pass the vocab to the matchers
|
||||||
|
and process phrase patterns.
|
||||||
|
name (str): Instance name of the current pipeline component. Typically
|
||||||
|
passed in automatically from the factory when the component is
|
||||||
|
added. Used to disable the current span ruler while creating
|
||||||
|
phrase patterns with the nlp object.
|
||||||
|
spans_key (Optional[str]): The spans key to save the spans under. If
|
||||||
|
`None`, no spans are saved. Defaults to "ruler".
|
||||||
|
spans_filter (Optional[Callable[[Iterable[Span], Iterable[Span]], List[Span]]):
|
||||||
|
The optional method to filter spans before they are assigned to
|
||||||
|
doc.spans. Defaults to `None`.
|
||||||
|
annotate_ents (bool): Whether to save spans to doc.ents. Defaults to
|
||||||
|
`False`.
|
||||||
|
ents_filter (Callable[[Iterable[Span], Iterable[Span]], List[Span]]):
|
||||||
|
The method to filter spans before they are assigned to doc.ents.
|
||||||
|
Defaults to `util.filter_chain_spans`.
|
||||||
|
phrase_matcher_attr (Optional[Union[int, str]]): Token attribute to
|
||||||
|
match on, passed to the internal PhraseMatcher as `attr`. Defaults
|
||||||
|
to `None`.
|
||||||
|
validate (bool): Whether patterns should be validated, passed to
|
||||||
|
Matcher and PhraseMatcher as `validate`.
|
||||||
|
overwrite (bool): Whether to remove any existing spans under this spans
|
||||||
|
key if `spans_key` is set, and/or to remove any ents under `doc.ents` if
|
||||||
|
`annotate_ents` is set. Defaults to `True`.
|
||||||
|
scorer (Optional[Callable]): The scoring method. Defaults to
|
||||||
|
spacy.pipeline.span_ruler.overlapping_labeled_spans_score.
|
||||||
|
|
||||||
|
DOCS: https://spacy.io/api/spanruler#init
|
||||||
|
"""
|
||||||
|
self.nlp = nlp
|
||||||
|
self.name = name
|
||||||
|
self.spans_key = spans_key
|
||||||
|
self.annotate_ents = annotate_ents
|
||||||
|
self.phrase_matcher_attr = phrase_matcher_attr
|
||||||
|
self.validate = validate
|
||||||
|
self.overwrite = overwrite
|
||||||
|
self.spans_filter = spans_filter
|
||||||
|
self.ents_filter = ents_filter
|
||||||
|
self.scorer = scorer
|
||||||
|
self._match_label_id_map: Dict[int, Dict[str, str]] = {}
|
||||||
|
self.clear()
|
||||||
|
|
||||||
|
def __len__(self) -> int:
|
||||||
|
"""The number of all labels added to the span ruler."""
|
||||||
|
return len(self._patterns)
|
||||||
|
|
||||||
|
def __contains__(self, label: str) -> bool:
|
||||||
|
"""Whether a label is present in the patterns."""
|
||||||
|
for label_id in self._match_label_id_map.values():
|
||||||
|
if label_id["label"] == label:
|
||||||
|
return True
|
||||||
|
return False
|
||||||
|
|
||||||
|
@property
|
||||||
|
def key(self) -> Optional[str]:
|
||||||
|
"""Key of the doc.spans dict to save the spans under."""
|
||||||
|
return self.spans_key
|
||||||
|
|
||||||
|
def __call__(self, doc: Doc) -> Doc:
|
||||||
|
"""Find matches in document and add them as entities.
|
||||||
|
|
||||||
|
doc (Doc): The Doc object in the pipeline.
|
||||||
|
RETURNS (Doc): The Doc with added entities, if available.
|
||||||
|
|
||||||
|
DOCS: https://spacy.io/api/spanruler#call
|
||||||
|
"""
|
||||||
|
error_handler = self.get_error_handler()
|
||||||
|
try:
|
||||||
|
matches = self.match(doc)
|
||||||
|
self.set_annotations(doc, matches)
|
||||||
|
return doc
|
||||||
|
except Exception as e:
|
||||||
|
return error_handler(self.name, self, [doc], e)
|
||||||
|
|
||||||
|
def match(self, doc: Doc):
|
||||||
|
self._require_patterns()
|
||||||
|
with warnings.catch_warnings():
|
||||||
|
warnings.filterwarnings("ignore", message="\\[W036")
|
||||||
|
matches = cast(
|
||||||
|
List[Tuple[int, int, int]],
|
||||||
|
list(self.matcher(doc)) + list(self.phrase_matcher(doc)),
|
||||||
|
)
|
||||||
|
deduplicated_matches = set(
|
||||||
|
Span(
|
||||||
|
doc,
|
||||||
|
start,
|
||||||
|
end,
|
||||||
|
label=self._match_label_id_map[m_id]["label"],
|
||||||
|
span_id=self._match_label_id_map[m_id]["id"],
|
||||||
|
)
|
||||||
|
for m_id, start, end in matches
|
||||||
|
if start != end
|
||||||
|
)
|
||||||
|
return sorted(list(deduplicated_matches))
|
||||||
|
|
||||||
|
def set_annotations(self, doc, matches):
|
||||||
|
"""Modify the document in place"""
|
||||||
|
# set doc.spans if spans_key is set
|
||||||
|
if self.key:
|
||||||
|
spans = []
|
||||||
|
if self.key in doc.spans and not self.overwrite:
|
||||||
|
spans = doc.spans[self.key]
|
||||||
|
spans.extend(
|
||||||
|
self.spans_filter(spans, matches) if self.spans_filter else matches
|
||||||
|
)
|
||||||
|
doc.spans[self.key] = spans
|
||||||
|
# set doc.ents if annotate_ents is set
|
||||||
|
if self.annotate_ents:
|
||||||
|
spans = []
|
||||||
|
if not self.overwrite:
|
||||||
|
spans = list(doc.ents)
|
||||||
|
spans = self.ents_filter(spans, matches)
|
||||||
|
try:
|
||||||
|
doc.ents = sorted(spans)
|
||||||
|
except ValueError:
|
||||||
|
raise ValueError(Errors.E854)
|
||||||
|
|
||||||
|
@property
|
||||||
|
def labels(self) -> Tuple[str, ...]:
|
||||||
|
"""All labels present in the match patterns.
|
||||||
|
|
||||||
|
RETURNS (set): The string labels.
|
||||||
|
|
||||||
|
DOCS: https://spacy.io/api/spanruler#labels
|
||||||
|
"""
|
||||||
|
return tuple(sorted(set([cast(str, p["label"]) for p in self._patterns])))
|
||||||
|
|
||||||
|
@property
|
||||||
|
def ids(self) -> Tuple[str, ...]:
|
||||||
|
"""All IDs present in the match patterns.
|
||||||
|
|
||||||
|
RETURNS (set): The string IDs.
|
||||||
|
|
||||||
|
DOCS: https://spacy.io/api/spanruler#ids
|
||||||
|
"""
|
||||||
|
return tuple(
|
||||||
|
sorted(set([cast(str, p.get("id")) for p in self._patterns]) - set([None]))
|
||||||
|
)
|
||||||
|
|
||||||
|
def initialize(
|
||||||
|
self,
|
||||||
|
get_examples: Callable[[], Iterable[Example]],
|
||||||
|
*,
|
||||||
|
nlp: Optional[Language] = None,
|
||||||
|
patterns: Optional[Sequence[PatternType]] = None,
|
||||||
|
):
|
||||||
|
"""Initialize the pipe for training.
|
||||||
|
|
||||||
|
get_examples (Callable[[], Iterable[Example]]): Function that
|
||||||
|
returns a representative sample of gold-standard Example objects.
|
||||||
|
nlp (Language): The current nlp object the component is part of.
|
||||||
|
patterns (Optional[Iterable[PatternType]]): The list of patterns.
|
||||||
|
|
||||||
|
DOCS: https://spacy.io/api/spanruler#initialize
|
||||||
|
"""
|
||||||
|
self.clear()
|
||||||
|
if patterns:
|
||||||
|
self.add_patterns(patterns) # type: ignore[arg-type]
|
||||||
|
|
||||||
|
@property
|
||||||
|
def patterns(self) -> List[PatternType]:
|
||||||
|
"""Get all patterns that were added to the span ruler.
|
||||||
|
|
||||||
|
RETURNS (list): The original patterns, one dictionary per pattern.
|
||||||
|
|
||||||
|
DOCS: https://spacy.io/api/spanruler#patterns
|
||||||
|
"""
|
||||||
|
return self._patterns
|
||||||
|
|
||||||
|
def add_patterns(self, patterns: List[PatternType]) -> None:
|
||||||
|
"""Add patterns to the span ruler. A pattern can either be a token
|
||||||
|
pattern (list of dicts) or a phrase pattern (string). For example:
|
||||||
|
{'label': 'ORG', 'pattern': 'Apple'}
|
||||||
|
{'label': 'ORG', 'pattern': 'Apple', 'id': 'apple'}
|
||||||
|
{'label': 'GPE', 'pattern': [{'lower': 'san'}, {'lower': 'francisco'}]}
|
||||||
|
|
||||||
|
patterns (list): The patterns to add.
|
||||||
|
|
||||||
|
DOCS: https://spacy.io/api/spanruler#add_patterns
|
||||||
|
"""
|
||||||
|
|
||||||
|
# disable the nlp components after this one in case they haven't been
|
||||||
|
# initialized / deserialized yet
|
||||||
|
try:
|
||||||
|
current_index = -1
|
||||||
|
for i, (name, pipe) in enumerate(self.nlp.pipeline):
|
||||||
|
if self == pipe:
|
||||||
|
current_index = i
|
||||||
|
break
|
||||||
|
subsequent_pipes = [pipe for pipe in self.nlp.pipe_names[current_index:]]
|
||||||
|
except ValueError:
|
||||||
|
subsequent_pipes = []
|
||||||
|
with self.nlp.select_pipes(disable=subsequent_pipes):
|
||||||
|
phrase_pattern_labels = []
|
||||||
|
phrase_pattern_texts = []
|
||||||
|
for entry in patterns:
|
||||||
|
p_label = cast(str, entry["label"])
|
||||||
|
p_id = cast(str, entry.get("id", ""))
|
||||||
|
label = repr((p_label, p_id))
|
||||||
|
self._match_label_id_map[self.nlp.vocab.strings.as_int(label)] = {
|
||||||
|
"label": p_label,
|
||||||
|
"id": p_id,
|
||||||
|
}
|
||||||
|
if isinstance(entry["pattern"], str):
|
||||||
|
phrase_pattern_labels.append(label)
|
||||||
|
phrase_pattern_texts.append(entry["pattern"])
|
||||||
|
elif isinstance(entry["pattern"], list):
|
||||||
|
self.matcher.add(label, [entry["pattern"]])
|
||||||
|
else:
|
||||||
|
raise ValueError(Errors.E097.format(pattern=entry["pattern"]))
|
||||||
|
self._patterns.append(entry)
|
||||||
|
for label, pattern in zip(
|
||||||
|
phrase_pattern_labels,
|
||||||
|
self.nlp.pipe(phrase_pattern_texts),
|
||||||
|
):
|
||||||
|
self.phrase_matcher.add(label, [pattern])
|
||||||
|
|
||||||
|
def clear(self) -> None:
|
||||||
|
"""Reset all patterns.
|
||||||
|
|
||||||
|
RETURNS: None
|
||||||
|
DOCS: https://spacy.io/api/spanruler#clear
|
||||||
|
"""
|
||||||
|
self._patterns: List[PatternType] = []
|
||||||
|
self.matcher: Matcher = Matcher(self.nlp.vocab, validate=self.validate)
|
||||||
|
self.phrase_matcher: PhraseMatcher = PhraseMatcher(
|
||||||
|
self.nlp.vocab,
|
||||||
|
attr=self.phrase_matcher_attr,
|
||||||
|
validate=self.validate,
|
||||||
|
)
|
||||||
|
|
||||||
|
def remove(self, label: str) -> None:
|
||||||
|
"""Remove a pattern by its label.
|
||||||
|
|
||||||
|
label (str): Label of the pattern to be removed.
|
||||||
|
RETURNS: None
|
||||||
|
DOCS: https://spacy.io/api/spanruler#remove
|
||||||
|
"""
|
||||||
|
if label not in self:
|
||||||
|
raise ValueError(
|
||||||
|
Errors.E1024.format(attr_type="label", label=label, component=self.name)
|
||||||
|
)
|
||||||
|
self._patterns = [p for p in self._patterns if p["label"] != label]
|
||||||
|
for m_label in self._match_label_id_map:
|
||||||
|
if self._match_label_id_map[m_label]["label"] == label:
|
||||||
|
m_label_str = self.nlp.vocab.strings.as_string(m_label)
|
||||||
|
if m_label_str in self.phrase_matcher:
|
||||||
|
self.phrase_matcher.remove(m_label_str)
|
||||||
|
if m_label_str in self.matcher:
|
||||||
|
self.matcher.remove(m_label_str)
|
||||||
|
|
||||||
|
def remove_by_id(self, pattern_id: str) -> None:
|
||||||
|
"""Remove a pattern by its pattern ID.
|
||||||
|
|
||||||
|
pattern_id (str): ID of the pattern to be removed.
|
||||||
|
RETURNS: None
|
||||||
|
DOCS: https://spacy.io/api/spanruler#remove_by_id
|
||||||
|
"""
|
||||||
|
orig_len = len(self)
|
||||||
|
self._patterns = [p for p in self._patterns if p.get("id") != pattern_id]
|
||||||
|
if orig_len == len(self):
|
||||||
|
raise ValueError(
|
||||||
|
Errors.E1024.format(
|
||||||
|
attr_type="ID", label=pattern_id, component=self.name
|
||||||
|
)
|
||||||
|
)
|
||||||
|
for m_label in self._match_label_id_map:
|
||||||
|
if self._match_label_id_map[m_label]["id"] == pattern_id:
|
||||||
|
m_label_str = self.nlp.vocab.strings.as_string(m_label)
|
||||||
|
if m_label_str in self.phrase_matcher:
|
||||||
|
self.phrase_matcher.remove(m_label_str)
|
||||||
|
if m_label_str in self.matcher:
|
||||||
|
self.matcher.remove(m_label_str)
|
||||||
|
|
||||||
|
def _require_patterns(self) -> None:
|
||||||
|
"""Raise a warning if this component has no patterns defined."""
|
||||||
|
if len(self) == 0:
|
||||||
|
warnings.warn(Warnings.W036.format(name=self.name))
|
||||||
|
|
||||||
|
def from_bytes(
|
||||||
|
self, bytes_data: bytes, *, exclude: Iterable[str] = SimpleFrozenList()
|
||||||
|
) -> "SpanRuler":
|
||||||
|
"""Load the span ruler from a bytestring.
|
||||||
|
|
||||||
|
bytes_data (bytes): The bytestring to load.
|
||||||
|
RETURNS (SpanRuler): The loaded span ruler.
|
||||||
|
|
||||||
|
DOCS: https://spacy.io/api/spanruler#from_bytes
|
||||||
|
"""
|
||||||
|
self.clear()
|
||||||
|
deserializers = {
|
||||||
|
"patterns": lambda b: self.add_patterns(srsly.json_loads(b)),
|
||||||
|
}
|
||||||
|
util.from_bytes(bytes_data, deserializers, exclude)
|
||||||
|
return self
|
||||||
|
|
||||||
|
def to_bytes(self, *, exclude: Iterable[str] = SimpleFrozenList()) -> bytes:
|
||||||
|
"""Serialize the span ruler to a bytestring.
|
||||||
|
|
||||||
|
RETURNS (bytes): The serialized patterns.
|
||||||
|
|
||||||
|
DOCS: https://spacy.io/api/spanruler#to_bytes
|
||||||
|
"""
|
||||||
|
serializers = {
|
||||||
|
"patterns": lambda: srsly.json_dumps(self.patterns),
|
||||||
|
}
|
||||||
|
return util.to_bytes(serializers, exclude)
|
||||||
|
|
||||||
|
def from_disk(
|
||||||
|
self, path: Union[str, Path], *, exclude: Iterable[str] = SimpleFrozenList()
|
||||||
|
) -> "SpanRuler":
|
||||||
|
"""Load the span ruler from a directory.
|
||||||
|
|
||||||
|
path (Union[str, Path]): A path to a directory.
|
||||||
|
RETURNS (SpanRuler): The loaded span ruler.
|
||||||
|
|
||||||
|
DOCS: https://spacy.io/api/spanruler#from_disk
|
||||||
|
"""
|
||||||
|
self.clear()
|
||||||
|
path = ensure_path(path)
|
||||||
|
deserializers = {
|
||||||
|
"patterns": lambda p: self.add_patterns(srsly.read_jsonl(p)),
|
||||||
|
}
|
||||||
|
util.from_disk(path, deserializers, {})
|
||||||
|
return self
|
||||||
|
|
||||||
|
def to_disk(
|
||||||
|
self, path: Union[str, Path], *, exclude: Iterable[str] = SimpleFrozenList()
|
||||||
|
) -> None:
|
||||||
|
"""Save the span ruler patterns to a directory.
|
||||||
|
|
||||||
|
path (Union[str, Path]): A path to a directory.
|
||||||
|
|
||||||
|
DOCS: https://spacy.io/api/spanruler#to_disk
|
||||||
|
"""
|
||||||
|
path = ensure_path(path)
|
||||||
|
serializers = {
|
||||||
|
"patterns": lambda p: srsly.write_jsonl(p, self.patterns),
|
||||||
|
}
|
||||||
|
util.to_disk(path, serializers, {})
|
|
@ -75,7 +75,7 @@ def build_ngram_suggester(sizes: List[int]) -> Suggester:
|
||||||
if spans:
|
if spans:
|
||||||
assert spans[-1].ndim == 2, spans[-1].shape
|
assert spans[-1].ndim == 2, spans[-1].shape
|
||||||
lengths.append(length)
|
lengths.append(length)
|
||||||
lengths_array = cast(Ints1d, ops.asarray(lengths, dtype="i"))
|
lengths_array = ops.asarray1i(lengths)
|
||||||
if len(spans) > 0:
|
if len(spans) > 0:
|
||||||
output = Ragged(ops.xp.vstack(spans), lengths_array)
|
output = Ragged(ops.xp.vstack(spans), lengths_array)
|
||||||
else:
|
else:
|
||||||
|
|
|
@ -192,7 +192,7 @@ class TextCategorizer(TrainablePipe):
|
||||||
if not any(len(doc) for doc in docs):
|
if not any(len(doc) for doc in docs):
|
||||||
# Handle cases where there are no tokens in any docs.
|
# Handle cases where there are no tokens in any docs.
|
||||||
tensors = [doc.tensor for doc in docs]
|
tensors = [doc.tensor for doc in docs]
|
||||||
xp = get_array_module(tensors)
|
xp = self.model.ops.xp
|
||||||
scores = xp.zeros((len(list(docs)), len(self.labels)))
|
scores = xp.zeros((len(list(docs)), len(self.labels)))
|
||||||
return scores
|
return scores
|
||||||
scores = self.model.predict(docs)
|
scores = self.model.predict(docs)
|
||||||
|
|
|
@ -1,4 +1,5 @@
|
||||||
from cymem.cymem cimport Pool
|
from cymem.cymem cimport Pool
|
||||||
|
from thinc.backends.cblas cimport CBlas
|
||||||
|
|
||||||
from ..vocab cimport Vocab
|
from ..vocab cimport Vocab
|
||||||
from .trainable_pipe cimport TrainablePipe
|
from .trainable_pipe cimport TrainablePipe
|
||||||
|
@ -12,7 +13,7 @@ cdef class Parser(TrainablePipe):
|
||||||
cdef readonly TransitionSystem moves
|
cdef readonly TransitionSystem moves
|
||||||
cdef public object _multitasks
|
cdef public object _multitasks
|
||||||
|
|
||||||
cdef void _parseC(self, StateC** states,
|
cdef void _parseC(self, CBlas cblas, StateC** states,
|
||||||
WeightsC weights, SizesC sizes) nogil
|
WeightsC weights, SizesC sizes) nogil
|
||||||
|
|
||||||
cdef void c_transition_batch(self, StateC** states, const float* scores,
|
cdef void c_transition_batch(self, StateC** states, const float* scores,
|
||||||
|
|
|
@ -9,7 +9,7 @@ from libc.stdlib cimport calloc, free
|
||||||
import random
|
import random
|
||||||
|
|
||||||
import srsly
|
import srsly
|
||||||
from thinc.api import set_dropout_rate, CupyOps
|
from thinc.api import get_ops, set_dropout_rate, CupyOps, NumpyOps
|
||||||
from thinc.extra.search cimport Beam
|
from thinc.extra.search cimport Beam
|
||||||
import numpy.random
|
import numpy.random
|
||||||
import numpy
|
import numpy
|
||||||
|
@ -30,6 +30,9 @@ from ..errors import Errors, Warnings
|
||||||
from .. import util
|
from .. import util
|
||||||
|
|
||||||
|
|
||||||
|
NUMPY_OPS = NumpyOps()
|
||||||
|
|
||||||
|
|
||||||
cdef class Parser(TrainablePipe):
|
cdef class Parser(TrainablePipe):
|
||||||
"""
|
"""
|
||||||
Base class of the DependencyParser and EntityRecognizer.
|
Base class of the DependencyParser and EntityRecognizer.
|
||||||
|
@ -259,6 +262,12 @@ cdef class Parser(TrainablePipe):
|
||||||
def greedy_parse(self, docs, drop=0.):
|
def greedy_parse(self, docs, drop=0.):
|
||||||
cdef vector[StateC*] states
|
cdef vector[StateC*] states
|
||||||
cdef StateClass state
|
cdef StateClass state
|
||||||
|
ops = self.model.ops
|
||||||
|
cdef CBlas cblas
|
||||||
|
if isinstance(ops, CupyOps):
|
||||||
|
cblas = NUMPY_OPS.cblas()
|
||||||
|
else:
|
||||||
|
cblas = ops.cblas()
|
||||||
self._ensure_labels_are_added(docs)
|
self._ensure_labels_are_added(docs)
|
||||||
set_dropout_rate(self.model, drop)
|
set_dropout_rate(self.model, drop)
|
||||||
batch = self.moves.init_batch(docs)
|
batch = self.moves.init_batch(docs)
|
||||||
|
@ -269,8 +278,7 @@ cdef class Parser(TrainablePipe):
|
||||||
states.push_back(state.c)
|
states.push_back(state.c)
|
||||||
sizes = get_c_sizes(model, states.size())
|
sizes = get_c_sizes(model, states.size())
|
||||||
with nogil:
|
with nogil:
|
||||||
self._parseC(&states[0],
|
self._parseC(cblas, &states[0], weights, sizes)
|
||||||
weights, sizes)
|
|
||||||
model.clear_memory()
|
model.clear_memory()
|
||||||
del model
|
del model
|
||||||
return batch
|
return batch
|
||||||
|
@ -297,14 +305,13 @@ cdef class Parser(TrainablePipe):
|
||||||
del model
|
del model
|
||||||
return list(batch)
|
return list(batch)
|
||||||
|
|
||||||
cdef void _parseC(self, StateC** states,
|
cdef void _parseC(self, CBlas cblas, StateC** states,
|
||||||
WeightsC weights, SizesC sizes) nogil:
|
WeightsC weights, SizesC sizes) nogil:
|
||||||
cdef int i, j
|
cdef int i, j
|
||||||
cdef vector[StateC*] unfinished
|
cdef vector[StateC*] unfinished
|
||||||
cdef ActivationsC activations = alloc_activations(sizes)
|
cdef ActivationsC activations = alloc_activations(sizes)
|
||||||
while sizes.states >= 1:
|
while sizes.states >= 1:
|
||||||
predict_states(&activations,
|
predict_states(cblas, &activations, states, &weights, sizes)
|
||||||
states, &weights, sizes)
|
|
||||||
# Validate actions, argmax, take action.
|
# Validate actions, argmax, take action.
|
||||||
self.c_transition_batch(states,
|
self.c_transition_batch(states,
|
||||||
activations.scores, sizes.classes, sizes.states)
|
activations.scores, sizes.classes, sizes.states)
|
||||||
|
|
|
@ -3,12 +3,13 @@ from typing import Iterable, TypeVar, TYPE_CHECKING
|
||||||
from .compat import Literal
|
from .compat import Literal
|
||||||
from enum import Enum
|
from enum import Enum
|
||||||
from pydantic import BaseModel, Field, ValidationError, validator, create_model
|
from pydantic import BaseModel, Field, ValidationError, validator, create_model
|
||||||
from pydantic import StrictStr, StrictInt, StrictFloat, StrictBool
|
from pydantic import StrictStr, StrictInt, StrictFloat, StrictBool, ConstrainedStr
|
||||||
from pydantic.main import ModelMetaclass
|
from pydantic.main import ModelMetaclass
|
||||||
from thinc.api import Optimizer, ConfigValidationError, Model
|
from thinc.api import Optimizer, ConfigValidationError, Model
|
||||||
from thinc.config import Promise
|
from thinc.config import Promise
|
||||||
from collections import defaultdict
|
from collections import defaultdict
|
||||||
import inspect
|
import inspect
|
||||||
|
import re
|
||||||
|
|
||||||
from .attrs import NAMES
|
from .attrs import NAMES
|
||||||
from .lookups import Lookups
|
from .lookups import Lookups
|
||||||
|
@ -104,7 +105,7 @@ def get_arg_model(
|
||||||
sig_args[param.name] = (annotation, default)
|
sig_args[param.name] = (annotation, default)
|
||||||
is_strict = strict and not has_variable
|
is_strict = strict and not has_variable
|
||||||
sig_args["__config__"] = ArgSchemaConfig if is_strict else ArgSchemaConfigExtra # type: ignore[assignment]
|
sig_args["__config__"] = ArgSchemaConfig if is_strict else ArgSchemaConfigExtra # type: ignore[assignment]
|
||||||
return create_model(name, **sig_args) # type: ignore[arg-type, return-value]
|
return create_model(name, **sig_args) # type: ignore[call-overload, arg-type, return-value]
|
||||||
|
|
||||||
|
|
||||||
def validate_init_settings(
|
def validate_init_settings(
|
||||||
|
@ -198,13 +199,18 @@ class TokenPatternNumber(BaseModel):
|
||||||
return v
|
return v
|
||||||
|
|
||||||
|
|
||||||
class TokenPatternOperator(str, Enum):
|
class TokenPatternOperatorSimple(str, Enum):
|
||||||
plus: StrictStr = StrictStr("+")
|
plus: StrictStr = StrictStr("+")
|
||||||
start: StrictStr = StrictStr("*")
|
star: StrictStr = StrictStr("*")
|
||||||
question: StrictStr = StrictStr("?")
|
question: StrictStr = StrictStr("?")
|
||||||
exclamation: StrictStr = StrictStr("!")
|
exclamation: StrictStr = StrictStr("!")
|
||||||
|
|
||||||
|
|
||||||
|
class TokenPatternOperatorMinMax(ConstrainedStr):
|
||||||
|
regex = re.compile("^({\d+}|{\d+,\d*}|{\d*,\d+})$")
|
||||||
|
|
||||||
|
|
||||||
|
TokenPatternOperator = Union[TokenPatternOperatorSimple, TokenPatternOperatorMinMax]
|
||||||
StringValue = Union[TokenPatternString, StrictStr]
|
StringValue = Union[TokenPatternString, StrictStr]
|
||||||
NumberValue = Union[TokenPatternNumber, StrictInt, StrictFloat]
|
NumberValue = Union[TokenPatternNumber, StrictInt, StrictFloat]
|
||||||
UnderscoreValue = Union[
|
UnderscoreValue = Union[
|
||||||
|
@ -485,3 +491,29 @@ class RecommendationSchema(BaseModel):
|
||||||
word_vectors: Optional[str] = None
|
word_vectors: Optional[str] = None
|
||||||
transformer: Optional[RecommendationTrf] = None
|
transformer: Optional[RecommendationTrf] = None
|
||||||
has_letters: bool = True
|
has_letters: bool = True
|
||||||
|
|
||||||
|
|
||||||
|
class DocJSONSchema(BaseModel):
|
||||||
|
"""
|
||||||
|
JSON/dict format for JSON representation of Doc objects.
|
||||||
|
"""
|
||||||
|
|
||||||
|
cats: Optional[Dict[StrictStr, StrictFloat]] = Field(
|
||||||
|
None, title="Categories with corresponding probabilities"
|
||||||
|
)
|
||||||
|
ents: Optional[List[Dict[StrictStr, Union[StrictInt, StrictStr]]]] = Field(
|
||||||
|
None, title="Information on entities"
|
||||||
|
)
|
||||||
|
sents: Optional[List[Dict[StrictStr, StrictInt]]] = Field(
|
||||||
|
None, title="Indices of sentences' start and end indices"
|
||||||
|
)
|
||||||
|
text: StrictStr = Field(..., title="Document text")
|
||||||
|
spans: Dict[StrictStr, List[Dict[StrictStr, Union[StrictStr, StrictInt]]]] = Field(
|
||||||
|
None, title="Span information - end/start indices, label, KB ID"
|
||||||
|
)
|
||||||
|
tokens: List[Dict[StrictStr, Union[StrictStr, StrictInt]]] = Field(
|
||||||
|
..., title="Token information - ID, start, annotations"
|
||||||
|
)
|
||||||
|
_: Optional[Dict[StrictStr, Any]] = Field(
|
||||||
|
None, title="Any custom data stored in the document's _ attribute"
|
||||||
|
)
|
||||||
|
|
|
@ -26,4 +26,4 @@ cdef class StringStore:
|
||||||
cdef public PreshMap _map
|
cdef public PreshMap _map
|
||||||
|
|
||||||
cdef const Utf8Str* intern_unicode(self, str py_string)
|
cdef const Utf8Str* intern_unicode(self, str py_string)
|
||||||
cdef const Utf8Str* _intern_utf8(self, char* utf8_string, int length)
|
cdef const Utf8Str* _intern_utf8(self, char* utf8_string, int length, hash_t* precalculated_hash)
|
||||||
|
|
|
@ -14,6 +14,13 @@ from .symbols import NAMES as SYMBOLS_BY_INT
|
||||||
from .errors import Errors
|
from .errors import Errors
|
||||||
from . import util
|
from . import util
|
||||||
|
|
||||||
|
# Not particularly elegant, but this is faster than `isinstance(key, numbers.Integral)`
|
||||||
|
cdef inline bint _try_coerce_to_hash(object key, hash_t* out_hash):
|
||||||
|
try:
|
||||||
|
out_hash[0] = key
|
||||||
|
return True
|
||||||
|
except:
|
||||||
|
return False
|
||||||
|
|
||||||
def get_string_id(key):
|
def get_string_id(key):
|
||||||
"""Get a string ID, handling the reserved symbols correctly. If the key is
|
"""Get a string ID, handling the reserved symbols correctly. If the key is
|
||||||
|
@ -22,15 +29,27 @@ def get_string_id(key):
|
||||||
This function optimises for convenience over performance, so shouldn't be
|
This function optimises for convenience over performance, so shouldn't be
|
||||||
used in tight loops.
|
used in tight loops.
|
||||||
"""
|
"""
|
||||||
if not isinstance(key, str):
|
cdef hash_t str_hash
|
||||||
return key
|
if isinstance(key, str):
|
||||||
elif key in SYMBOLS_BY_STR:
|
if len(key) == 0:
|
||||||
return SYMBOLS_BY_STR[key]
|
return 0
|
||||||
elif not key:
|
|
||||||
return 0
|
symbol = SYMBOLS_BY_STR.get(key, None)
|
||||||
|
if symbol is not None:
|
||||||
|
return symbol
|
||||||
|
else:
|
||||||
|
chars = key.encode("utf8")
|
||||||
|
return hash_utf8(chars, len(chars))
|
||||||
|
elif _try_coerce_to_hash(key, &str_hash):
|
||||||
|
# Coerce the integral key to the expected primitive hash type.
|
||||||
|
# This ensures that custom/overloaded "primitive" data types
|
||||||
|
# such as those implemented by numpy are not inadvertently used
|
||||||
|
# downsteam (as these are internally implemented as custom PyObjects
|
||||||
|
# whose comparison operators can incur a significant overhead).
|
||||||
|
return str_hash
|
||||||
else:
|
else:
|
||||||
chars = key.encode("utf8")
|
# TODO: Raise an error instead
|
||||||
return hash_utf8(chars, len(chars))
|
return key
|
||||||
|
|
||||||
|
|
||||||
cpdef hash_t hash_string(str string) except 0:
|
cpdef hash_t hash_string(str string) except 0:
|
||||||
|
@ -110,28 +129,36 @@ cdef class StringStore:
|
||||||
string_or_id (bytes, str or uint64): The value to encode.
|
string_or_id (bytes, str or uint64): The value to encode.
|
||||||
Returns (str / uint64): The value to be retrieved.
|
Returns (str / uint64): The value to be retrieved.
|
||||||
"""
|
"""
|
||||||
if isinstance(string_or_id, str) and len(string_or_id) == 0:
|
cdef hash_t str_hash
|
||||||
return 0
|
cdef Utf8Str* utf8str = NULL
|
||||||
elif string_or_id == 0:
|
|
||||||
return ""
|
|
||||||
elif string_or_id in SYMBOLS_BY_STR:
|
|
||||||
return SYMBOLS_BY_STR[string_or_id]
|
|
||||||
cdef hash_t key
|
|
||||||
if isinstance(string_or_id, str):
|
if isinstance(string_or_id, str):
|
||||||
key = hash_string(string_or_id)
|
if len(string_or_id) == 0:
|
||||||
return key
|
return 0
|
||||||
elif isinstance(string_or_id, bytes):
|
|
||||||
key = hash_utf8(string_or_id, len(string_or_id))
|
# Return early if the string is found in the symbols LUT.
|
||||||
return key
|
symbol = SYMBOLS_BY_STR.get(string_or_id, None)
|
||||||
elif string_or_id < len(SYMBOLS_BY_INT):
|
if symbol is not None:
|
||||||
return SYMBOLS_BY_INT[string_or_id]
|
return symbol
|
||||||
else:
|
|
||||||
key = string_or_id
|
|
||||||
utf8str = <Utf8Str*>self._map.get(key)
|
|
||||||
if utf8str is NULL:
|
|
||||||
raise KeyError(Errors.E018.format(hash_value=string_or_id))
|
|
||||||
else:
|
else:
|
||||||
return decode_Utf8Str(utf8str)
|
return hash_string(string_or_id)
|
||||||
|
elif isinstance(string_or_id, bytes):
|
||||||
|
return hash_utf8(string_or_id, len(string_or_id))
|
||||||
|
elif _try_coerce_to_hash(string_or_id, &str_hash):
|
||||||
|
if str_hash == 0:
|
||||||
|
return ""
|
||||||
|
elif str_hash < len(SYMBOLS_BY_INT):
|
||||||
|
return SYMBOLS_BY_INT[str_hash]
|
||||||
|
else:
|
||||||
|
utf8str = <Utf8Str*>self._map.get(str_hash)
|
||||||
|
else:
|
||||||
|
# TODO: Raise an error instead
|
||||||
|
utf8str = <Utf8Str*>self._map.get(string_or_id)
|
||||||
|
|
||||||
|
if utf8str is NULL:
|
||||||
|
raise KeyError(Errors.E018.format(hash_value=string_or_id))
|
||||||
|
else:
|
||||||
|
return decode_Utf8Str(utf8str)
|
||||||
|
|
||||||
def as_int(self, key):
|
def as_int(self, key):
|
||||||
"""If key is an int, return it; otherwise, get the int value."""
|
"""If key is an int, return it; otherwise, get the int value."""
|
||||||
|
@ -153,19 +180,22 @@ cdef class StringStore:
|
||||||
string (str): The string to add.
|
string (str): The string to add.
|
||||||
RETURNS (uint64): The string's hash value.
|
RETURNS (uint64): The string's hash value.
|
||||||
"""
|
"""
|
||||||
|
cdef hash_t str_hash
|
||||||
if isinstance(string, str):
|
if isinstance(string, str):
|
||||||
if string in SYMBOLS_BY_STR:
|
if string in SYMBOLS_BY_STR:
|
||||||
return SYMBOLS_BY_STR[string]
|
return SYMBOLS_BY_STR[string]
|
||||||
key = hash_string(string)
|
|
||||||
self.intern_unicode(string)
|
string = string.encode("utf8")
|
||||||
|
str_hash = hash_utf8(string, len(string))
|
||||||
|
self._intern_utf8(string, len(string), &str_hash)
|
||||||
elif isinstance(string, bytes):
|
elif isinstance(string, bytes):
|
||||||
if string in SYMBOLS_BY_STR:
|
if string in SYMBOLS_BY_STR:
|
||||||
return SYMBOLS_BY_STR[string]
|
return SYMBOLS_BY_STR[string]
|
||||||
key = hash_utf8(string, len(string))
|
str_hash = hash_utf8(string, len(string))
|
||||||
self._intern_utf8(string, len(string))
|
self._intern_utf8(string, len(string), &str_hash)
|
||||||
else:
|
else:
|
||||||
raise TypeError(Errors.E017.format(value_type=type(string)))
|
raise TypeError(Errors.E017.format(value_type=type(string)))
|
||||||
return key
|
return str_hash
|
||||||
|
|
||||||
def __len__(self):
|
def __len__(self):
|
||||||
"""The number of strings in the store.
|
"""The number of strings in the store.
|
||||||
|
@ -174,30 +204,29 @@ cdef class StringStore:
|
||||||
"""
|
"""
|
||||||
return self.keys.size()
|
return self.keys.size()
|
||||||
|
|
||||||
def __contains__(self, string not None):
|
def __contains__(self, string_or_id not None):
|
||||||
"""Check whether a string is in the store.
|
"""Check whether a string or ID is in the store.
|
||||||
|
|
||||||
string (str): The string to check.
|
string_or_id (str or int): The string to check.
|
||||||
RETURNS (bool): Whether the store contains the string.
|
RETURNS (bool): Whether the store contains the string.
|
||||||
"""
|
"""
|
||||||
cdef hash_t key
|
cdef hash_t str_hash
|
||||||
if isinstance(string, int) or isinstance(string, long):
|
if isinstance(string_or_id, str):
|
||||||
if string == 0:
|
if len(string_or_id) == 0:
|
||||||
return True
|
return True
|
||||||
key = string
|
elif string_or_id in SYMBOLS_BY_STR:
|
||||||
elif len(string) == 0:
|
return True
|
||||||
return True
|
str_hash = hash_string(string_or_id)
|
||||||
elif string in SYMBOLS_BY_STR:
|
elif _try_coerce_to_hash(string_or_id, &str_hash):
|
||||||
return True
|
pass
|
||||||
elif isinstance(string, str):
|
|
||||||
key = hash_string(string)
|
|
||||||
else:
|
else:
|
||||||
string = string.encode("utf8")
|
# TODO: Raise an error instead
|
||||||
key = hash_utf8(string, len(string))
|
return self._map.get(string_or_id) is not NULL
|
||||||
if key < len(SYMBOLS_BY_INT):
|
|
||||||
|
if str_hash < len(SYMBOLS_BY_INT):
|
||||||
return True
|
return True
|
||||||
else:
|
else:
|
||||||
return self._map.get(key) is not NULL
|
return self._map.get(str_hash) is not NULL
|
||||||
|
|
||||||
def __iter__(self):
|
def __iter__(self):
|
||||||
"""Iterate over the strings in the store, in order.
|
"""Iterate over the strings in the store, in order.
|
||||||
|
@ -272,13 +301,13 @@ cdef class StringStore:
|
||||||
cdef const Utf8Str* intern_unicode(self, str py_string):
|
cdef const Utf8Str* intern_unicode(self, str py_string):
|
||||||
# 0 means missing, but we don't bother offsetting the index.
|
# 0 means missing, but we don't bother offsetting the index.
|
||||||
cdef bytes byte_string = py_string.encode("utf8")
|
cdef bytes byte_string = py_string.encode("utf8")
|
||||||
return self._intern_utf8(byte_string, len(byte_string))
|
return self._intern_utf8(byte_string, len(byte_string), NULL)
|
||||||
|
|
||||||
@cython.final
|
@cython.final
|
||||||
cdef const Utf8Str* _intern_utf8(self, char* utf8_string, int length):
|
cdef const Utf8Str* _intern_utf8(self, char* utf8_string, int length, hash_t* precalculated_hash):
|
||||||
# TODO: This function's API/behaviour is an unholy mess...
|
# TODO: This function's API/behaviour is an unholy mess...
|
||||||
# 0 means missing, but we don't bother offsetting the index.
|
# 0 means missing, but we don't bother offsetting the index.
|
||||||
cdef hash_t key = hash_utf8(utf8_string, length)
|
cdef hash_t key = precalculated_hash[0] if precalculated_hash is not NULL else hash_utf8(utf8_string, length)
|
||||||
cdef Utf8Str* value = <Utf8Str*>self._map.get(key)
|
cdef Utf8Str* value = <Utf8Str*>self._map.get(key)
|
||||||
if value is not NULL:
|
if value is not NULL:
|
||||||
return value
|
return value
|
||||||
|
|
|
@ -1,5 +1,11 @@
|
||||||
import pytest
|
import pytest
|
||||||
from spacy.util import get_lang_class
|
from spacy.util import get_lang_class
|
||||||
|
from hypothesis import settings
|
||||||
|
|
||||||
|
# Functionally disable deadline settings for tests
|
||||||
|
# to prevent spurious test failures in CI builds.
|
||||||
|
settings.register_profile("no_deadlines", deadline=2 * 60 * 1000) # in ms
|
||||||
|
settings.load_profile("no_deadlines")
|
||||||
|
|
||||||
|
|
||||||
def pytest_addoption(parser):
|
def pytest_addoption(parser):
|
||||||
|
|
|
@ -11,7 +11,7 @@ from spacy.lang.en import English
|
||||||
from spacy.lang.xx import MultiLanguage
|
from spacy.lang.xx import MultiLanguage
|
||||||
from spacy.language import Language
|
from spacy.language import Language
|
||||||
from spacy.lexeme import Lexeme
|
from spacy.lexeme import Lexeme
|
||||||
from spacy.tokens import Doc, Span, Token
|
from spacy.tokens import Doc, Span, SpanGroup, Token
|
||||||
from spacy.vocab import Vocab
|
from spacy.vocab import Vocab
|
||||||
|
|
||||||
from .test_underscore import clean_underscore # noqa: F401
|
from .test_underscore import clean_underscore # noqa: F401
|
||||||
|
@ -964,3 +964,13 @@ def test_doc_spans_copy(en_tokenizer):
|
||||||
assert weakref.ref(doc1) == doc1.spans.doc_ref
|
assert weakref.ref(doc1) == doc1.spans.doc_ref
|
||||||
doc2 = doc1.copy()
|
doc2 = doc1.copy()
|
||||||
assert weakref.ref(doc2) == doc2.spans.doc_ref
|
assert weakref.ref(doc2) == doc2.spans.doc_ref
|
||||||
|
|
||||||
|
|
||||||
|
def test_doc_spans_setdefault(en_tokenizer):
|
||||||
|
doc = en_tokenizer("Some text about Colombia and the Czech Republic")
|
||||||
|
doc.spans.setdefault("key1")
|
||||||
|
assert len(doc.spans["key1"]) == 0
|
||||||
|
doc.spans.setdefault("key2", default=[doc[0:1]])
|
||||||
|
assert len(doc.spans["key2"]) == 1
|
||||||
|
doc.spans.setdefault("key3", default=SpanGroup(doc, spans=[doc[0:1], doc[1:2]]))
|
||||||
|
assert len(doc.spans["key3"]) == 2
|
||||||
|
|
191
spacy/tests/doc/test_json_doc_conversion.py
Normal file
191
spacy/tests/doc/test_json_doc_conversion.py
Normal file
|
@ -0,0 +1,191 @@
|
||||||
|
import pytest
|
||||||
|
import spacy
|
||||||
|
from spacy import schemas
|
||||||
|
from spacy.tokens import Doc, Span
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture()
|
||||||
|
def doc(en_vocab):
|
||||||
|
words = ["c", "d", "e"]
|
||||||
|
pos = ["VERB", "NOUN", "NOUN"]
|
||||||
|
tags = ["VBP", "NN", "NN"]
|
||||||
|
heads = [0, 0, 1]
|
||||||
|
deps = ["ROOT", "dobj", "dobj"]
|
||||||
|
ents = ["O", "B-ORG", "O"]
|
||||||
|
morphs = ["Feat1=A", "Feat1=B", "Feat1=A|Feat2=D"]
|
||||||
|
|
||||||
|
return Doc(
|
||||||
|
en_vocab,
|
||||||
|
words=words,
|
||||||
|
pos=pos,
|
||||||
|
tags=tags,
|
||||||
|
heads=heads,
|
||||||
|
deps=deps,
|
||||||
|
ents=ents,
|
||||||
|
morphs=morphs,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture()
|
||||||
|
def doc_without_deps(en_vocab):
|
||||||
|
words = ["c", "d", "e"]
|
||||||
|
pos = ["VERB", "NOUN", "NOUN"]
|
||||||
|
tags = ["VBP", "NN", "NN"]
|
||||||
|
ents = ["O", "B-ORG", "O"]
|
||||||
|
morphs = ["Feat1=A", "Feat1=B", "Feat1=A|Feat2=D"]
|
||||||
|
|
||||||
|
return Doc(
|
||||||
|
en_vocab,
|
||||||
|
words=words,
|
||||||
|
pos=pos,
|
||||||
|
tags=tags,
|
||||||
|
ents=ents,
|
||||||
|
morphs=morphs,
|
||||||
|
sent_starts=[True, False, True],
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_doc_to_json(doc):
|
||||||
|
json_doc = doc.to_json()
|
||||||
|
assert json_doc["text"] == "c d e "
|
||||||
|
assert len(json_doc["tokens"]) == 3
|
||||||
|
assert json_doc["tokens"][0]["pos"] == "VERB"
|
||||||
|
assert json_doc["tokens"][0]["tag"] == "VBP"
|
||||||
|
assert json_doc["tokens"][0]["dep"] == "ROOT"
|
||||||
|
assert len(json_doc["ents"]) == 1
|
||||||
|
assert json_doc["ents"][0]["start"] == 2 # character offset!
|
||||||
|
assert json_doc["ents"][0]["end"] == 3 # character offset!
|
||||||
|
assert json_doc["ents"][0]["label"] == "ORG"
|
||||||
|
assert not schemas.validate(schemas.DocJSONSchema, json_doc)
|
||||||
|
|
||||||
|
|
||||||
|
def test_doc_to_json_underscore(doc):
|
||||||
|
Doc.set_extension("json_test1", default=False)
|
||||||
|
Doc.set_extension("json_test2", default=False)
|
||||||
|
doc._.json_test1 = "hello world"
|
||||||
|
doc._.json_test2 = [1, 2, 3]
|
||||||
|
json_doc = doc.to_json(underscore=["json_test1", "json_test2"])
|
||||||
|
assert "_" in json_doc
|
||||||
|
assert json_doc["_"]["json_test1"] == "hello world"
|
||||||
|
assert json_doc["_"]["json_test2"] == [1, 2, 3]
|
||||||
|
assert not schemas.validate(schemas.DocJSONSchema, json_doc)
|
||||||
|
|
||||||
|
|
||||||
|
def test_doc_to_json_underscore_error_attr(doc):
|
||||||
|
"""Test that Doc.to_json() raises an error if a custom attribute doesn't
|
||||||
|
exist in the ._ space."""
|
||||||
|
with pytest.raises(ValueError):
|
||||||
|
doc.to_json(underscore=["json_test3"])
|
||||||
|
|
||||||
|
|
||||||
|
def test_doc_to_json_underscore_error_serialize(doc):
|
||||||
|
"""Test that Doc.to_json() raises an error if a custom attribute value
|
||||||
|
isn't JSON-serializable."""
|
||||||
|
Doc.set_extension("json_test4", method=lambda doc: doc.text)
|
||||||
|
with pytest.raises(ValueError):
|
||||||
|
doc.to_json(underscore=["json_test4"])
|
||||||
|
|
||||||
|
|
||||||
|
def test_doc_to_json_span(doc):
|
||||||
|
"""Test that Doc.to_json() includes spans"""
|
||||||
|
doc.spans["test"] = [Span(doc, 0, 2, "test"), Span(doc, 0, 1, "test")]
|
||||||
|
json_doc = doc.to_json()
|
||||||
|
assert "spans" in json_doc
|
||||||
|
assert len(json_doc["spans"]) == 1
|
||||||
|
assert len(json_doc["spans"]["test"]) == 2
|
||||||
|
assert json_doc["spans"]["test"][0]["start"] == 0
|
||||||
|
assert not schemas.validate(schemas.DocJSONSchema, json_doc)
|
||||||
|
|
||||||
|
|
||||||
|
def test_json_to_doc(doc):
|
||||||
|
new_doc = Doc(doc.vocab).from_json(doc.to_json(), validate=True)
|
||||||
|
new_tokens = [token for token in new_doc]
|
||||||
|
assert new_doc.text == doc.text == "c d e "
|
||||||
|
assert len(new_tokens) == len([token for token in doc]) == 3
|
||||||
|
assert new_tokens[0].pos == doc[0].pos
|
||||||
|
assert new_tokens[0].tag == doc[0].tag
|
||||||
|
assert new_tokens[0].dep == doc[0].dep
|
||||||
|
assert new_tokens[0].head.idx == doc[0].head.idx
|
||||||
|
assert new_tokens[0].lemma == doc[0].lemma
|
||||||
|
assert len(new_doc.ents) == 1
|
||||||
|
assert new_doc.ents[0].start == 1
|
||||||
|
assert new_doc.ents[0].end == 2
|
||||||
|
assert new_doc.ents[0].label_ == "ORG"
|
||||||
|
|
||||||
|
|
||||||
|
def test_json_to_doc_underscore(doc):
|
||||||
|
if not Doc.has_extension("json_test1"):
|
||||||
|
Doc.set_extension("json_test1", default=False)
|
||||||
|
if not Doc.has_extension("json_test2"):
|
||||||
|
Doc.set_extension("json_test2", default=False)
|
||||||
|
|
||||||
|
doc._.json_test1 = "hello world"
|
||||||
|
doc._.json_test2 = [1, 2, 3]
|
||||||
|
json_doc = doc.to_json(underscore=["json_test1", "json_test2"])
|
||||||
|
new_doc = Doc(doc.vocab).from_json(json_doc, validate=True)
|
||||||
|
assert all([new_doc.has_extension(f"json_test{i}") for i in range(1, 3)])
|
||||||
|
assert new_doc._.json_test1 == "hello world"
|
||||||
|
assert new_doc._.json_test2 == [1, 2, 3]
|
||||||
|
|
||||||
|
|
||||||
|
def test_json_to_doc_spans(doc):
|
||||||
|
"""Test that Doc.from_json() includes correct.spans."""
|
||||||
|
doc.spans["test"] = [
|
||||||
|
Span(doc, 0, 2, label="test"),
|
||||||
|
Span(doc, 0, 1, label="test", kb_id=7),
|
||||||
|
]
|
||||||
|
json_doc = doc.to_json()
|
||||||
|
new_doc = Doc(doc.vocab).from_json(json_doc, validate=True)
|
||||||
|
assert len(new_doc.spans) == 1
|
||||||
|
assert len(new_doc.spans["test"]) == 2
|
||||||
|
for i in range(2):
|
||||||
|
assert new_doc.spans["test"][i].start == doc.spans["test"][i].start
|
||||||
|
assert new_doc.spans["test"][i].end == doc.spans["test"][i].end
|
||||||
|
assert new_doc.spans["test"][i].label == doc.spans["test"][i].label
|
||||||
|
assert new_doc.spans["test"][i].kb_id == doc.spans["test"][i].kb_id
|
||||||
|
|
||||||
|
|
||||||
|
def test_json_to_doc_sents(doc, doc_without_deps):
|
||||||
|
"""Test that Doc.from_json() includes correct.sents."""
|
||||||
|
for test_doc in (doc, doc_without_deps):
|
||||||
|
json_doc = test_doc.to_json()
|
||||||
|
new_doc = Doc(doc.vocab).from_json(json_doc, validate=True)
|
||||||
|
assert [sent.text for sent in test_doc.sents] == [
|
||||||
|
sent.text for sent in new_doc.sents
|
||||||
|
]
|
||||||
|
assert [token.is_sent_start for token in test_doc] == [
|
||||||
|
token.is_sent_start for token in new_doc
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def test_json_to_doc_cats(doc):
|
||||||
|
"""Test that Doc.from_json() includes correct .cats."""
|
||||||
|
cats = {"A": 0.3, "B": 0.7}
|
||||||
|
doc.cats = cats
|
||||||
|
json_doc = doc.to_json()
|
||||||
|
new_doc = Doc(doc.vocab).from_json(json_doc, validate=True)
|
||||||
|
assert new_doc.cats == cats
|
||||||
|
|
||||||
|
|
||||||
|
def test_json_to_doc_spaces():
|
||||||
|
"""Test that Doc.from_json() preserves spaces correctly."""
|
||||||
|
doc = spacy.blank("en")("This is just brilliant.")
|
||||||
|
json_doc = doc.to_json()
|
||||||
|
new_doc = Doc(doc.vocab).from_json(json_doc, validate=True)
|
||||||
|
assert doc.text == new_doc.text
|
||||||
|
|
||||||
|
|
||||||
|
def test_json_to_doc_attribute_consistency(doc):
|
||||||
|
"""Test that Doc.from_json() raises an exception if tokens don't all have the same set of properties."""
|
||||||
|
doc_json = doc.to_json()
|
||||||
|
doc_json["tokens"][1].pop("morph")
|
||||||
|
with pytest.raises(ValueError):
|
||||||
|
Doc(doc.vocab).from_json(doc_json)
|
||||||
|
|
||||||
|
|
||||||
|
def test_json_to_doc_validation_error(doc):
|
||||||
|
"""Test that Doc.from_json() raises an exception when validating invalid input."""
|
||||||
|
doc_json = doc.to_json()
|
||||||
|
doc_json.pop("tokens")
|
||||||
|
with pytest.raises(ValueError):
|
||||||
|
Doc(doc.vocab).from_json(doc_json, validate=True)
|
|
@ -5,11 +5,9 @@ from spacy.compat import pickle
|
||||||
def test_pickle_single_doc():
|
def test_pickle_single_doc():
|
||||||
nlp = Language()
|
nlp = Language()
|
||||||
doc = nlp("pickle roundtrip")
|
doc = nlp("pickle roundtrip")
|
||||||
doc._context = 3
|
|
||||||
data = pickle.dumps(doc, 1)
|
data = pickle.dumps(doc, 1)
|
||||||
doc2 = pickle.loads(data)
|
doc2 = pickle.loads(data)
|
||||||
assert doc2.text == "pickle roundtrip"
|
assert doc2.text == "pickle roundtrip"
|
||||||
assert doc2._context == 3
|
|
||||||
|
|
||||||
|
|
||||||
def test_list_of_docs_pickles_efficiently():
|
def test_list_of_docs_pickles_efficiently():
|
||||||
|
|
|
@ -428,10 +428,19 @@ def test_span_string_label_kb_id(doc):
|
||||||
assert span.kb_id == doc.vocab.strings["Q342"]
|
assert span.kb_id == doc.vocab.strings["Q342"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_span_string_label_id(doc):
|
||||||
|
span = Span(doc, 0, 1, label="hello", span_id="Q342")
|
||||||
|
assert span.label_ == "hello"
|
||||||
|
assert span.label == doc.vocab.strings["hello"]
|
||||||
|
assert span.id_ == "Q342"
|
||||||
|
assert span.id == doc.vocab.strings["Q342"]
|
||||||
|
|
||||||
|
|
||||||
def test_span_attrs_writable(doc):
|
def test_span_attrs_writable(doc):
|
||||||
span = Span(doc, 0, 1)
|
span = Span(doc, 0, 1)
|
||||||
span.label_ = "label"
|
span.label_ = "label"
|
||||||
span.kb_id_ = "kb_id"
|
span.kb_id_ = "kb_id"
|
||||||
|
span.id_ = "id"
|
||||||
|
|
||||||
|
|
||||||
def test_span_ents_property(doc):
|
def test_span_ents_property(doc):
|
||||||
|
@ -619,6 +628,9 @@ def test_span_comparison(doc):
|
||||||
assert Span(doc, 0, 4, "LABEL", kb_id="KB_ID") <= Span(doc, 1, 3)
|
assert Span(doc, 0, 4, "LABEL", kb_id="KB_ID") <= Span(doc, 1, 3)
|
||||||
assert Span(doc, 1, 3) > Span(doc, 0, 4, "LABEL", kb_id="KB_ID")
|
assert Span(doc, 1, 3) > Span(doc, 0, 4, "LABEL", kb_id="KB_ID")
|
||||||
assert Span(doc, 1, 3) >= Span(doc, 0, 4, "LABEL", kb_id="KB_ID")
|
assert Span(doc, 1, 3) >= Span(doc, 0, 4, "LABEL", kb_id="KB_ID")
|
||||||
|
|
||||||
|
# Different id
|
||||||
|
assert Span(doc, 1, 3, span_id="AAA") < Span(doc, 1, 3, span_id="BBB")
|
||||||
# fmt: on
|
# fmt: on
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -1,72 +0,0 @@
|
||||||
import pytest
|
|
||||||
from spacy.tokens import Doc, Span
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture()
|
|
||||||
def doc(en_vocab):
|
|
||||||
words = ["c", "d", "e"]
|
|
||||||
pos = ["VERB", "NOUN", "NOUN"]
|
|
||||||
tags = ["VBP", "NN", "NN"]
|
|
||||||
heads = [0, 0, 0]
|
|
||||||
deps = ["ROOT", "dobj", "dobj"]
|
|
||||||
ents = ["O", "B-ORG", "O"]
|
|
||||||
morphs = ["Feat1=A", "Feat1=B", "Feat1=A|Feat2=D"]
|
|
||||||
return Doc(
|
|
||||||
en_vocab,
|
|
||||||
words=words,
|
|
||||||
pos=pos,
|
|
||||||
tags=tags,
|
|
||||||
heads=heads,
|
|
||||||
deps=deps,
|
|
||||||
ents=ents,
|
|
||||||
morphs=morphs,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def test_doc_to_json(doc):
|
|
||||||
json_doc = doc.to_json()
|
|
||||||
assert json_doc["text"] == "c d e "
|
|
||||||
assert len(json_doc["tokens"]) == 3
|
|
||||||
assert json_doc["tokens"][0]["pos"] == "VERB"
|
|
||||||
assert json_doc["tokens"][0]["tag"] == "VBP"
|
|
||||||
assert json_doc["tokens"][0]["dep"] == "ROOT"
|
|
||||||
assert len(json_doc["ents"]) == 1
|
|
||||||
assert json_doc["ents"][0]["start"] == 2 # character offset!
|
|
||||||
assert json_doc["ents"][0]["end"] == 3 # character offset!
|
|
||||||
assert json_doc["ents"][0]["label"] == "ORG"
|
|
||||||
|
|
||||||
|
|
||||||
def test_doc_to_json_underscore(doc):
|
|
||||||
Doc.set_extension("json_test1", default=False)
|
|
||||||
Doc.set_extension("json_test2", default=False)
|
|
||||||
doc._.json_test1 = "hello world"
|
|
||||||
doc._.json_test2 = [1, 2, 3]
|
|
||||||
json_doc = doc.to_json(underscore=["json_test1", "json_test2"])
|
|
||||||
assert "_" in json_doc
|
|
||||||
assert json_doc["_"]["json_test1"] == "hello world"
|
|
||||||
assert json_doc["_"]["json_test2"] == [1, 2, 3]
|
|
||||||
|
|
||||||
|
|
||||||
def test_doc_to_json_underscore_error_attr(doc):
|
|
||||||
"""Test that Doc.to_json() raises an error if a custom attribute doesn't
|
|
||||||
exist in the ._ space."""
|
|
||||||
with pytest.raises(ValueError):
|
|
||||||
doc.to_json(underscore=["json_test3"])
|
|
||||||
|
|
||||||
|
|
||||||
def test_doc_to_json_underscore_error_serialize(doc):
|
|
||||||
"""Test that Doc.to_json() raises an error if a custom attribute value
|
|
||||||
isn't JSON-serializable."""
|
|
||||||
Doc.set_extension("json_test4", method=lambda doc: doc.text)
|
|
||||||
with pytest.raises(ValueError):
|
|
||||||
doc.to_json(underscore=["json_test4"])
|
|
||||||
|
|
||||||
|
|
||||||
def test_doc_to_json_span(doc):
|
|
||||||
"""Test that Doc.to_json() includes spans"""
|
|
||||||
doc.spans["test"] = [Span(doc, 0, 2, "test"), Span(doc, 0, 1, "test")]
|
|
||||||
json_doc = doc.to_json()
|
|
||||||
assert "spans" in json_doc
|
|
||||||
assert len(json_doc["spans"]) == 1
|
|
||||||
assert len(json_doc["spans"]["test"]) == 2
|
|
||||||
assert json_doc["spans"]["test"][0]["start"] == 0
|
|
8
spacy/tests/lang/bg/test_tokenizer.py
Normal file
8
spacy/tests/lang/bg/test_tokenizer.py
Normal file
|
@ -0,0 +1,8 @@
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
|
||||||
|
def test_bg_tokenizer_handles_final_diacritics(bg_tokenizer):
|
||||||
|
text = "Ня̀маше яйца̀. Ня̀маше яйца̀."
|
||||||
|
tokens = bg_tokenizer(text)
|
||||||
|
assert tokens[1].text == "яйца̀"
|
||||||
|
assert tokens[2].text == "."
|
|
@ -167,3 +167,12 @@ def test_issue3521(en_tokenizer, word):
|
||||||
tok = en_tokenizer(word)[1]
|
tok = en_tokenizer(word)[1]
|
||||||
# 'not' and 'would' should be stopwords, also in their abbreviated forms
|
# 'not' and 'would' should be stopwords, also in their abbreviated forms
|
||||||
assert tok.is_stop
|
assert tok.is_stop
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.issue(10699)
|
||||||
|
@pytest.mark.parametrize("text", ["theses", "thisre"])
|
||||||
|
def test_issue10699(en_tokenizer, text):
|
||||||
|
"""Test that 'theses' and 'thisre' are excluded from the contractions
|
||||||
|
generated by the English tokenizer exceptions."""
|
||||||
|
tokens = en_tokenizer(text)
|
||||||
|
assert len(tokens) == 1
|
||||||
|
|
|
@ -1,3 +1,4 @@
|
||||||
|
from string import punctuation
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
|
|
||||||
|
@ -122,3 +123,36 @@ def test_ru_tokenizer_splits_bracket_period(ru_tokenizer):
|
||||||
text = "(Раз, два, три, проверка)."
|
text = "(Раз, два, три, проверка)."
|
||||||
tokens = ru_tokenizer(text)
|
tokens = ru_tokenizer(text)
|
||||||
assert tokens[len(tokens) - 1].text == "."
|
assert tokens[len(tokens) - 1].text == "."
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"text",
|
||||||
|
[
|
||||||
|
"рекоменду́я подда́ть жару́. Самого́ Баргамота",
|
||||||
|
"РЕКОМЕНДУ́Я ПОДДА́ТЬ ЖАРУ́. САМОГО́ БАРГАМОТА",
|
||||||
|
"рекоменду̍я подда̍ть жару̍.Самого̍ Баргамота",
|
||||||
|
"рекоменду̍я подда̍ть жару̍.'Самого̍ Баргамота",
|
||||||
|
"рекоменду̍я подда̍ть жару̍,самого̍ Баргамота",
|
||||||
|
"рекоменду̍я подда̍ть жару̍:самого̍ Баргамота",
|
||||||
|
"рекоменду̍я подда̍ть жару̍. самого̍ Баргамота",
|
||||||
|
"рекоменду̍я подда̍ть жару̍, самого̍ Баргамота",
|
||||||
|
"рекоменду̍я подда̍ть жару̍: самого̍ Баргамота",
|
||||||
|
"рекоменду̍я подда̍ть жару̍-самого̍ Баргамота",
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_ru_tokenizer_handles_final_diacritics(ru_tokenizer, text):
|
||||||
|
tokens = ru_tokenizer(text)
|
||||||
|
assert tokens[2].text in ("жару́", "ЖАРУ́", "жару̍")
|
||||||
|
assert tokens[3].text in punctuation
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"text",
|
||||||
|
[
|
||||||
|
"РЕКОМЕНДУ́Я ПОДДА́ТЬ ЖАРУ́.САМОГО́ БАРГАМОТА",
|
||||||
|
"рекоменду̍я подда̍ть жару́.самого́ Баргамота",
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_ru_tokenizer_handles_final_diacritic_and_period(ru_tokenizer, text):
|
||||||
|
tokens = ru_tokenizer(text)
|
||||||
|
assert tokens[2].text.lower() == "жару́.самого́"
|
||||||
|
|
|
@ -140,3 +140,10 @@ def test_uk_tokenizer_splits_bracket_period(uk_tokenizer):
|
||||||
text = "(Раз, два, три, проверка)."
|
text = "(Раз, два, три, проверка)."
|
||||||
tokens = uk_tokenizer(text)
|
tokens = uk_tokenizer(text)
|
||||||
assert tokens[len(tokens) - 1].text == "."
|
assert tokens[len(tokens) - 1].text == "."
|
||||||
|
|
||||||
|
|
||||||
|
def test_uk_tokenizer_handles_final_diacritics(uk_tokenizer):
|
||||||
|
text = "Хлібі́в не було́. Хлібі́в не було́."
|
||||||
|
tokens = uk_tokenizer(text)
|
||||||
|
assert tokens[2].text == "було́"
|
||||||
|
assert tokens[3].text == "."
|
||||||
|
|
|
@ -316,6 +316,20 @@ def test_dependency_matcher_precedence_ops(en_vocab, op, num_matches):
|
||||||
("the", "brown", "$--", 0),
|
("the", "brown", "$--", 0),
|
||||||
("brown", "the", "$--", 1),
|
("brown", "the", "$--", 1),
|
||||||
("brown", "brown", "$--", 0),
|
("brown", "brown", "$--", 0),
|
||||||
|
("quick", "fox", "<++", 1),
|
||||||
|
("quick", "over", "<++", 0),
|
||||||
|
("over", "jumped", "<++", 0),
|
||||||
|
("the", "fox", "<++", 2),
|
||||||
|
("brown", "fox", "<--", 0),
|
||||||
|
("fox", "jumped", "<--", 0),
|
||||||
|
("fox", "over", "<--", 1),
|
||||||
|
("jumped", "over", ">++", 1),
|
||||||
|
("fox", "lazy", ">++", 0),
|
||||||
|
("over", "the", ">++", 0),
|
||||||
|
("brown", "fox", ">--", 0),
|
||||||
|
("fox", "brown", ">--", 1),
|
||||||
|
("jumped", "fox", ">--", 1),
|
||||||
|
("fox", "the", ">--", 2),
|
||||||
],
|
],
|
||||||
)
|
)
|
||||||
def test_dependency_matcher_ops(en_vocab, doc, left, right, op, num_matches):
|
def test_dependency_matcher_ops(en_vocab, doc, left, right, op, num_matches):
|
||||||
|
|
|
@ -476,6 +476,17 @@ def test_matcher_extension_set_membership(en_vocab):
|
||||||
assert len(matches) == 0
|
assert len(matches) == 0
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.xfail(reason="IN predicate must handle sequence values in extensions")
|
||||||
|
def test_matcher_extension_in_set_predicate(en_vocab):
|
||||||
|
matcher = Matcher(en_vocab)
|
||||||
|
Token.set_extension("ext", default=[])
|
||||||
|
pattern = [{"_": {"ext": {"IN": ["A", "C"]}}}]
|
||||||
|
matcher.add("M", [pattern])
|
||||||
|
doc = Doc(en_vocab, words=["a", "b", "c"])
|
||||||
|
doc[0]._.ext = ["A", "B"]
|
||||||
|
assert len(matcher(doc)) == 1
|
||||||
|
|
||||||
|
|
||||||
def test_matcher_basic_check(en_vocab):
|
def test_matcher_basic_check(en_vocab):
|
||||||
matcher = Matcher(en_vocab)
|
matcher = Matcher(en_vocab)
|
||||||
# Potential mistake: pass in pattern instead of list of patterns
|
# Potential mistake: pass in pattern instead of list of patterns
|
||||||
|
@ -669,3 +680,38 @@ def test_matcher_ent_iob_key(en_vocab):
|
||||||
assert matches[0] == "Maria"
|
assert matches[0] == "Maria"
|
||||||
assert matches[1] == "Maria Esperanza"
|
assert matches[1] == "Maria Esperanza"
|
||||||
assert matches[2] == "Esperanza"
|
assert matches[2] == "Esperanza"
|
||||||
|
|
||||||
|
|
||||||
|
def test_matcher_min_max_operator(en_vocab):
|
||||||
|
# Exactly n matches {n}
|
||||||
|
doc = Doc(
|
||||||
|
en_vocab,
|
||||||
|
words=["foo", "bar", "foo", "foo", "bar", "foo", "foo", "foo", "bar", "bar"],
|
||||||
|
)
|
||||||
|
matcher = Matcher(en_vocab)
|
||||||
|
pattern = [{"ORTH": "foo", "OP": "{3}"}]
|
||||||
|
matcher.add("TEST", [pattern])
|
||||||
|
|
||||||
|
matches1 = [doc[start:end].text for _, start, end in matcher(doc)]
|
||||||
|
assert len(matches1) == 1
|
||||||
|
|
||||||
|
# At least n matches {n,}
|
||||||
|
matcher = Matcher(en_vocab)
|
||||||
|
pattern = [{"ORTH": "foo", "OP": "{2,}"}]
|
||||||
|
matcher.add("TEST", [pattern])
|
||||||
|
matches2 = [doc[start:end].text for _, start, end in matcher(doc)]
|
||||||
|
assert len(matches2) == 4
|
||||||
|
|
||||||
|
# At most m matches {,m}
|
||||||
|
matcher = Matcher(en_vocab)
|
||||||
|
pattern = [{"ORTH": "foo", "OP": "{,2}"}]
|
||||||
|
matcher.add("TEST", [pattern])
|
||||||
|
matches3 = [doc[start:end].text for _, start, end in matcher(doc)]
|
||||||
|
assert len(matches3) == 9
|
||||||
|
|
||||||
|
# At least n matches and most m matches {n,m}
|
||||||
|
matcher = Matcher(en_vocab)
|
||||||
|
pattern = [{"ORTH": "foo", "OP": "{2,3}"}]
|
||||||
|
matcher.add("TEST", [pattern])
|
||||||
|
matches4 = [doc[start:end].text for _, start, end in matcher(doc)]
|
||||||
|
assert len(matches4) == 4
|
||||||
|
|
|
@ -699,6 +699,10 @@ def test_matcher_with_alignments_greedy_longest(en_vocab):
|
||||||
("aaaa", "a a a a a?", [0, 1, 2, 3]),
|
("aaaa", "a a a a a?", [0, 1, 2, 3]),
|
||||||
("aaab", "a+ a b", [0, 0, 1, 2]),
|
("aaab", "a+ a b", [0, 0, 1, 2]),
|
||||||
("aaab", "a+ a+ b", [0, 0, 1, 2]),
|
("aaab", "a+ a+ b", [0, 0, 1, 2]),
|
||||||
|
("aaab", "a{2,} b", [0, 0, 0, 1]),
|
||||||
|
("aaab", "a{,3} b", [0, 0, 0, 1]),
|
||||||
|
("aaab", "a{2} b", [0, 0, 1]),
|
||||||
|
("aaab", "a{2,3} b", [0, 0, 0, 1]),
|
||||||
]
|
]
|
||||||
for string, pattern_str, result in cases:
|
for string, pattern_str, result in cases:
|
||||||
matcher = Matcher(en_vocab)
|
matcher = Matcher(en_vocab)
|
||||||
|
@ -711,6 +715,8 @@ def test_matcher_with_alignments_greedy_longest(en_vocab):
|
||||||
pattern.append({"ORTH": part[0], "OP": "*"})
|
pattern.append({"ORTH": part[0], "OP": "*"})
|
||||||
elif part.endswith("?"):
|
elif part.endswith("?"):
|
||||||
pattern.append({"ORTH": part[0], "OP": "?"})
|
pattern.append({"ORTH": part[0], "OP": "?"})
|
||||||
|
elif part.endswith("}"):
|
||||||
|
pattern.append({"ORTH": part[0], "OP": part[1:]})
|
||||||
else:
|
else:
|
||||||
pattern.append({"ORTH": part})
|
pattern.append({"ORTH": part})
|
||||||
matcher.add("PATTERN", [pattern], greedy="LONGEST")
|
matcher.add("PATTERN", [pattern], greedy="LONGEST")
|
||||||
|
@ -722,7 +728,7 @@ def test_matcher_with_alignments_greedy_longest(en_vocab):
|
||||||
assert expected == result, (string, pattern_str, s, e, n_matches)
|
assert expected == result, (string, pattern_str, s, e, n_matches)
|
||||||
|
|
||||||
|
|
||||||
def test_matcher_with_alignments_nongreedy(en_vocab):
|
def test_matcher_with_alignments_non_greedy(en_vocab):
|
||||||
cases = [
|
cases = [
|
||||||
(0, "aaab", "a* b", [[0, 1], [0, 0, 1], [0, 0, 0, 1], [1]]),
|
(0, "aaab", "a* b", [[0, 1], [0, 0, 1], [0, 0, 0, 1], [1]]),
|
||||||
(1, "baab", "b a* b", [[0, 1, 1, 2]]),
|
(1, "baab", "b a* b", [[0, 1, 1, 2]]),
|
||||||
|
@ -752,6 +758,10 @@ def test_matcher_with_alignments_nongreedy(en_vocab):
|
||||||
(15, "aaaa", "a a a a a?", [[0, 1, 2, 3]]),
|
(15, "aaaa", "a a a a a?", [[0, 1, 2, 3]]),
|
||||||
(16, "aaab", "a+ a b", [[0, 1, 2], [0, 0, 1, 2]]),
|
(16, "aaab", "a+ a b", [[0, 1, 2], [0, 0, 1, 2]]),
|
||||||
(17, "aaab", "a+ a+ b", [[0, 1, 2], [0, 0, 1, 2]]),
|
(17, "aaab", "a+ a+ b", [[0, 1, 2], [0, 0, 1, 2]]),
|
||||||
|
(18, "aaab", "a{2,} b", [[0, 0, 1], [0, 0, 0, 1]]),
|
||||||
|
(19, "aaab", "a{3} b", [[0, 0, 0, 1]]),
|
||||||
|
(20, "aaab", "a{2} b", [[0, 0, 1]]),
|
||||||
|
(21, "aaab", "a{2,3} b", [[0, 0, 1], [0, 0, 0, 1]]),
|
||||||
]
|
]
|
||||||
for case_id, string, pattern_str, results in cases:
|
for case_id, string, pattern_str, results in cases:
|
||||||
matcher = Matcher(en_vocab)
|
matcher = Matcher(en_vocab)
|
||||||
|
@ -764,6 +774,8 @@ def test_matcher_with_alignments_nongreedy(en_vocab):
|
||||||
pattern.append({"ORTH": part[0], "OP": "*"})
|
pattern.append({"ORTH": part[0], "OP": "*"})
|
||||||
elif part.endswith("?"):
|
elif part.endswith("?"):
|
||||||
pattern.append({"ORTH": part[0], "OP": "?"})
|
pattern.append({"ORTH": part[0], "OP": "?"})
|
||||||
|
elif part.endswith("}"):
|
||||||
|
pattern.append({"ORTH": part[0], "OP": part[1:]})
|
||||||
else:
|
else:
|
||||||
pattern.append({"ORTH": part})
|
pattern.append({"ORTH": part})
|
||||||
|
|
||||||
|
|
|
@ -14,6 +14,14 @@ TEST_PATTERNS = [
|
||||||
('[{"TEXT": "foo"}, {"LOWER": "bar"}]', 1, 1),
|
('[{"TEXT": "foo"}, {"LOWER": "bar"}]', 1, 1),
|
||||||
([{"ENT_IOB": "foo"}], 1, 1),
|
([{"ENT_IOB": "foo"}], 1, 1),
|
||||||
([1, 2, 3], 3, 1),
|
([1, 2, 3], 3, 1),
|
||||||
|
([{"TEXT": "foo", "OP": "{,}"}], 1, 1),
|
||||||
|
([{"TEXT": "foo", "OP": "{,4}4"}], 1, 1),
|
||||||
|
([{"TEXT": "foo", "OP": "{a,3}"}], 1, 1),
|
||||||
|
([{"TEXT": "foo", "OP": "{a}"}], 1, 1),
|
||||||
|
([{"TEXT": "foo", "OP": "{,a}"}], 1, 1),
|
||||||
|
([{"TEXT": "foo", "OP": "{1,2,3}"}], 1, 1),
|
||||||
|
([{"TEXT": "foo", "OP": "{1, 3}"}], 1, 1),
|
||||||
|
([{"TEXT": "foo", "OP": "{-2}"}], 1, 1),
|
||||||
# Bad patterns flagged outside of Matcher
|
# Bad patterns flagged outside of Matcher
|
||||||
([{"_": {"foo": "bar", "baz": {"IN": "foo"}}}], 2, 0), # prev: (1, 0)
|
([{"_": {"foo": "bar", "baz": {"IN": "foo"}}}], 2, 0), # prev: (1, 0)
|
||||||
# Bad patterns not flagged with minimal checks
|
# Bad patterns not flagged with minimal checks
|
||||||
|
@ -38,6 +46,7 @@ TEST_PATTERNS = [
|
||||||
([{"SENT_START": True}], 0, 0),
|
([{"SENT_START": True}], 0, 0),
|
||||||
([{"ENT_ID": "STRING"}], 0, 0),
|
([{"ENT_ID": "STRING"}], 0, 0),
|
||||||
([{"ENT_KB_ID": "STRING"}], 0, 0),
|
([{"ENT_KB_ID": "STRING"}], 0, 0),
|
||||||
|
([{"TEXT": "ha", "OP": "{3}"}], 0, 0),
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -122,6 +122,36 @@ def test_issue6839(en_vocab):
|
||||||
assert matches
|
assert matches
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.issue(10643)
|
||||||
|
def test_issue10643(en_vocab):
|
||||||
|
"""Ensure overlapping terms can be removed from PhraseMatcher"""
|
||||||
|
|
||||||
|
# fmt: off
|
||||||
|
words = ["Only", "save", "out", "the", "binary", "data", "for", "the", "individual", "components", "."]
|
||||||
|
# fmt: on
|
||||||
|
doc = Doc(en_vocab, words=words)
|
||||||
|
terms = {
|
||||||
|
"0": Doc(en_vocab, words=["binary"]),
|
||||||
|
"1": Doc(en_vocab, words=["binary", "data"]),
|
||||||
|
}
|
||||||
|
matcher = PhraseMatcher(en_vocab)
|
||||||
|
for match_id, term in terms.items():
|
||||||
|
matcher.add(match_id, [term])
|
||||||
|
|
||||||
|
matches = matcher(doc)
|
||||||
|
assert matches == [(en_vocab.strings["0"], 4, 5), (en_vocab.strings["1"], 4, 6)]
|
||||||
|
|
||||||
|
matcher.remove("0")
|
||||||
|
assert len(matcher) == 1
|
||||||
|
new_matches = matcher(doc)
|
||||||
|
assert new_matches == [(en_vocab.strings["1"], 4, 6)]
|
||||||
|
|
||||||
|
matcher.remove("1")
|
||||||
|
assert len(matcher) == 0
|
||||||
|
no_matches = matcher(doc)
|
||||||
|
assert not no_matches
|
||||||
|
|
||||||
|
|
||||||
def test_matcher_phrase_matcher(en_vocab):
|
def test_matcher_phrase_matcher(en_vocab):
|
||||||
doc = Doc(en_vocab, words=["I", "like", "Google", "Now", "best"])
|
doc = Doc(en_vocab, words=["I", "like", "Google", "Now", "best"])
|
||||||
# intermediate phrase
|
# intermediate phrase
|
||||||
|
|
|
@ -10,7 +10,7 @@ from spacy.lang.it import Italian
|
||||||
from spacy.language import Language
|
from spacy.language import Language
|
||||||
from spacy.lookups import Lookups
|
from spacy.lookups import Lookups
|
||||||
from spacy.pipeline._parser_internals.ner import BiluoPushDown
|
from spacy.pipeline._parser_internals.ner import BiluoPushDown
|
||||||
from spacy.training import Example, iob_to_biluo
|
from spacy.training import Example, iob_to_biluo, split_bilu_label
|
||||||
from spacy.tokens import Doc, Span
|
from spacy.tokens import Doc, Span
|
||||||
from spacy.vocab import Vocab
|
from spacy.vocab import Vocab
|
||||||
import logging
|
import logging
|
||||||
|
@ -110,6 +110,9 @@ def test_issue2385():
|
||||||
# maintain support for iob2 format
|
# maintain support for iob2 format
|
||||||
tags3 = ("B-PERSON", "I-PERSON", "B-PERSON")
|
tags3 = ("B-PERSON", "I-PERSON", "B-PERSON")
|
||||||
assert iob_to_biluo(tags3) == ["B-PERSON", "L-PERSON", "U-PERSON"]
|
assert iob_to_biluo(tags3) == ["B-PERSON", "L-PERSON", "U-PERSON"]
|
||||||
|
# ensure it works with hyphens in the name
|
||||||
|
tags4 = ("B-MULTI-PERSON", "I-MULTI-PERSON", "B-MULTI-PERSON")
|
||||||
|
assert iob_to_biluo(tags4) == ["B-MULTI-PERSON", "L-MULTI-PERSON", "U-MULTI-PERSON"]
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.issue(2800)
|
@pytest.mark.issue(2800)
|
||||||
|
@ -154,6 +157,24 @@ def test_issue3209():
|
||||||
assert ner2.move_names == move_names
|
assert ner2.move_names == move_names
|
||||||
|
|
||||||
|
|
||||||
|
def test_labels_from_BILUO():
|
||||||
|
"""Test that labels are inferred correctly when there's a - in label."""
|
||||||
|
nlp = English()
|
||||||
|
ner = nlp.add_pipe("ner")
|
||||||
|
ner.add_label("LARGE-ANIMAL")
|
||||||
|
nlp.initialize()
|
||||||
|
move_names = [
|
||||||
|
"O",
|
||||||
|
"B-LARGE-ANIMAL",
|
||||||
|
"I-LARGE-ANIMAL",
|
||||||
|
"L-LARGE-ANIMAL",
|
||||||
|
"U-LARGE-ANIMAL",
|
||||||
|
]
|
||||||
|
labels = {"LARGE-ANIMAL"}
|
||||||
|
assert ner.move_names == move_names
|
||||||
|
assert set(ner.labels) == labels
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.issue(4267)
|
@pytest.mark.issue(4267)
|
||||||
def test_issue4267():
|
def test_issue4267():
|
||||||
"""Test that running an entity_ruler after ner gives consistent results"""
|
"""Test that running an entity_ruler after ner gives consistent results"""
|
||||||
|
@ -298,7 +319,7 @@ def test_oracle_moves_missing_B(en_vocab):
|
||||||
elif tag == "O":
|
elif tag == "O":
|
||||||
moves.add_action(move_types.index("O"), "")
|
moves.add_action(move_types.index("O"), "")
|
||||||
else:
|
else:
|
||||||
action, label = tag.split("-")
|
action, label = split_bilu_label(tag)
|
||||||
moves.add_action(move_types.index("B"), label)
|
moves.add_action(move_types.index("B"), label)
|
||||||
moves.add_action(move_types.index("I"), label)
|
moves.add_action(move_types.index("I"), label)
|
||||||
moves.add_action(move_types.index("L"), label)
|
moves.add_action(move_types.index("L"), label)
|
||||||
|
@ -324,7 +345,7 @@ def test_oracle_moves_whitespace(en_vocab):
|
||||||
elif tag == "O":
|
elif tag == "O":
|
||||||
moves.add_action(move_types.index("O"), "")
|
moves.add_action(move_types.index("O"), "")
|
||||||
else:
|
else:
|
||||||
action, label = tag.split("-")
|
action, label = split_bilu_label(tag)
|
||||||
moves.add_action(move_types.index(action), label)
|
moves.add_action(move_types.index(action), label)
|
||||||
moves.get_oracle_sequence(example)
|
moves.get_oracle_sequence(example)
|
||||||
|
|
||||||
|
|
|
@ -49,7 +49,9 @@ def test_parser_contains_cycle(tree, cyclic_tree, partial_tree, multirooted_tree
|
||||||
assert contains_cycle(multirooted_tree) is None
|
assert contains_cycle(multirooted_tree) is None
|
||||||
|
|
||||||
|
|
||||||
def test_parser_is_nonproj_arc(nonproj_tree, partial_tree, multirooted_tree):
|
def test_parser_is_nonproj_arc(
|
||||||
|
cyclic_tree, nonproj_tree, partial_tree, multirooted_tree
|
||||||
|
):
|
||||||
assert is_nonproj_arc(0, nonproj_tree) is False
|
assert is_nonproj_arc(0, nonproj_tree) is False
|
||||||
assert is_nonproj_arc(1, nonproj_tree) is False
|
assert is_nonproj_arc(1, nonproj_tree) is False
|
||||||
assert is_nonproj_arc(2, nonproj_tree) is False
|
assert is_nonproj_arc(2, nonproj_tree) is False
|
||||||
|
@ -62,15 +64,23 @@ def test_parser_is_nonproj_arc(nonproj_tree, partial_tree, multirooted_tree):
|
||||||
assert is_nonproj_arc(7, partial_tree) is False
|
assert is_nonproj_arc(7, partial_tree) is False
|
||||||
assert is_nonproj_arc(17, multirooted_tree) is False
|
assert is_nonproj_arc(17, multirooted_tree) is False
|
||||||
assert is_nonproj_arc(16, multirooted_tree) is True
|
assert is_nonproj_arc(16, multirooted_tree) is True
|
||||||
|
with pytest.raises(
|
||||||
|
ValueError, match=r"Found cycle in dependency graph: \[1, 2, 2, 4, 5, 3, 2\]"
|
||||||
|
):
|
||||||
|
is_nonproj_arc(6, cyclic_tree)
|
||||||
|
|
||||||
|
|
||||||
def test_parser_is_nonproj_tree(
|
def test_parser_is_nonproj_tree(
|
||||||
proj_tree, nonproj_tree, partial_tree, multirooted_tree
|
proj_tree, cyclic_tree, nonproj_tree, partial_tree, multirooted_tree
|
||||||
):
|
):
|
||||||
assert is_nonproj_tree(proj_tree) is False
|
assert is_nonproj_tree(proj_tree) is False
|
||||||
assert is_nonproj_tree(nonproj_tree) is True
|
assert is_nonproj_tree(nonproj_tree) is True
|
||||||
assert is_nonproj_tree(partial_tree) is False
|
assert is_nonproj_tree(partial_tree) is False
|
||||||
assert is_nonproj_tree(multirooted_tree) is True
|
assert is_nonproj_tree(multirooted_tree) is True
|
||||||
|
with pytest.raises(
|
||||||
|
ValueError, match=r"Found cycle in dependency graph: \[1, 2, 2, 4, 5, 3, 2\]"
|
||||||
|
):
|
||||||
|
is_nonproj_tree(cyclic_tree)
|
||||||
|
|
||||||
|
|
||||||
def test_parser_pseudoprojectivity(en_vocab):
|
def test_parser_pseudoprojectivity(en_vocab):
|
||||||
|
@ -84,8 +94,10 @@ def test_parser_pseudoprojectivity(en_vocab):
|
||||||
tree = [1, 2, 2]
|
tree = [1, 2, 2]
|
||||||
nonproj_tree = [1, 2, 2, 4, 5, 2, 7, 4, 2]
|
nonproj_tree = [1, 2, 2, 4, 5, 2, 7, 4, 2]
|
||||||
nonproj_tree2 = [9, 1, 3, 1, 5, 6, 9, 8, 6, 1, 6, 12, 13, 10, 1]
|
nonproj_tree2 = [9, 1, 3, 1, 5, 6, 9, 8, 6, 1, 6, 12, 13, 10, 1]
|
||||||
|
cyclic_tree = [1, 2, 2, 4, 5, 3, 2]
|
||||||
labels = ["det", "nsubj", "root", "det", "dobj", "aux", "nsubj", "acl", "punct"]
|
labels = ["det", "nsubj", "root", "det", "dobj", "aux", "nsubj", "acl", "punct"]
|
||||||
labels2 = ["advmod", "root", "det", "nsubj", "advmod", "det", "dobj", "det", "nmod", "aux", "nmod", "advmod", "det", "amod", "punct"]
|
labels2 = ["advmod", "root", "det", "nsubj", "advmod", "det", "dobj", "det", "nmod", "aux", "nmod", "advmod", "det", "amod", "punct"]
|
||||||
|
cyclic_labels = ["det", "nsubj", "root", "det", "dobj", "aux", "punct"]
|
||||||
# fmt: on
|
# fmt: on
|
||||||
assert nonproj.decompose("X||Y") == ("X", "Y")
|
assert nonproj.decompose("X||Y") == ("X", "Y")
|
||||||
assert nonproj.decompose("X") == ("X", "")
|
assert nonproj.decompose("X") == ("X", "")
|
||||||
|
@ -97,6 +109,8 @@ def test_parser_pseudoprojectivity(en_vocab):
|
||||||
assert nonproj.get_smallest_nonproj_arc_slow(nonproj_tree2) == 10
|
assert nonproj.get_smallest_nonproj_arc_slow(nonproj_tree2) == 10
|
||||||
# fmt: off
|
# fmt: off
|
||||||
proj_heads, deco_labels = nonproj.projectivize(nonproj_tree, labels)
|
proj_heads, deco_labels = nonproj.projectivize(nonproj_tree, labels)
|
||||||
|
with pytest.raises(ValueError, match=r'Found cycle in dependency graph: \[1, 2, 2, 4, 5, 3, 2\]'):
|
||||||
|
nonproj.projectivize(cyclic_tree, cyclic_labels)
|
||||||
assert proj_heads == [1, 2, 2, 4, 5, 2, 7, 5, 2]
|
assert proj_heads == [1, 2, 2, 4, 5, 2, 7, 5, 2]
|
||||||
assert deco_labels == ["det", "nsubj", "root", "det", "dobj", "aux",
|
assert deco_labels == ["det", "nsubj", "root", "det", "dobj", "aux",
|
||||||
"nsubj", "acl||dobj", "punct"]
|
"nsubj", "acl||dobj", "punct"]
|
||||||
|
|
|
@ -12,6 +12,7 @@ from spacy.vocab import Vocab
|
||||||
from ...pipeline import DependencyParser
|
from ...pipeline import DependencyParser
|
||||||
from ...pipeline.dep_parser import DEFAULT_PARSER_MODEL
|
from ...pipeline.dep_parser import DEFAULT_PARSER_MODEL
|
||||||
from ..util import apply_transition_sequence, make_tempdir
|
from ..util import apply_transition_sequence, make_tempdir
|
||||||
|
from ...pipeline.tok2vec import DEFAULT_TOK2VEC_MODEL
|
||||||
|
|
||||||
TRAIN_DATA = [
|
TRAIN_DATA = [
|
||||||
(
|
(
|
||||||
|
@ -395,6 +396,34 @@ def test_overfitting_IO(pipe_name):
|
||||||
assert_equal(batch_deps_1, no_batch_deps)
|
assert_equal(batch_deps_1, no_batch_deps)
|
||||||
|
|
||||||
|
|
||||||
|
# fmt: off
|
||||||
|
@pytest.mark.slow
|
||||||
|
@pytest.mark.parametrize("pipe_name", ["parser", "beam_parser"])
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"parser_config",
|
||||||
|
[
|
||||||
|
# TransitionBasedParser V1
|
||||||
|
({"@architectures": "spacy.TransitionBasedParser.v1", "tok2vec": DEFAULT_TOK2VEC_MODEL, "state_type": "parser", "extra_state_tokens": False, "hidden_width": 64, "maxout_pieces": 2, "use_upper": True}),
|
||||||
|
# TransitionBasedParser V2
|
||||||
|
({"@architectures": "spacy.TransitionBasedParser.v2", "tok2vec": DEFAULT_TOK2VEC_MODEL, "state_type": "parser", "extra_state_tokens": False, "hidden_width": 64, "maxout_pieces": 2, "use_upper": True}),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
# fmt: on
|
||||||
|
def test_parser_configs(pipe_name, parser_config):
|
||||||
|
pipe_config = {"model": parser_config}
|
||||||
|
nlp = English()
|
||||||
|
parser = nlp.add_pipe(pipe_name, config=pipe_config)
|
||||||
|
train_examples = []
|
||||||
|
for text, annotations in TRAIN_DATA:
|
||||||
|
train_examples.append(Example.from_dict(nlp.make_doc(text), annotations))
|
||||||
|
for dep in annotations.get("deps", []):
|
||||||
|
parser.add_label(dep)
|
||||||
|
optimizer = nlp.initialize()
|
||||||
|
for i in range(5):
|
||||||
|
losses = {}
|
||||||
|
nlp.update(train_examples, sgd=optimizer, losses=losses)
|
||||||
|
|
||||||
|
|
||||||
def test_beam_parser_scores():
|
def test_beam_parser_scores():
|
||||||
# Test that we can get confidence values out of the beam_parser pipe
|
# Test that we can get confidence values out of the beam_parser pipe
|
||||||
beam_width = 16
|
beam_width = 16
|
||||||
|
|
|
@ -1,4 +1,4 @@
|
||||||
from typing import Callable, Iterable
|
from typing import Callable, Iterable, Dict, Any
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
from numpy.testing import assert_equal
|
from numpy.testing import assert_equal
|
||||||
|
@ -14,7 +14,7 @@ from spacy.pipeline.legacy import EntityLinker_v1
|
||||||
from spacy.pipeline.tok2vec import DEFAULT_TOK2VEC_MODEL
|
from spacy.pipeline.tok2vec import DEFAULT_TOK2VEC_MODEL
|
||||||
from spacy.scorer import Scorer
|
from spacy.scorer import Scorer
|
||||||
from spacy.tests.util import make_tempdir
|
from spacy.tests.util import make_tempdir
|
||||||
from spacy.tokens import Span
|
from spacy.tokens import Span, Doc
|
||||||
from spacy.training import Example
|
from spacy.training import Example
|
||||||
from spacy.util import ensure_path
|
from spacy.util import ensure_path
|
||||||
from spacy.vocab import Vocab
|
from spacy.vocab import Vocab
|
||||||
|
@ -207,7 +207,7 @@ def test_no_entities():
|
||||||
nlp.add_pipe("sentencizer", first=True)
|
nlp.add_pipe("sentencizer", first=True)
|
||||||
|
|
||||||
# this will run the pipeline on the examples and shouldn't crash
|
# this will run the pipeline on the examples and shouldn't crash
|
||||||
results = nlp.evaluate(train_examples)
|
nlp.evaluate(train_examples)
|
||||||
|
|
||||||
|
|
||||||
def test_partial_links():
|
def test_partial_links():
|
||||||
|
@ -1063,7 +1063,7 @@ def test_no_gold_ents(patterns):
|
||||||
"entity_linker", config={"use_gold_ents": False}, last=True
|
"entity_linker", config={"use_gold_ents": False}, last=True
|
||||||
)
|
)
|
||||||
entity_linker.set_kb(create_kb)
|
entity_linker.set_kb(create_kb)
|
||||||
assert entity_linker.use_gold_ents == False
|
assert entity_linker.use_gold_ents is False
|
||||||
|
|
||||||
optimizer = nlp.initialize(get_examples=lambda: train_examples)
|
optimizer = nlp.initialize(get_examples=lambda: train_examples)
|
||||||
for i in range(2):
|
for i in range(2):
|
||||||
|
@ -1074,4 +1074,101 @@ def test_no_gold_ents(patterns):
|
||||||
nlp.add_pipe("sentencizer", first=True)
|
nlp.add_pipe("sentencizer", first=True)
|
||||||
|
|
||||||
# this will run the pipeline on the examples and shouldn't crash
|
# this will run the pipeline on the examples and shouldn't crash
|
||||||
results = nlp.evaluate(train_examples)
|
nlp.evaluate(train_examples)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.issue(9575)
|
||||||
|
def test_tokenization_mismatch():
|
||||||
|
nlp = English()
|
||||||
|
# include a matching entity so that update isn't skipped
|
||||||
|
doc1 = Doc(
|
||||||
|
nlp.vocab,
|
||||||
|
words=["Kirby", "123456"],
|
||||||
|
spaces=[True, False],
|
||||||
|
ents=["B-CHARACTER", "B-CARDINAL"],
|
||||||
|
)
|
||||||
|
doc2 = Doc(
|
||||||
|
nlp.vocab,
|
||||||
|
words=["Kirby", "123", "456"],
|
||||||
|
spaces=[True, False, False],
|
||||||
|
ents=["B-CHARACTER", "B-CARDINAL", "B-CARDINAL"],
|
||||||
|
)
|
||||||
|
|
||||||
|
eg = Example(doc1, doc2)
|
||||||
|
train_examples = [eg]
|
||||||
|
vector_length = 3
|
||||||
|
|
||||||
|
def create_kb(vocab):
|
||||||
|
# create placeholder KB
|
||||||
|
mykb = KnowledgeBase(vocab, entity_vector_length=vector_length)
|
||||||
|
mykb.add_entity(entity="Q613241", freq=12, entity_vector=[6, -4, 3])
|
||||||
|
mykb.add_alias("Kirby", ["Q613241"], [0.9])
|
||||||
|
return mykb
|
||||||
|
|
||||||
|
entity_linker = nlp.add_pipe("entity_linker", last=True)
|
||||||
|
entity_linker.set_kb(create_kb)
|
||||||
|
|
||||||
|
optimizer = nlp.initialize(get_examples=lambda: train_examples)
|
||||||
|
for i in range(2):
|
||||||
|
losses = {}
|
||||||
|
nlp.update(train_examples, sgd=optimizer, losses=losses)
|
||||||
|
|
||||||
|
nlp.add_pipe("sentencizer", first=True)
|
||||||
|
nlp.evaluate(train_examples)
|
||||||
|
|
||||||
|
|
||||||
|
# fmt: off
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"meet_threshold,config",
|
||||||
|
[
|
||||||
|
(False, {"@architectures": "spacy.EntityLinker.v2", "tok2vec": DEFAULT_TOK2VEC_MODEL}),
|
||||||
|
(True, {"@architectures": "spacy.EntityLinker.v2", "tok2vec": DEFAULT_TOK2VEC_MODEL}),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
# fmt: on
|
||||||
|
def test_threshold(meet_threshold: bool, config: Dict[str, Any]):
|
||||||
|
"""Tests abstention threshold.
|
||||||
|
meet_threshold (bool): Whether to configure NEL setup so that confidence threshold is met.
|
||||||
|
config (Dict[str, Any]): NEL architecture config.
|
||||||
|
"""
|
||||||
|
nlp = English()
|
||||||
|
nlp.add_pipe("sentencizer")
|
||||||
|
text = "Mahler's Symphony No. 8 was beautiful."
|
||||||
|
entities = [(0, 6, "PERSON")]
|
||||||
|
links = {(0, 6): {"Q7304": 1.0}}
|
||||||
|
sent_starts = [1, -1, 0, 0, 0, 0, 0, 0, 0]
|
||||||
|
entity_id = "Q7304"
|
||||||
|
doc = nlp(text)
|
||||||
|
train_examples = [
|
||||||
|
Example.from_dict(
|
||||||
|
doc, {"entities": entities, "links": links, "sent_starts": sent_starts}
|
||||||
|
)
|
||||||
|
]
|
||||||
|
|
||||||
|
def create_kb(vocab):
|
||||||
|
# create artificial KB
|
||||||
|
mykb = KnowledgeBase(vocab, entity_vector_length=3)
|
||||||
|
mykb.add_entity(entity=entity_id, freq=12, entity_vector=[6, -4, 3])
|
||||||
|
mykb.add_alias(
|
||||||
|
alias="Mahler",
|
||||||
|
entities=[entity_id],
|
||||||
|
probabilities=[1 if meet_threshold else 0.01],
|
||||||
|
)
|
||||||
|
return mykb
|
||||||
|
|
||||||
|
# Create the Entity Linker component and add it to the pipeline
|
||||||
|
entity_linker = nlp.add_pipe(
|
||||||
|
"entity_linker",
|
||||||
|
last=True,
|
||||||
|
config={"threshold": 0.99, "model": config},
|
||||||
|
)
|
||||||
|
entity_linker.set_kb(create_kb) # type: ignore
|
||||||
|
nlp.initialize(get_examples=lambda: train_examples)
|
||||||
|
|
||||||
|
# Add a custom rule-based component to mimick NER
|
||||||
|
ruler = nlp.add_pipe("entity_ruler", before="entity_linker")
|
||||||
|
ruler.add_patterns([{"label": "PERSON", "pattern": [{"LOWER": "mahler"}]}]) # type: ignore
|
||||||
|
doc = nlp(text)
|
||||||
|
|
||||||
|
assert len(doc.ents) == 1
|
||||||
|
assert doc.ents[0].kb_id_ == entity_id if meet_threshold else EntityLinker.NIL
|
||||||
|
|
|
@ -5,12 +5,15 @@ from spacy.tokens import Doc, Span
|
||||||
from spacy.language import Language
|
from spacy.language import Language
|
||||||
from spacy.lang.en import English
|
from spacy.lang.en import English
|
||||||
from spacy.pipeline import EntityRuler, EntityRecognizer, merge_entities
|
from spacy.pipeline import EntityRuler, EntityRecognizer, merge_entities
|
||||||
|
from spacy.pipeline import SpanRuler
|
||||||
from spacy.pipeline.ner import DEFAULT_NER_MODEL
|
from spacy.pipeline.ner import DEFAULT_NER_MODEL
|
||||||
from spacy.errors import MatchPatternError
|
from spacy.errors import MatchPatternError
|
||||||
from spacy.tests.util import make_tempdir
|
from spacy.tests.util import make_tempdir
|
||||||
|
|
||||||
from thinc.api import NumpyOps, get_current_ops
|
from thinc.api import NumpyOps, get_current_ops
|
||||||
|
|
||||||
|
ENTITY_RULERS = ["entity_ruler", "future_entity_ruler"]
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture
|
@pytest.fixture
|
||||||
def nlp():
|
def nlp():
|
||||||
|
@ -37,12 +40,14 @@ def add_ent_component(doc):
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.issue(3345)
|
@pytest.mark.issue(3345)
|
||||||
def test_issue3345():
|
@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS)
|
||||||
|
def test_issue3345(entity_ruler_factory):
|
||||||
"""Test case where preset entity crosses sentence boundary."""
|
"""Test case where preset entity crosses sentence boundary."""
|
||||||
nlp = English()
|
nlp = English()
|
||||||
doc = Doc(nlp.vocab, words=["I", "live", "in", "New", "York"])
|
doc = Doc(nlp.vocab, words=["I", "live", "in", "New", "York"])
|
||||||
doc[4].is_sent_start = True
|
doc[4].is_sent_start = True
|
||||||
ruler = EntityRuler(nlp, patterns=[{"label": "GPE", "pattern": "New York"}])
|
ruler = nlp.add_pipe(entity_ruler_factory, name="entity_ruler")
|
||||||
|
ruler.add_patterns([{"label": "GPE", "pattern": "New York"}])
|
||||||
cfg = {"model": DEFAULT_NER_MODEL}
|
cfg = {"model": DEFAULT_NER_MODEL}
|
||||||
model = registry.resolve(cfg, validate=True)["model"]
|
model = registry.resolve(cfg, validate=True)["model"]
|
||||||
ner = EntityRecognizer(doc.vocab, model)
|
ner = EntityRecognizer(doc.vocab, model)
|
||||||
|
@ -60,13 +65,18 @@ def test_issue3345():
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.issue(4849)
|
@pytest.mark.issue(4849)
|
||||||
def test_issue4849():
|
@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS)
|
||||||
|
def test_issue4849(entity_ruler_factory):
|
||||||
nlp = English()
|
nlp = English()
|
||||||
patterns = [
|
patterns = [
|
||||||
{"label": "PERSON", "pattern": "joe biden", "id": "joe-biden"},
|
{"label": "PERSON", "pattern": "joe biden", "id": "joe-biden"},
|
||||||
{"label": "PERSON", "pattern": "bernie sanders", "id": "bernie-sanders"},
|
{"label": "PERSON", "pattern": "bernie sanders", "id": "bernie-sanders"},
|
||||||
]
|
]
|
||||||
ruler = nlp.add_pipe("entity_ruler", config={"phrase_matcher_attr": "LOWER"})
|
ruler = nlp.add_pipe(
|
||||||
|
entity_ruler_factory,
|
||||||
|
name="entity_ruler",
|
||||||
|
config={"phrase_matcher_attr": "LOWER"},
|
||||||
|
)
|
||||||
ruler.add_patterns(patterns)
|
ruler.add_patterns(patterns)
|
||||||
text = """
|
text = """
|
||||||
The left is starting to take aim at Democratic front-runner Joe Biden.
|
The left is starting to take aim at Democratic front-runner Joe Biden.
|
||||||
|
@ -86,10 +96,11 @@ def test_issue4849():
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.issue(5918)
|
@pytest.mark.issue(5918)
|
||||||
def test_issue5918():
|
@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS)
|
||||||
|
def test_issue5918(entity_ruler_factory):
|
||||||
# Test edge case when merging entities.
|
# Test edge case when merging entities.
|
||||||
nlp = English()
|
nlp = English()
|
||||||
ruler = nlp.add_pipe("entity_ruler")
|
ruler = nlp.add_pipe(entity_ruler_factory, name="entity_ruler")
|
||||||
patterns = [
|
patterns = [
|
||||||
{"label": "ORG", "pattern": "Digicon Inc"},
|
{"label": "ORG", "pattern": "Digicon Inc"},
|
||||||
{"label": "ORG", "pattern": "Rotan Mosle Inc's"},
|
{"label": "ORG", "pattern": "Rotan Mosle Inc's"},
|
||||||
|
@ -114,9 +125,10 @@ def test_issue5918():
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.issue(8168)
|
@pytest.mark.issue(8168)
|
||||||
def test_issue8168():
|
@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS)
|
||||||
|
def test_issue8168(entity_ruler_factory):
|
||||||
nlp = English()
|
nlp = English()
|
||||||
ruler = nlp.add_pipe("entity_ruler")
|
ruler = nlp.add_pipe(entity_ruler_factory, name="entity_ruler")
|
||||||
patterns = [
|
patterns = [
|
||||||
{"label": "ORG", "pattern": "Apple"},
|
{"label": "ORG", "pattern": "Apple"},
|
||||||
{
|
{
|
||||||
|
@ -131,14 +143,17 @@ def test_issue8168():
|
||||||
},
|
},
|
||||||
]
|
]
|
||||||
ruler.add_patterns(patterns)
|
ruler.add_patterns(patterns)
|
||||||
|
doc = nlp("San Francisco San Fran")
|
||||||
assert ruler._ent_ids == {8043148519967183733: ("GPE", "san-francisco")}
|
assert all(t.ent_id_ == "san-francisco" for t in doc)
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.issue(8216)
|
@pytest.mark.issue(8216)
|
||||||
def test_entity_ruler_fix8216(nlp, patterns):
|
@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS)
|
||||||
|
def test_entity_ruler_fix8216(nlp, patterns, entity_ruler_factory):
|
||||||
"""Test that patterns don't get added excessively."""
|
"""Test that patterns don't get added excessively."""
|
||||||
ruler = nlp.add_pipe("entity_ruler", config={"validate": True})
|
ruler = nlp.add_pipe(
|
||||||
|
entity_ruler_factory, name="entity_ruler", config={"validate": True}
|
||||||
|
)
|
||||||
ruler.add_patterns(patterns)
|
ruler.add_patterns(patterns)
|
||||||
pattern_count = sum(len(mm) for mm in ruler.matcher._patterns.values())
|
pattern_count = sum(len(mm) for mm in ruler.matcher._patterns.values())
|
||||||
assert pattern_count > 0
|
assert pattern_count > 0
|
||||||
|
@ -147,13 +162,16 @@ def test_entity_ruler_fix8216(nlp, patterns):
|
||||||
assert after_count == pattern_count
|
assert after_count == pattern_count
|
||||||
|
|
||||||
|
|
||||||
def test_entity_ruler_init(nlp, patterns):
|
@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS)
|
||||||
ruler = EntityRuler(nlp, patterns=patterns)
|
def test_entity_ruler_init(nlp, patterns, entity_ruler_factory):
|
||||||
|
ruler = nlp.add_pipe(entity_ruler_factory, name="entity_ruler")
|
||||||
|
ruler.add_patterns(patterns)
|
||||||
assert len(ruler) == len(patterns)
|
assert len(ruler) == len(patterns)
|
||||||
assert len(ruler.labels) == 4
|
assert len(ruler.labels) == 4
|
||||||
assert "HELLO" in ruler
|
assert "HELLO" in ruler
|
||||||
assert "BYE" in ruler
|
assert "BYE" in ruler
|
||||||
ruler = nlp.add_pipe("entity_ruler")
|
nlp.remove_pipe("entity_ruler")
|
||||||
|
ruler = nlp.add_pipe(entity_ruler_factory, name="entity_ruler")
|
||||||
ruler.add_patterns(patterns)
|
ruler.add_patterns(patterns)
|
||||||
doc = nlp("hello world bye bye")
|
doc = nlp("hello world bye bye")
|
||||||
assert len(doc.ents) == 2
|
assert len(doc.ents) == 2
|
||||||
|
@ -161,20 +179,23 @@ def test_entity_ruler_init(nlp, patterns):
|
||||||
assert doc.ents[1].label_ == "BYE"
|
assert doc.ents[1].label_ == "BYE"
|
||||||
|
|
||||||
|
|
||||||
def test_entity_ruler_no_patterns_warns(nlp):
|
@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS)
|
||||||
ruler = EntityRuler(nlp)
|
def test_entity_ruler_no_patterns_warns(nlp, entity_ruler_factory):
|
||||||
|
ruler = nlp.add_pipe(entity_ruler_factory, name="entity_ruler")
|
||||||
assert len(ruler) == 0
|
assert len(ruler) == 0
|
||||||
assert len(ruler.labels) == 0
|
assert len(ruler.labels) == 0
|
||||||
nlp.add_pipe("entity_ruler")
|
nlp.remove_pipe("entity_ruler")
|
||||||
|
nlp.add_pipe(entity_ruler_factory, name="entity_ruler")
|
||||||
assert nlp.pipe_names == ["entity_ruler"]
|
assert nlp.pipe_names == ["entity_ruler"]
|
||||||
with pytest.warns(UserWarning):
|
with pytest.warns(UserWarning):
|
||||||
doc = nlp("hello world bye bye")
|
doc = nlp("hello world bye bye")
|
||||||
assert len(doc.ents) == 0
|
assert len(doc.ents) == 0
|
||||||
|
|
||||||
|
|
||||||
def test_entity_ruler_init_patterns(nlp, patterns):
|
@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS)
|
||||||
|
def test_entity_ruler_init_patterns(nlp, patterns, entity_ruler_factory):
|
||||||
# initialize with patterns
|
# initialize with patterns
|
||||||
ruler = nlp.add_pipe("entity_ruler")
|
ruler = nlp.add_pipe(entity_ruler_factory, name="entity_ruler")
|
||||||
assert len(ruler.labels) == 0
|
assert len(ruler.labels) == 0
|
||||||
ruler.initialize(lambda: [], patterns=patterns)
|
ruler.initialize(lambda: [], patterns=patterns)
|
||||||
assert len(ruler.labels) == 4
|
assert len(ruler.labels) == 4
|
||||||
|
@ -186,7 +207,7 @@ def test_entity_ruler_init_patterns(nlp, patterns):
|
||||||
nlp.config["initialize"]["components"]["entity_ruler"] = {
|
nlp.config["initialize"]["components"]["entity_ruler"] = {
|
||||||
"patterns": {"@misc": "entity_ruler_patterns"}
|
"patterns": {"@misc": "entity_ruler_patterns"}
|
||||||
}
|
}
|
||||||
ruler = nlp.add_pipe("entity_ruler")
|
ruler = nlp.add_pipe(entity_ruler_factory, name="entity_ruler")
|
||||||
assert len(ruler.labels) == 0
|
assert len(ruler.labels) == 0
|
||||||
nlp.initialize()
|
nlp.initialize()
|
||||||
assert len(ruler.labels) == 4
|
assert len(ruler.labels) == 4
|
||||||
|
@ -195,18 +216,20 @@ def test_entity_ruler_init_patterns(nlp, patterns):
|
||||||
assert doc.ents[1].label_ == "BYE"
|
assert doc.ents[1].label_ == "BYE"
|
||||||
|
|
||||||
|
|
||||||
def test_entity_ruler_init_clear(nlp, patterns):
|
@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS)
|
||||||
|
def test_entity_ruler_init_clear(nlp, patterns, entity_ruler_factory):
|
||||||
"""Test that initialization clears patterns."""
|
"""Test that initialization clears patterns."""
|
||||||
ruler = nlp.add_pipe("entity_ruler")
|
ruler = nlp.add_pipe(entity_ruler_factory, name="entity_ruler")
|
||||||
ruler.add_patterns(patterns)
|
ruler.add_patterns(patterns)
|
||||||
assert len(ruler.labels) == 4
|
assert len(ruler.labels) == 4
|
||||||
ruler.initialize(lambda: [])
|
ruler.initialize(lambda: [])
|
||||||
assert len(ruler.labels) == 0
|
assert len(ruler.labels) == 0
|
||||||
|
|
||||||
|
|
||||||
def test_entity_ruler_clear(nlp, patterns):
|
@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS)
|
||||||
|
def test_entity_ruler_clear(nlp, patterns, entity_ruler_factory):
|
||||||
"""Test that initialization clears patterns."""
|
"""Test that initialization clears patterns."""
|
||||||
ruler = nlp.add_pipe("entity_ruler")
|
ruler = nlp.add_pipe(entity_ruler_factory, name="entity_ruler")
|
||||||
ruler.add_patterns(patterns)
|
ruler.add_patterns(patterns)
|
||||||
assert len(ruler.labels) == 4
|
assert len(ruler.labels) == 4
|
||||||
doc = nlp("hello world")
|
doc = nlp("hello world")
|
||||||
|
@ -218,8 +241,9 @@ def test_entity_ruler_clear(nlp, patterns):
|
||||||
assert len(doc.ents) == 0
|
assert len(doc.ents) == 0
|
||||||
|
|
||||||
|
|
||||||
def test_entity_ruler_existing(nlp, patterns):
|
@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS)
|
||||||
ruler = nlp.add_pipe("entity_ruler")
|
def test_entity_ruler_existing(nlp, patterns, entity_ruler_factory):
|
||||||
|
ruler = nlp.add_pipe(entity_ruler_factory, name="entity_ruler")
|
||||||
ruler.add_patterns(patterns)
|
ruler.add_patterns(patterns)
|
||||||
nlp.add_pipe("add_ent", before="entity_ruler")
|
nlp.add_pipe("add_ent", before="entity_ruler")
|
||||||
doc = nlp("OH HELLO WORLD bye bye")
|
doc = nlp("OH HELLO WORLD bye bye")
|
||||||
|
@ -228,8 +252,11 @@ def test_entity_ruler_existing(nlp, patterns):
|
||||||
assert doc.ents[1].label_ == "BYE"
|
assert doc.ents[1].label_ == "BYE"
|
||||||
|
|
||||||
|
|
||||||
def test_entity_ruler_existing_overwrite(nlp, patterns):
|
@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS)
|
||||||
ruler = nlp.add_pipe("entity_ruler", config={"overwrite_ents": True})
|
def test_entity_ruler_existing_overwrite(nlp, patterns, entity_ruler_factory):
|
||||||
|
ruler = nlp.add_pipe(
|
||||||
|
entity_ruler_factory, name="entity_ruler", config={"overwrite_ents": True}
|
||||||
|
)
|
||||||
ruler.add_patterns(patterns)
|
ruler.add_patterns(patterns)
|
||||||
nlp.add_pipe("add_ent", before="entity_ruler")
|
nlp.add_pipe("add_ent", before="entity_ruler")
|
||||||
doc = nlp("OH HELLO WORLD bye bye")
|
doc = nlp("OH HELLO WORLD bye bye")
|
||||||
|
@ -239,8 +266,11 @@ def test_entity_ruler_existing_overwrite(nlp, patterns):
|
||||||
assert doc.ents[1].label_ == "BYE"
|
assert doc.ents[1].label_ == "BYE"
|
||||||
|
|
||||||
|
|
||||||
def test_entity_ruler_existing_complex(nlp, patterns):
|
@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS)
|
||||||
ruler = nlp.add_pipe("entity_ruler", config={"overwrite_ents": True})
|
def test_entity_ruler_existing_complex(nlp, patterns, entity_ruler_factory):
|
||||||
|
ruler = nlp.add_pipe(
|
||||||
|
entity_ruler_factory, name="entity_ruler", config={"overwrite_ents": True}
|
||||||
|
)
|
||||||
ruler.add_patterns(patterns)
|
ruler.add_patterns(patterns)
|
||||||
nlp.add_pipe("add_ent", before="entity_ruler")
|
nlp.add_pipe("add_ent", before="entity_ruler")
|
||||||
doc = nlp("foo foo bye bye")
|
doc = nlp("foo foo bye bye")
|
||||||
|
@ -251,8 +281,11 @@ def test_entity_ruler_existing_complex(nlp, patterns):
|
||||||
assert len(doc.ents[1]) == 2
|
assert len(doc.ents[1]) == 2
|
||||||
|
|
||||||
|
|
||||||
def test_entity_ruler_entity_id(nlp, patterns):
|
@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS)
|
||||||
ruler = nlp.add_pipe("entity_ruler", config={"overwrite_ents": True})
|
def test_entity_ruler_entity_id(nlp, patterns, entity_ruler_factory):
|
||||||
|
ruler = nlp.add_pipe(
|
||||||
|
entity_ruler_factory, name="entity_ruler", config={"overwrite_ents": True}
|
||||||
|
)
|
||||||
ruler.add_patterns(patterns)
|
ruler.add_patterns(patterns)
|
||||||
doc = nlp("Apple is a technology company")
|
doc = nlp("Apple is a technology company")
|
||||||
assert len(doc.ents) == 1
|
assert len(doc.ents) == 1
|
||||||
|
@ -260,18 +293,21 @@ def test_entity_ruler_entity_id(nlp, patterns):
|
||||||
assert doc.ents[0].ent_id_ == "a1"
|
assert doc.ents[0].ent_id_ == "a1"
|
||||||
|
|
||||||
|
|
||||||
def test_entity_ruler_cfg_ent_id_sep(nlp, patterns):
|
@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS)
|
||||||
|
def test_entity_ruler_cfg_ent_id_sep(nlp, patterns, entity_ruler_factory):
|
||||||
config = {"overwrite_ents": True, "ent_id_sep": "**"}
|
config = {"overwrite_ents": True, "ent_id_sep": "**"}
|
||||||
ruler = nlp.add_pipe("entity_ruler", config=config)
|
ruler = nlp.add_pipe(entity_ruler_factory, name="entity_ruler", config=config)
|
||||||
ruler.add_patterns(patterns)
|
ruler.add_patterns(patterns)
|
||||||
assert "TECH_ORG**a1" in ruler.phrase_patterns
|
|
||||||
doc = nlp("Apple is a technology company")
|
doc = nlp("Apple is a technology company")
|
||||||
|
if isinstance(ruler, EntityRuler):
|
||||||
|
assert "TECH_ORG**a1" in ruler.phrase_patterns
|
||||||
assert len(doc.ents) == 1
|
assert len(doc.ents) == 1
|
||||||
assert doc.ents[0].label_ == "TECH_ORG"
|
assert doc.ents[0].label_ == "TECH_ORG"
|
||||||
assert doc.ents[0].ent_id_ == "a1"
|
assert doc.ents[0].ent_id_ == "a1"
|
||||||
|
|
||||||
|
|
||||||
def test_entity_ruler_serialize_bytes(nlp, patterns):
|
@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS)
|
||||||
|
def test_entity_ruler_serialize_bytes(nlp, patterns, entity_ruler_factory):
|
||||||
ruler = EntityRuler(nlp, patterns=patterns)
|
ruler = EntityRuler(nlp, patterns=patterns)
|
||||||
assert len(ruler) == len(patterns)
|
assert len(ruler) == len(patterns)
|
||||||
assert len(ruler.labels) == 4
|
assert len(ruler.labels) == 4
|
||||||
|
@ -288,7 +324,10 @@ def test_entity_ruler_serialize_bytes(nlp, patterns):
|
||||||
assert sorted(new_ruler.labels) == sorted(ruler.labels)
|
assert sorted(new_ruler.labels) == sorted(ruler.labels)
|
||||||
|
|
||||||
|
|
||||||
def test_entity_ruler_serialize_phrase_matcher_attr_bytes(nlp, patterns):
|
@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS)
|
||||||
|
def test_entity_ruler_serialize_phrase_matcher_attr_bytes(
|
||||||
|
nlp, patterns, entity_ruler_factory
|
||||||
|
):
|
||||||
ruler = EntityRuler(nlp, phrase_matcher_attr="LOWER", patterns=patterns)
|
ruler = EntityRuler(nlp, phrase_matcher_attr="LOWER", patterns=patterns)
|
||||||
assert len(ruler) == len(patterns)
|
assert len(ruler) == len(patterns)
|
||||||
assert len(ruler.labels) == 4
|
assert len(ruler.labels) == 4
|
||||||
|
@ -303,8 +342,9 @@ def test_entity_ruler_serialize_phrase_matcher_attr_bytes(nlp, patterns):
|
||||||
assert new_ruler.phrase_matcher_attr == "LOWER"
|
assert new_ruler.phrase_matcher_attr == "LOWER"
|
||||||
|
|
||||||
|
|
||||||
def test_entity_ruler_validate(nlp):
|
@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS)
|
||||||
ruler = EntityRuler(nlp)
|
def test_entity_ruler_validate(nlp, entity_ruler_factory):
|
||||||
|
ruler = nlp.add_pipe(entity_ruler_factory, name="entity_ruler")
|
||||||
validated_ruler = EntityRuler(nlp, validate=True)
|
validated_ruler = EntityRuler(nlp, validate=True)
|
||||||
|
|
||||||
valid_pattern = {"label": "HELLO", "pattern": [{"LOWER": "HELLO"}]}
|
valid_pattern = {"label": "HELLO", "pattern": [{"LOWER": "HELLO"}]}
|
||||||
|
@ -322,32 +362,35 @@ def test_entity_ruler_validate(nlp):
|
||||||
validated_ruler.add_patterns([invalid_pattern])
|
validated_ruler.add_patterns([invalid_pattern])
|
||||||
|
|
||||||
|
|
||||||
def test_entity_ruler_properties(nlp, patterns):
|
@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS)
|
||||||
|
def test_entity_ruler_properties(nlp, patterns, entity_ruler_factory):
|
||||||
ruler = EntityRuler(nlp, patterns=patterns, overwrite_ents=True)
|
ruler = EntityRuler(nlp, patterns=patterns, overwrite_ents=True)
|
||||||
assert sorted(ruler.labels) == sorted(["HELLO", "BYE", "COMPLEX", "TECH_ORG"])
|
assert sorted(ruler.labels) == sorted(["HELLO", "BYE", "COMPLEX", "TECH_ORG"])
|
||||||
assert sorted(ruler.ent_ids) == ["a1", "a2"]
|
assert sorted(ruler.ent_ids) == ["a1", "a2"]
|
||||||
|
|
||||||
|
|
||||||
def test_entity_ruler_overlapping_spans(nlp):
|
@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS)
|
||||||
ruler = EntityRuler(nlp)
|
def test_entity_ruler_overlapping_spans(nlp, entity_ruler_factory):
|
||||||
|
ruler = nlp.add_pipe(entity_ruler_factory, name="entity_ruler")
|
||||||
patterns = [
|
patterns = [
|
||||||
{"label": "FOOBAR", "pattern": "foo bar"},
|
{"label": "FOOBAR", "pattern": "foo bar"},
|
||||||
{"label": "BARBAZ", "pattern": "bar baz"},
|
{"label": "BARBAZ", "pattern": "bar baz"},
|
||||||
]
|
]
|
||||||
ruler.add_patterns(patterns)
|
ruler.add_patterns(patterns)
|
||||||
doc = ruler(nlp.make_doc("foo bar baz"))
|
doc = nlp("foo bar baz")
|
||||||
assert len(doc.ents) == 1
|
assert len(doc.ents) == 1
|
||||||
assert doc.ents[0].label_ == "FOOBAR"
|
assert doc.ents[0].label_ == "FOOBAR"
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize("n_process", [1, 2])
|
@pytest.mark.parametrize("n_process", [1, 2])
|
||||||
def test_entity_ruler_multiprocessing(nlp, n_process):
|
@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS)
|
||||||
|
def test_entity_ruler_multiprocessing(nlp, n_process, entity_ruler_factory):
|
||||||
if isinstance(get_current_ops, NumpyOps) or n_process < 2:
|
if isinstance(get_current_ops, NumpyOps) or n_process < 2:
|
||||||
texts = ["I enjoy eating Pizza Hut pizza."]
|
texts = ["I enjoy eating Pizza Hut pizza."]
|
||||||
|
|
||||||
patterns = [{"label": "FASTFOOD", "pattern": "Pizza Hut", "id": "1234"}]
|
patterns = [{"label": "FASTFOOD", "pattern": "Pizza Hut", "id": "1234"}]
|
||||||
|
|
||||||
ruler = nlp.add_pipe("entity_ruler")
|
ruler = nlp.add_pipe(entity_ruler_factory, name="entity_ruler")
|
||||||
ruler.add_patterns(patterns)
|
ruler.add_patterns(patterns)
|
||||||
|
|
||||||
for doc in nlp.pipe(texts, n_process=2):
|
for doc in nlp.pipe(texts, n_process=2):
|
||||||
|
@ -355,8 +398,9 @@ def test_entity_ruler_multiprocessing(nlp, n_process):
|
||||||
assert ent.ent_id_ == "1234"
|
assert ent.ent_id_ == "1234"
|
||||||
|
|
||||||
|
|
||||||
def test_entity_ruler_serialize_jsonl(nlp, patterns):
|
@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS)
|
||||||
ruler = nlp.add_pipe("entity_ruler")
|
def test_entity_ruler_serialize_jsonl(nlp, patterns, entity_ruler_factory):
|
||||||
|
ruler = nlp.add_pipe(entity_ruler_factory, name="entity_ruler")
|
||||||
ruler.add_patterns(patterns)
|
ruler.add_patterns(patterns)
|
||||||
with make_tempdir() as d:
|
with make_tempdir() as d:
|
||||||
ruler.to_disk(d / "test_ruler.jsonl")
|
ruler.to_disk(d / "test_ruler.jsonl")
|
||||||
|
@ -365,8 +409,9 @@ def test_entity_ruler_serialize_jsonl(nlp, patterns):
|
||||||
ruler.from_disk(d / "non_existing.jsonl") # read from a bad jsonl file
|
ruler.from_disk(d / "non_existing.jsonl") # read from a bad jsonl file
|
||||||
|
|
||||||
|
|
||||||
def test_entity_ruler_serialize_dir(nlp, patterns):
|
@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS)
|
||||||
ruler = nlp.add_pipe("entity_ruler")
|
def test_entity_ruler_serialize_dir(nlp, patterns, entity_ruler_factory):
|
||||||
|
ruler = nlp.add_pipe(entity_ruler_factory, name="entity_ruler")
|
||||||
ruler.add_patterns(patterns)
|
ruler.add_patterns(patterns)
|
||||||
with make_tempdir() as d:
|
with make_tempdir() as d:
|
||||||
ruler.to_disk(d / "test_ruler")
|
ruler.to_disk(d / "test_ruler")
|
||||||
|
@ -375,52 +420,65 @@ def test_entity_ruler_serialize_dir(nlp, patterns):
|
||||||
ruler.from_disk(d / "non_existing_dir") # read from a bad directory
|
ruler.from_disk(d / "non_existing_dir") # read from a bad directory
|
||||||
|
|
||||||
|
|
||||||
def test_entity_ruler_remove_basic(nlp):
|
@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS)
|
||||||
ruler = EntityRuler(nlp)
|
def test_entity_ruler_remove_basic(nlp, entity_ruler_factory):
|
||||||
|
ruler = nlp.add_pipe(entity_ruler_factory, name="entity_ruler")
|
||||||
patterns = [
|
patterns = [
|
||||||
{"label": "PERSON", "pattern": "Duygu", "id": "duygu"},
|
{"label": "PERSON", "pattern": "Dina", "id": "dina"},
|
||||||
{"label": "ORG", "pattern": "ACME", "id": "acme"},
|
{"label": "ORG", "pattern": "ACME", "id": "acme"},
|
||||||
{"label": "ORG", "pattern": "ACM"},
|
{"label": "ORG", "pattern": "ACM"},
|
||||||
]
|
]
|
||||||
ruler.add_patterns(patterns)
|
ruler.add_patterns(patterns)
|
||||||
doc = ruler(nlp.make_doc("Duygu went to school"))
|
doc = nlp("Dina went to school")
|
||||||
assert len(ruler.patterns) == 3
|
assert len(ruler.patterns) == 3
|
||||||
assert len(doc.ents) == 1
|
assert len(doc.ents) == 1
|
||||||
|
if isinstance(ruler, EntityRuler):
|
||||||
|
assert "PERSON||dina" in ruler.phrase_matcher
|
||||||
assert doc.ents[0].label_ == "PERSON"
|
assert doc.ents[0].label_ == "PERSON"
|
||||||
assert doc.ents[0].text == "Duygu"
|
assert doc.ents[0].text == "Dina"
|
||||||
assert "PERSON||duygu" in ruler.phrase_matcher
|
if isinstance(ruler, EntityRuler):
|
||||||
ruler.remove("duygu")
|
ruler.remove("dina")
|
||||||
doc = ruler(nlp.make_doc("Duygu went to school"))
|
else:
|
||||||
|
ruler.remove_by_id("dina")
|
||||||
|
doc = nlp("Dina went to school")
|
||||||
assert len(doc.ents) == 0
|
assert len(doc.ents) == 0
|
||||||
assert "PERSON||duygu" not in ruler.phrase_matcher
|
if isinstance(ruler, EntityRuler):
|
||||||
|
assert "PERSON||dina" not in ruler.phrase_matcher
|
||||||
assert len(ruler.patterns) == 2
|
assert len(ruler.patterns) == 2
|
||||||
|
|
||||||
|
|
||||||
def test_entity_ruler_remove_same_id_multiple_patterns(nlp):
|
@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS)
|
||||||
ruler = EntityRuler(nlp)
|
def test_entity_ruler_remove_same_id_multiple_patterns(nlp, entity_ruler_factory):
|
||||||
|
ruler = nlp.add_pipe(entity_ruler_factory, name="entity_ruler")
|
||||||
patterns = [
|
patterns = [
|
||||||
{"label": "PERSON", "pattern": "Duygu", "id": "duygu"},
|
{"label": "PERSON", "pattern": "Dina", "id": "dina"},
|
||||||
{"label": "ORG", "pattern": "DuyguCorp", "id": "duygu"},
|
{"label": "ORG", "pattern": "DinaCorp", "id": "dina"},
|
||||||
{"label": "ORG", "pattern": "ACME", "id": "acme"},
|
{"label": "ORG", "pattern": "ACME", "id": "acme"},
|
||||||
]
|
]
|
||||||
ruler.add_patterns(patterns)
|
ruler.add_patterns(patterns)
|
||||||
doc = ruler(nlp.make_doc("Duygu founded DuyguCorp and ACME."))
|
doc = nlp("Dina founded DinaCorp and ACME.")
|
||||||
assert len(ruler.patterns) == 3
|
assert len(ruler.patterns) == 3
|
||||||
assert "PERSON||duygu" in ruler.phrase_matcher
|
if isinstance(ruler, EntityRuler):
|
||||||
assert "ORG||duygu" in ruler.phrase_matcher
|
assert "PERSON||dina" in ruler.phrase_matcher
|
||||||
|
assert "ORG||dina" in ruler.phrase_matcher
|
||||||
assert len(doc.ents) == 3
|
assert len(doc.ents) == 3
|
||||||
ruler.remove("duygu")
|
if isinstance(ruler, EntityRuler):
|
||||||
doc = ruler(nlp.make_doc("Duygu founded DuyguCorp and ACME."))
|
ruler.remove("dina")
|
||||||
|
else:
|
||||||
|
ruler.remove_by_id("dina")
|
||||||
|
doc = nlp("Dina founded DinaCorp and ACME.")
|
||||||
assert len(ruler.patterns) == 1
|
assert len(ruler.patterns) == 1
|
||||||
assert "PERSON||duygu" not in ruler.phrase_matcher
|
if isinstance(ruler, EntityRuler):
|
||||||
assert "ORG||duygu" not in ruler.phrase_matcher
|
assert "PERSON||dina" not in ruler.phrase_matcher
|
||||||
|
assert "ORG||dina" not in ruler.phrase_matcher
|
||||||
assert len(doc.ents) == 1
|
assert len(doc.ents) == 1
|
||||||
|
|
||||||
|
|
||||||
def test_entity_ruler_remove_nonexisting_pattern(nlp):
|
@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS)
|
||||||
ruler = EntityRuler(nlp)
|
def test_entity_ruler_remove_nonexisting_pattern(nlp, entity_ruler_factory):
|
||||||
|
ruler = nlp.add_pipe(entity_ruler_factory, name="entity_ruler")
|
||||||
patterns = [
|
patterns = [
|
||||||
{"label": "PERSON", "pattern": "Duygu", "id": "duygu"},
|
{"label": "PERSON", "pattern": "Dina", "id": "dina"},
|
||||||
{"label": "ORG", "pattern": "ACME", "id": "acme"},
|
{"label": "ORG", "pattern": "ACME", "id": "acme"},
|
||||||
{"label": "ORG", "pattern": "ACM"},
|
{"label": "ORG", "pattern": "ACM"},
|
||||||
]
|
]
|
||||||
|
@ -428,82 +486,108 @@ def test_entity_ruler_remove_nonexisting_pattern(nlp):
|
||||||
assert len(ruler.patterns) == 3
|
assert len(ruler.patterns) == 3
|
||||||
with pytest.raises(ValueError):
|
with pytest.raises(ValueError):
|
||||||
ruler.remove("nepattern")
|
ruler.remove("nepattern")
|
||||||
assert len(ruler.patterns) == 3
|
if isinstance(ruler, SpanRuler):
|
||||||
|
with pytest.raises(ValueError):
|
||||||
|
ruler.remove_by_id("nepattern")
|
||||||
|
|
||||||
|
|
||||||
def test_entity_ruler_remove_several_patterns(nlp):
|
@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS)
|
||||||
ruler = EntityRuler(nlp)
|
def test_entity_ruler_remove_several_patterns(nlp, entity_ruler_factory):
|
||||||
|
ruler = nlp.add_pipe(entity_ruler_factory, name="entity_ruler")
|
||||||
patterns = [
|
patterns = [
|
||||||
{"label": "PERSON", "pattern": "Duygu", "id": "duygu"},
|
{"label": "PERSON", "pattern": "Dina", "id": "dina"},
|
||||||
{"label": "ORG", "pattern": "ACME", "id": "acme"},
|
{"label": "ORG", "pattern": "ACME", "id": "acme"},
|
||||||
{"label": "ORG", "pattern": "ACM"},
|
{"label": "ORG", "pattern": "ACM"},
|
||||||
]
|
]
|
||||||
ruler.add_patterns(patterns)
|
ruler.add_patterns(patterns)
|
||||||
doc = ruler(nlp.make_doc("Duygu founded her company ACME."))
|
doc = nlp("Dina founded her company ACME.")
|
||||||
assert len(ruler.patterns) == 3
|
assert len(ruler.patterns) == 3
|
||||||
assert len(doc.ents) == 2
|
assert len(doc.ents) == 2
|
||||||
assert doc.ents[0].label_ == "PERSON"
|
assert doc.ents[0].label_ == "PERSON"
|
||||||
assert doc.ents[0].text == "Duygu"
|
assert doc.ents[0].text == "Dina"
|
||||||
assert doc.ents[1].label_ == "ORG"
|
assert doc.ents[1].label_ == "ORG"
|
||||||
assert doc.ents[1].text == "ACME"
|
assert doc.ents[1].text == "ACME"
|
||||||
ruler.remove("duygu")
|
if isinstance(ruler, EntityRuler):
|
||||||
doc = ruler(nlp.make_doc("Duygu founded her company ACME"))
|
ruler.remove("dina")
|
||||||
|
else:
|
||||||
|
ruler.remove_by_id("dina")
|
||||||
|
doc = nlp("Dina founded her company ACME")
|
||||||
assert len(ruler.patterns) == 2
|
assert len(ruler.patterns) == 2
|
||||||
assert len(doc.ents) == 1
|
assert len(doc.ents) == 1
|
||||||
assert doc.ents[0].label_ == "ORG"
|
assert doc.ents[0].label_ == "ORG"
|
||||||
assert doc.ents[0].text == "ACME"
|
assert doc.ents[0].text == "ACME"
|
||||||
ruler.remove("acme")
|
if isinstance(ruler, EntityRuler):
|
||||||
doc = ruler(nlp.make_doc("Duygu founded her company ACME"))
|
ruler.remove("acme")
|
||||||
|
else:
|
||||||
|
ruler.remove_by_id("acme")
|
||||||
|
doc = nlp("Dina founded her company ACME")
|
||||||
assert len(ruler.patterns) == 1
|
assert len(ruler.patterns) == 1
|
||||||
assert len(doc.ents) == 0
|
assert len(doc.ents) == 0
|
||||||
|
|
||||||
|
|
||||||
def test_entity_ruler_remove_patterns_in_a_row(nlp):
|
@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS)
|
||||||
ruler = EntityRuler(nlp)
|
def test_entity_ruler_remove_patterns_in_a_row(nlp, entity_ruler_factory):
|
||||||
|
ruler = nlp.add_pipe(entity_ruler_factory, name="entity_ruler")
|
||||||
patterns = [
|
patterns = [
|
||||||
{"label": "PERSON", "pattern": "Duygu", "id": "duygu"},
|
{"label": "PERSON", "pattern": "Dina", "id": "dina"},
|
||||||
{"label": "ORG", "pattern": "ACME", "id": "acme"},
|
{"label": "ORG", "pattern": "ACME", "id": "acme"},
|
||||||
{"label": "DATE", "pattern": "her birthday", "id": "bday"},
|
{"label": "DATE", "pattern": "her birthday", "id": "bday"},
|
||||||
{"label": "ORG", "pattern": "ACM"},
|
{"label": "ORG", "pattern": "ACM"},
|
||||||
]
|
]
|
||||||
ruler.add_patterns(patterns)
|
ruler.add_patterns(patterns)
|
||||||
doc = ruler(nlp.make_doc("Duygu founded her company ACME on her birthday"))
|
doc = nlp("Dina founded her company ACME on her birthday")
|
||||||
assert len(doc.ents) == 3
|
assert len(doc.ents) == 3
|
||||||
assert doc.ents[0].label_ == "PERSON"
|
assert doc.ents[0].label_ == "PERSON"
|
||||||
assert doc.ents[0].text == "Duygu"
|
assert doc.ents[0].text == "Dina"
|
||||||
assert doc.ents[1].label_ == "ORG"
|
assert doc.ents[1].label_ == "ORG"
|
||||||
assert doc.ents[1].text == "ACME"
|
assert doc.ents[1].text == "ACME"
|
||||||
assert doc.ents[2].label_ == "DATE"
|
assert doc.ents[2].label_ == "DATE"
|
||||||
assert doc.ents[2].text == "her birthday"
|
assert doc.ents[2].text == "her birthday"
|
||||||
ruler.remove("duygu")
|
if isinstance(ruler, EntityRuler):
|
||||||
ruler.remove("acme")
|
ruler.remove("dina")
|
||||||
ruler.remove("bday")
|
ruler.remove("acme")
|
||||||
doc = ruler(nlp.make_doc("Duygu went to school"))
|
ruler.remove("bday")
|
||||||
|
else:
|
||||||
|
ruler.remove_by_id("dina")
|
||||||
|
ruler.remove_by_id("acme")
|
||||||
|
ruler.remove_by_id("bday")
|
||||||
|
doc = nlp("Dina went to school")
|
||||||
assert len(doc.ents) == 0
|
assert len(doc.ents) == 0
|
||||||
|
|
||||||
|
|
||||||
def test_entity_ruler_remove_all_patterns(nlp):
|
@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS)
|
||||||
ruler = EntityRuler(nlp)
|
def test_entity_ruler_remove_all_patterns(nlp, entity_ruler_factory):
|
||||||
|
ruler = nlp.add_pipe(entity_ruler_factory, name="entity_ruler")
|
||||||
patterns = [
|
patterns = [
|
||||||
{"label": "PERSON", "pattern": "Duygu", "id": "duygu"},
|
{"label": "PERSON", "pattern": "Dina", "id": "dina"},
|
||||||
{"label": "ORG", "pattern": "ACME", "id": "acme"},
|
{"label": "ORG", "pattern": "ACME", "id": "acme"},
|
||||||
{"label": "DATE", "pattern": "her birthday", "id": "bday"},
|
{"label": "DATE", "pattern": "her birthday", "id": "bday"},
|
||||||
]
|
]
|
||||||
ruler.add_patterns(patterns)
|
ruler.add_patterns(patterns)
|
||||||
assert len(ruler.patterns) == 3
|
assert len(ruler.patterns) == 3
|
||||||
ruler.remove("duygu")
|
if isinstance(ruler, EntityRuler):
|
||||||
|
ruler.remove("dina")
|
||||||
|
else:
|
||||||
|
ruler.remove_by_id("dina")
|
||||||
assert len(ruler.patterns) == 2
|
assert len(ruler.patterns) == 2
|
||||||
ruler.remove("acme")
|
if isinstance(ruler, EntityRuler):
|
||||||
|
ruler.remove("acme")
|
||||||
|
else:
|
||||||
|
ruler.remove_by_id("acme")
|
||||||
assert len(ruler.patterns) == 1
|
assert len(ruler.patterns) == 1
|
||||||
ruler.remove("bday")
|
if isinstance(ruler, EntityRuler):
|
||||||
|
ruler.remove("bday")
|
||||||
|
else:
|
||||||
|
ruler.remove_by_id("bday")
|
||||||
assert len(ruler.patterns) == 0
|
assert len(ruler.patterns) == 0
|
||||||
with pytest.warns(UserWarning):
|
with pytest.warns(UserWarning):
|
||||||
doc = ruler(nlp.make_doc("Duygu founded her company ACME on her birthday"))
|
doc = nlp("Dina founded her company ACME on her birthday")
|
||||||
assert len(doc.ents) == 0
|
assert len(doc.ents) == 0
|
||||||
|
|
||||||
|
|
||||||
def test_entity_ruler_remove_and_add(nlp):
|
@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS)
|
||||||
ruler = EntityRuler(nlp)
|
def test_entity_ruler_remove_and_add(nlp, entity_ruler_factory):
|
||||||
|
ruler = nlp.add_pipe(entity_ruler_factory, name="entity_ruler")
|
||||||
patterns = [{"label": "DATE", "pattern": "last time"}]
|
patterns = [{"label": "DATE", "pattern": "last time"}]
|
||||||
ruler.add_patterns(patterns)
|
ruler.add_patterns(patterns)
|
||||||
doc = ruler(
|
doc = ruler(
|
||||||
|
@ -524,7 +608,10 @@ def test_entity_ruler_remove_and_add(nlp):
|
||||||
assert doc.ents[0].text == "last time"
|
assert doc.ents[0].text == "last time"
|
||||||
assert doc.ents[1].label_ == "DATE"
|
assert doc.ents[1].label_ == "DATE"
|
||||||
assert doc.ents[1].text == "this time"
|
assert doc.ents[1].text == "this time"
|
||||||
ruler.remove("ttime")
|
if isinstance(ruler, EntityRuler):
|
||||||
|
ruler.remove("ttime")
|
||||||
|
else:
|
||||||
|
ruler.remove_by_id("ttime")
|
||||||
doc = ruler(
|
doc = ruler(
|
||||||
nlp.make_doc("I saw him last time we met, this time he brought some flowers")
|
nlp.make_doc("I saw him last time we met, this time he brought some flowers")
|
||||||
)
|
)
|
||||||
|
@ -547,7 +634,10 @@ def test_entity_ruler_remove_and_add(nlp):
|
||||||
)
|
)
|
||||||
assert len(ruler.patterns) == 3
|
assert len(ruler.patterns) == 3
|
||||||
assert len(doc.ents) == 3
|
assert len(doc.ents) == 3
|
||||||
ruler.remove("ttime")
|
if isinstance(ruler, EntityRuler):
|
||||||
|
ruler.remove("ttime")
|
||||||
|
else:
|
||||||
|
ruler.remove_by_id("ttime")
|
||||||
doc = ruler(
|
doc = ruler(
|
||||||
nlp.make_doc(
|
nlp.make_doc(
|
||||||
"I saw him last time we met, this time he brought some flowers, another time some chocolate."
|
"I saw him last time we met, this time he brought some flowers, another time some chocolate."
|
||||||
|
|
|
@ -119,6 +119,7 @@ def test_pipe_class_component_config():
|
||||||
self.value1 = value1
|
self.value1 = value1
|
||||||
self.value2 = value2
|
self.value2 = value2
|
||||||
self.is_base = True
|
self.is_base = True
|
||||||
|
self.name = name
|
||||||
|
|
||||||
def __call__(self, doc: Doc) -> Doc:
|
def __call__(self, doc: Doc) -> Doc:
|
||||||
return doc
|
return doc
|
||||||
|
@ -141,12 +142,16 @@ def test_pipe_class_component_config():
|
||||||
nlp.add_pipe(name)
|
nlp.add_pipe(name)
|
||||||
with pytest.raises(ConfigValidationError): # invalid config
|
with pytest.raises(ConfigValidationError): # invalid config
|
||||||
nlp.add_pipe(name, config={"value1": "10", "value2": "hello"})
|
nlp.add_pipe(name, config={"value1": "10", "value2": "hello"})
|
||||||
nlp.add_pipe(name, config={"value1": 10, "value2": "hello"})
|
with pytest.warns(UserWarning):
|
||||||
|
nlp.add_pipe(
|
||||||
|
name, config={"value1": 10, "value2": "hello", "name": "wrong_name"}
|
||||||
|
)
|
||||||
pipe = nlp.get_pipe(name)
|
pipe = nlp.get_pipe(name)
|
||||||
assert isinstance(pipe.nlp, Language)
|
assert isinstance(pipe.nlp, Language)
|
||||||
assert pipe.value1 == 10
|
assert pipe.value1 == 10
|
||||||
assert pipe.value2 == "hello"
|
assert pipe.value2 == "hello"
|
||||||
assert pipe.is_base is True
|
assert pipe.is_base is True
|
||||||
|
assert pipe.name == name
|
||||||
|
|
||||||
nlp_en = English()
|
nlp_en = English()
|
||||||
with pytest.raises(ConfigValidationError): # invalid config
|
with pytest.raises(ConfigValidationError): # invalid config
|
||||||
|
|
|
@ -4,13 +4,14 @@ import numpy
|
||||||
import pytest
|
import pytest
|
||||||
from thinc.api import get_current_ops
|
from thinc.api import get_current_ops
|
||||||
|
|
||||||
|
import spacy
|
||||||
from spacy.lang.en import English
|
from spacy.lang.en import English
|
||||||
from spacy.lang.en.syntax_iterators import noun_chunks
|
from spacy.lang.en.syntax_iterators import noun_chunks
|
||||||
from spacy.language import Language
|
from spacy.language import Language
|
||||||
from spacy.pipeline import TrainablePipe
|
from spacy.pipeline import TrainablePipe
|
||||||
from spacy.tokens import Doc
|
from spacy.tokens import Doc
|
||||||
from spacy.training import Example
|
from spacy.training import Example
|
||||||
from spacy.util import SimpleFrozenList, get_arg_names
|
from spacy.util import SimpleFrozenList, get_arg_names, make_tempdir
|
||||||
from spacy.vocab import Vocab
|
from spacy.vocab import Vocab
|
||||||
|
|
||||||
|
|
||||||
|
@ -602,3 +603,52 @@ def test_update_with_annotates():
|
||||||
assert results[component] == "".join(eg.predicted.text for eg in examples)
|
assert results[component] == "".join(eg.predicted.text for eg in examples)
|
||||||
for component in components - set(components_to_annotate):
|
for component in components - set(components_to_annotate):
|
||||||
assert results[component] == ""
|
assert results[component] == ""
|
||||||
|
|
||||||
|
|
||||||
|
def test_load_disable_enable() -> None:
|
||||||
|
"""
|
||||||
|
Tests spacy.load() with dis-/enabling components.
|
||||||
|
"""
|
||||||
|
|
||||||
|
base_nlp = English()
|
||||||
|
for pipe in ("sentencizer", "tagger", "parser"):
|
||||||
|
base_nlp.add_pipe(pipe)
|
||||||
|
|
||||||
|
with make_tempdir() as tmp_dir:
|
||||||
|
base_nlp.to_disk(tmp_dir)
|
||||||
|
to_disable = ["parser", "tagger"]
|
||||||
|
to_enable = ["tagger", "parser"]
|
||||||
|
|
||||||
|
# Setting only `disable`.
|
||||||
|
nlp = spacy.load(tmp_dir, disable=to_disable)
|
||||||
|
assert all([comp_name in nlp.disabled for comp_name in to_disable])
|
||||||
|
|
||||||
|
# Setting only `enable`.
|
||||||
|
nlp = spacy.load(tmp_dir, enable=to_enable)
|
||||||
|
assert all(
|
||||||
|
[
|
||||||
|
(comp_name in nlp.disabled) is (comp_name not in to_enable)
|
||||||
|
for comp_name in nlp.component_names
|
||||||
|
]
|
||||||
|
)
|
||||||
|
|
||||||
|
# Testing consistent enable/disable combination.
|
||||||
|
nlp = spacy.load(
|
||||||
|
tmp_dir,
|
||||||
|
enable=to_enable,
|
||||||
|
disable=[
|
||||||
|
comp_name
|
||||||
|
for comp_name in nlp.component_names
|
||||||
|
if comp_name not in to_enable
|
||||||
|
],
|
||||||
|
)
|
||||||
|
assert all(
|
||||||
|
[
|
||||||
|
(comp_name in nlp.disabled) is (comp_name not in to_enable)
|
||||||
|
for comp_name in nlp.component_names
|
||||||
|
]
|
||||||
|
)
|
||||||
|
|
||||||
|
# Inconsistent enable/disable combination.
|
||||||
|
with pytest.raises(ValueError):
|
||||||
|
spacy.load(tmp_dir, enable=to_enable, disable=["parser"])
|
||||||
|
|
465
spacy/tests/pipeline/test_span_ruler.py
Normal file
465
spacy/tests/pipeline/test_span_ruler.py
Normal file
|
@ -0,0 +1,465 @@
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
import spacy
|
||||||
|
from spacy import registry
|
||||||
|
from spacy.errors import MatchPatternError
|
||||||
|
from spacy.tokens import Span
|
||||||
|
from spacy.training import Example
|
||||||
|
from spacy.tests.util import make_tempdir
|
||||||
|
|
||||||
|
from thinc.api import NumpyOps, get_current_ops
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
@registry.misc("span_ruler_patterns")
|
||||||
|
def patterns():
|
||||||
|
return [
|
||||||
|
{"label": "HELLO", "pattern": "hello world", "id": "hello1"},
|
||||||
|
{"label": "BYE", "pattern": [{"LOWER": "bye"}, {"LOWER": "bye"}]},
|
||||||
|
{"label": "HELLO", "pattern": [{"ORTH": "HELLO"}], "id": "hello2"},
|
||||||
|
{"label": "COMPLEX", "pattern": [{"ORTH": "foo", "OP": "*"}]},
|
||||||
|
{"label": "TECH_ORG", "pattern": "Apple"},
|
||||||
|
{"label": "TECH_ORG", "pattern": "Microsoft"},
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def overlapping_patterns():
|
||||||
|
return [
|
||||||
|
{"label": "FOOBAR", "pattern": "foo bar"},
|
||||||
|
{"label": "BARBAZ", "pattern": "bar baz"},
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def person_org_patterns():
|
||||||
|
return [
|
||||||
|
{"label": "PERSON", "pattern": "Dina"},
|
||||||
|
{"label": "ORG", "pattern": "ACME"},
|
||||||
|
{"label": "ORG", "pattern": "ACM"},
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def person_org_date_patterns(person_org_patterns):
|
||||||
|
return person_org_patterns + [{"label": "DATE", "pattern": "June 14th"}]
|
||||||
|
|
||||||
|
|
||||||
|
def test_span_ruler_add_empty(patterns):
|
||||||
|
"""Test that patterns don't get added excessively."""
|
||||||
|
nlp = spacy.blank("xx")
|
||||||
|
ruler = nlp.add_pipe("span_ruler", config={"validate": True})
|
||||||
|
ruler.add_patterns(patterns)
|
||||||
|
pattern_count = sum(len(mm) for mm in ruler.matcher._patterns.values())
|
||||||
|
assert pattern_count > 0
|
||||||
|
ruler.add_patterns([])
|
||||||
|
after_count = sum(len(mm) for mm in ruler.matcher._patterns.values())
|
||||||
|
assert after_count == pattern_count
|
||||||
|
|
||||||
|
|
||||||
|
def test_span_ruler_init(patterns):
|
||||||
|
nlp = spacy.blank("xx")
|
||||||
|
ruler = nlp.add_pipe("span_ruler")
|
||||||
|
ruler.add_patterns(patterns)
|
||||||
|
assert len(ruler) == len(patterns)
|
||||||
|
assert len(ruler.labels) == 4
|
||||||
|
assert "HELLO" in ruler
|
||||||
|
assert "BYE" in ruler
|
||||||
|
doc = nlp("hello world bye bye")
|
||||||
|
assert len(doc.spans["ruler"]) == 2
|
||||||
|
assert doc.spans["ruler"][0].label_ == "HELLO"
|
||||||
|
assert doc.spans["ruler"][0].id_ == "hello1"
|
||||||
|
assert doc.spans["ruler"][1].label_ == "BYE"
|
||||||
|
assert doc.spans["ruler"][1].id_ == ""
|
||||||
|
|
||||||
|
|
||||||
|
def test_span_ruler_no_patterns_warns():
|
||||||
|
nlp = spacy.blank("xx")
|
||||||
|
ruler = nlp.add_pipe("span_ruler")
|
||||||
|
assert len(ruler) == 0
|
||||||
|
assert len(ruler.labels) == 0
|
||||||
|
assert nlp.pipe_names == ["span_ruler"]
|
||||||
|
with pytest.warns(UserWarning):
|
||||||
|
doc = nlp("hello world bye bye")
|
||||||
|
assert len(doc.spans["ruler"]) == 0
|
||||||
|
|
||||||
|
|
||||||
|
def test_span_ruler_init_patterns(patterns):
|
||||||
|
# initialize with patterns
|
||||||
|
nlp = spacy.blank("xx")
|
||||||
|
ruler = nlp.add_pipe("span_ruler")
|
||||||
|
assert len(ruler.labels) == 0
|
||||||
|
ruler.initialize(lambda: [], patterns=patterns)
|
||||||
|
assert len(ruler.labels) == 4
|
||||||
|
doc = nlp("hello world bye bye")
|
||||||
|
assert doc.spans["ruler"][0].label_ == "HELLO"
|
||||||
|
assert doc.spans["ruler"][1].label_ == "BYE"
|
||||||
|
nlp.remove_pipe("span_ruler")
|
||||||
|
# initialize with patterns from misc registry
|
||||||
|
nlp.config["initialize"]["components"]["span_ruler"] = {
|
||||||
|
"patterns": {"@misc": "span_ruler_patterns"}
|
||||||
|
}
|
||||||
|
ruler = nlp.add_pipe("span_ruler")
|
||||||
|
assert len(ruler.labels) == 0
|
||||||
|
nlp.initialize()
|
||||||
|
assert len(ruler.labels) == 4
|
||||||
|
doc = nlp("hello world bye bye")
|
||||||
|
assert doc.spans["ruler"][0].label_ == "HELLO"
|
||||||
|
assert doc.spans["ruler"][1].label_ == "BYE"
|
||||||
|
|
||||||
|
|
||||||
|
def test_span_ruler_init_clear(patterns):
|
||||||
|
"""Test that initialization clears patterns."""
|
||||||
|
nlp = spacy.blank("xx")
|
||||||
|
ruler = nlp.add_pipe("span_ruler")
|
||||||
|
ruler.add_patterns(patterns)
|
||||||
|
assert len(ruler.labels) == 4
|
||||||
|
ruler.initialize(lambda: [])
|
||||||
|
assert len(ruler.labels) == 0
|
||||||
|
|
||||||
|
|
||||||
|
def test_span_ruler_clear(patterns):
|
||||||
|
nlp = spacy.blank("xx")
|
||||||
|
ruler = nlp.add_pipe("span_ruler")
|
||||||
|
ruler.add_patterns(patterns)
|
||||||
|
assert len(ruler.labels) == 4
|
||||||
|
doc = nlp("hello world")
|
||||||
|
assert len(doc.spans["ruler"]) == 1
|
||||||
|
ruler.clear()
|
||||||
|
assert len(ruler.labels) == 0
|
||||||
|
with pytest.warns(UserWarning):
|
||||||
|
doc = nlp("hello world")
|
||||||
|
assert len(doc.spans["ruler"]) == 0
|
||||||
|
|
||||||
|
|
||||||
|
def test_span_ruler_existing(patterns):
|
||||||
|
nlp = spacy.blank("xx")
|
||||||
|
ruler = nlp.add_pipe("span_ruler", config={"overwrite": False})
|
||||||
|
ruler.add_patterns(patterns)
|
||||||
|
doc = nlp.make_doc("OH HELLO WORLD bye bye")
|
||||||
|
doc.spans["ruler"] = [doc[0:2]]
|
||||||
|
doc = nlp(doc)
|
||||||
|
assert len(doc.spans["ruler"]) == 3
|
||||||
|
assert doc.spans["ruler"][0] == doc[0:2]
|
||||||
|
assert doc.spans["ruler"][1].label_ == "HELLO"
|
||||||
|
assert doc.spans["ruler"][1].id_ == "hello2"
|
||||||
|
assert doc.spans["ruler"][2].label_ == "BYE"
|
||||||
|
assert doc.spans["ruler"][2].id_ == ""
|
||||||
|
|
||||||
|
|
||||||
|
def test_span_ruler_existing_overwrite(patterns):
|
||||||
|
nlp = spacy.blank("xx")
|
||||||
|
ruler = nlp.add_pipe("span_ruler", config={"overwrite": True})
|
||||||
|
ruler.add_patterns(patterns)
|
||||||
|
doc = nlp.make_doc("OH HELLO WORLD bye bye")
|
||||||
|
doc.spans["ruler"] = [doc[0:2]]
|
||||||
|
doc = nlp(doc)
|
||||||
|
assert len(doc.spans["ruler"]) == 2
|
||||||
|
assert doc.spans["ruler"][0].label_ == "HELLO"
|
||||||
|
assert doc.spans["ruler"][0].text == "HELLO"
|
||||||
|
assert doc.spans["ruler"][1].label_ == "BYE"
|
||||||
|
|
||||||
|
|
||||||
|
def test_span_ruler_serialize_bytes(patterns):
|
||||||
|
nlp = spacy.blank("xx")
|
||||||
|
ruler = nlp.add_pipe("span_ruler")
|
||||||
|
ruler.add_patterns(patterns)
|
||||||
|
assert len(ruler) == len(patterns)
|
||||||
|
assert len(ruler.labels) == 4
|
||||||
|
ruler_bytes = ruler.to_bytes()
|
||||||
|
new_nlp = spacy.blank("xx")
|
||||||
|
new_ruler = new_nlp.add_pipe("span_ruler")
|
||||||
|
assert len(new_ruler) == 0
|
||||||
|
assert len(new_ruler.labels) == 0
|
||||||
|
new_ruler = new_ruler.from_bytes(ruler_bytes)
|
||||||
|
assert len(new_ruler) == len(patterns)
|
||||||
|
assert len(new_ruler.labels) == 4
|
||||||
|
assert len(new_ruler.patterns) == len(ruler.patterns)
|
||||||
|
for pattern in ruler.patterns:
|
||||||
|
assert pattern in new_ruler.patterns
|
||||||
|
assert sorted(new_ruler.labels) == sorted(ruler.labels)
|
||||||
|
|
||||||
|
|
||||||
|
def test_span_ruler_validate():
|
||||||
|
nlp = spacy.blank("xx")
|
||||||
|
ruler = nlp.add_pipe("span_ruler")
|
||||||
|
validated_ruler = nlp.add_pipe(
|
||||||
|
"span_ruler", name="validated_span_ruler", config={"validate": True}
|
||||||
|
)
|
||||||
|
|
||||||
|
valid_pattern = {"label": "HELLO", "pattern": [{"LOWER": "HELLO"}]}
|
||||||
|
invalid_pattern = {"label": "HELLO", "pattern": [{"ASDF": "HELLO"}]}
|
||||||
|
|
||||||
|
# invalid pattern raises error without validate
|
||||||
|
with pytest.raises(ValueError):
|
||||||
|
ruler.add_patterns([invalid_pattern])
|
||||||
|
|
||||||
|
# valid pattern is added without errors with validate
|
||||||
|
validated_ruler.add_patterns([valid_pattern])
|
||||||
|
|
||||||
|
# invalid pattern raises error with validate
|
||||||
|
with pytest.raises(MatchPatternError):
|
||||||
|
validated_ruler.add_patterns([invalid_pattern])
|
||||||
|
|
||||||
|
|
||||||
|
def test_span_ruler_properties(patterns):
|
||||||
|
nlp = spacy.blank("xx")
|
||||||
|
ruler = nlp.add_pipe("span_ruler", config={"overwrite": True})
|
||||||
|
ruler.add_patterns(patterns)
|
||||||
|
assert sorted(ruler.labels) == sorted(set([p["label"] for p in patterns]))
|
||||||
|
|
||||||
|
|
||||||
|
def test_span_ruler_overlapping_spans(overlapping_patterns):
|
||||||
|
nlp = spacy.blank("xx")
|
||||||
|
ruler = nlp.add_pipe("span_ruler")
|
||||||
|
ruler.add_patterns(overlapping_patterns)
|
||||||
|
doc = ruler(nlp.make_doc("foo bar baz"))
|
||||||
|
assert len(doc.spans["ruler"]) == 2
|
||||||
|
assert doc.spans["ruler"][0].label_ == "FOOBAR"
|
||||||
|
assert doc.spans["ruler"][1].label_ == "BARBAZ"
|
||||||
|
|
||||||
|
|
||||||
|
def test_span_ruler_scorer(overlapping_patterns):
|
||||||
|
nlp = spacy.blank("xx")
|
||||||
|
ruler = nlp.add_pipe("span_ruler")
|
||||||
|
ruler.add_patterns(overlapping_patterns)
|
||||||
|
text = "foo bar baz"
|
||||||
|
pred_doc = ruler(nlp.make_doc(text))
|
||||||
|
assert len(pred_doc.spans["ruler"]) == 2
|
||||||
|
assert pred_doc.spans["ruler"][0].label_ == "FOOBAR"
|
||||||
|
assert pred_doc.spans["ruler"][1].label_ == "BARBAZ"
|
||||||
|
|
||||||
|
ref_doc = nlp.make_doc(text)
|
||||||
|
ref_doc.spans["ruler"] = [Span(ref_doc, 0, 2, label="FOOBAR")]
|
||||||
|
scores = nlp.evaluate([Example(pred_doc, ref_doc)])
|
||||||
|
assert scores["spans_ruler_p"] == 0.5
|
||||||
|
assert scores["spans_ruler_r"] == 1.0
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("n_process", [1, 2])
|
||||||
|
def test_span_ruler_multiprocessing(n_process):
|
||||||
|
if isinstance(get_current_ops, NumpyOps) or n_process < 2:
|
||||||
|
texts = ["I enjoy eating Pizza Hut pizza."]
|
||||||
|
|
||||||
|
patterns = [{"label": "FASTFOOD", "pattern": "Pizza Hut"}]
|
||||||
|
|
||||||
|
nlp = spacy.blank("xx")
|
||||||
|
ruler = nlp.add_pipe("span_ruler")
|
||||||
|
ruler.add_patterns(patterns)
|
||||||
|
|
||||||
|
for doc in nlp.pipe(texts, n_process=2):
|
||||||
|
for ent in doc.spans["ruler"]:
|
||||||
|
assert ent.label_ == "FASTFOOD"
|
||||||
|
|
||||||
|
|
||||||
|
def test_span_ruler_serialize_dir(patterns):
|
||||||
|
nlp = spacy.blank("xx")
|
||||||
|
ruler = nlp.add_pipe("span_ruler")
|
||||||
|
ruler.add_patterns(patterns)
|
||||||
|
with make_tempdir() as d:
|
||||||
|
ruler.to_disk(d / "test_ruler")
|
||||||
|
ruler.from_disk(d / "test_ruler") # read from an existing directory
|
||||||
|
with pytest.raises(ValueError):
|
||||||
|
ruler.from_disk(d / "non_existing_dir") # read from a bad directory
|
||||||
|
|
||||||
|
|
||||||
|
def test_span_ruler_remove_basic(person_org_patterns):
|
||||||
|
nlp = spacy.blank("xx")
|
||||||
|
ruler = nlp.add_pipe("span_ruler")
|
||||||
|
ruler.add_patterns(person_org_patterns)
|
||||||
|
doc = ruler(nlp.make_doc("Dina went to school"))
|
||||||
|
assert len(ruler.patterns) == 3
|
||||||
|
assert len(doc.spans["ruler"]) == 1
|
||||||
|
assert doc.spans["ruler"][0].label_ == "PERSON"
|
||||||
|
assert doc.spans["ruler"][0].text == "Dina"
|
||||||
|
ruler.remove("PERSON")
|
||||||
|
doc = ruler(nlp.make_doc("Dina went to school"))
|
||||||
|
assert len(doc.spans["ruler"]) == 0
|
||||||
|
assert len(ruler.patterns) == 2
|
||||||
|
|
||||||
|
|
||||||
|
def test_span_ruler_remove_nonexisting_pattern(person_org_patterns):
|
||||||
|
nlp = spacy.blank("xx")
|
||||||
|
ruler = nlp.add_pipe("span_ruler")
|
||||||
|
ruler.add_patterns(person_org_patterns)
|
||||||
|
assert len(ruler.patterns) == 3
|
||||||
|
with pytest.raises(ValueError):
|
||||||
|
ruler.remove("NE")
|
||||||
|
with pytest.raises(ValueError):
|
||||||
|
ruler.remove_by_id("NE")
|
||||||
|
|
||||||
|
|
||||||
|
def test_span_ruler_remove_several_patterns(person_org_patterns):
|
||||||
|
nlp = spacy.blank("xx")
|
||||||
|
ruler = nlp.add_pipe("span_ruler")
|
||||||
|
ruler.add_patterns(person_org_patterns)
|
||||||
|
doc = ruler(nlp.make_doc("Dina founded the company ACME."))
|
||||||
|
assert len(ruler.patterns) == 3
|
||||||
|
assert len(doc.spans["ruler"]) == 2
|
||||||
|
assert doc.spans["ruler"][0].label_ == "PERSON"
|
||||||
|
assert doc.spans["ruler"][0].text == "Dina"
|
||||||
|
assert doc.spans["ruler"][1].label_ == "ORG"
|
||||||
|
assert doc.spans["ruler"][1].text == "ACME"
|
||||||
|
ruler.remove("PERSON")
|
||||||
|
doc = ruler(nlp.make_doc("Dina founded the company ACME"))
|
||||||
|
assert len(ruler.patterns) == 2
|
||||||
|
assert len(doc.spans["ruler"]) == 1
|
||||||
|
assert doc.spans["ruler"][0].label_ == "ORG"
|
||||||
|
assert doc.spans["ruler"][0].text == "ACME"
|
||||||
|
ruler.remove("ORG")
|
||||||
|
with pytest.warns(UserWarning):
|
||||||
|
doc = ruler(nlp.make_doc("Dina founded the company ACME"))
|
||||||
|
assert len(ruler.patterns) == 0
|
||||||
|
assert len(doc.spans["ruler"]) == 0
|
||||||
|
|
||||||
|
|
||||||
|
def test_span_ruler_remove_patterns_in_a_row(person_org_date_patterns):
|
||||||
|
nlp = spacy.blank("xx")
|
||||||
|
ruler = nlp.add_pipe("span_ruler")
|
||||||
|
ruler.add_patterns(person_org_date_patterns)
|
||||||
|
doc = ruler(nlp.make_doc("Dina founded the company ACME on June 14th"))
|
||||||
|
assert len(doc.spans["ruler"]) == 3
|
||||||
|
assert doc.spans["ruler"][0].label_ == "PERSON"
|
||||||
|
assert doc.spans["ruler"][0].text == "Dina"
|
||||||
|
assert doc.spans["ruler"][1].label_ == "ORG"
|
||||||
|
assert doc.spans["ruler"][1].text == "ACME"
|
||||||
|
assert doc.spans["ruler"][2].label_ == "DATE"
|
||||||
|
assert doc.spans["ruler"][2].text == "June 14th"
|
||||||
|
ruler.remove("ORG")
|
||||||
|
ruler.remove("DATE")
|
||||||
|
doc = ruler(nlp.make_doc("Dina went to school"))
|
||||||
|
assert len(doc.spans["ruler"]) == 1
|
||||||
|
|
||||||
|
|
||||||
|
def test_span_ruler_remove_all_patterns(person_org_date_patterns):
|
||||||
|
nlp = spacy.blank("xx")
|
||||||
|
ruler = nlp.add_pipe("span_ruler")
|
||||||
|
ruler.add_patterns(person_org_date_patterns)
|
||||||
|
assert len(ruler.patterns) == 4
|
||||||
|
ruler.remove("PERSON")
|
||||||
|
assert len(ruler.patterns) == 3
|
||||||
|
ruler.remove("ORG")
|
||||||
|
assert len(ruler.patterns) == 1
|
||||||
|
ruler.remove("DATE")
|
||||||
|
assert len(ruler.patterns) == 0
|
||||||
|
with pytest.warns(UserWarning):
|
||||||
|
doc = ruler(nlp.make_doc("Dina founded the company ACME on June 14th"))
|
||||||
|
assert len(doc.spans["ruler"]) == 0
|
||||||
|
|
||||||
|
|
||||||
|
def test_span_ruler_remove_and_add():
|
||||||
|
nlp = spacy.blank("xx")
|
||||||
|
ruler = nlp.add_pipe("span_ruler")
|
||||||
|
patterns1 = [{"label": "DATE1", "pattern": "last time"}]
|
||||||
|
ruler.add_patterns(patterns1)
|
||||||
|
doc = ruler(
|
||||||
|
nlp.make_doc("I saw him last time we met, this time he brought some flowers")
|
||||||
|
)
|
||||||
|
assert len(ruler.patterns) == 1
|
||||||
|
assert len(doc.spans["ruler"]) == 1
|
||||||
|
assert doc.spans["ruler"][0].label_ == "DATE1"
|
||||||
|
assert doc.spans["ruler"][0].text == "last time"
|
||||||
|
patterns2 = [{"label": "DATE2", "pattern": "this time"}]
|
||||||
|
ruler.add_patterns(patterns2)
|
||||||
|
doc = ruler(
|
||||||
|
nlp.make_doc("I saw him last time we met, this time he brought some flowers")
|
||||||
|
)
|
||||||
|
assert len(ruler.patterns) == 2
|
||||||
|
assert len(doc.spans["ruler"]) == 2
|
||||||
|
assert doc.spans["ruler"][0].label_ == "DATE1"
|
||||||
|
assert doc.spans["ruler"][0].text == "last time"
|
||||||
|
assert doc.spans["ruler"][1].label_ == "DATE2"
|
||||||
|
assert doc.spans["ruler"][1].text == "this time"
|
||||||
|
ruler.remove("DATE1")
|
||||||
|
doc = ruler(
|
||||||
|
nlp.make_doc("I saw him last time we met, this time he brought some flowers")
|
||||||
|
)
|
||||||
|
assert len(ruler.patterns) == 1
|
||||||
|
assert len(doc.spans["ruler"]) == 1
|
||||||
|
assert doc.spans["ruler"][0].label_ == "DATE2"
|
||||||
|
assert doc.spans["ruler"][0].text == "this time"
|
||||||
|
ruler.add_patterns(patterns1)
|
||||||
|
doc = ruler(
|
||||||
|
nlp.make_doc("I saw him last time we met, this time he brought some flowers")
|
||||||
|
)
|
||||||
|
assert len(ruler.patterns) == 2
|
||||||
|
assert len(doc.spans["ruler"]) == 2
|
||||||
|
patterns3 = [{"label": "DATE3", "pattern": "another time"}]
|
||||||
|
ruler.add_patterns(patterns3)
|
||||||
|
doc = ruler(
|
||||||
|
nlp.make_doc(
|
||||||
|
"I saw him last time we met, this time he brought some flowers, another time some chocolate."
|
||||||
|
)
|
||||||
|
)
|
||||||
|
assert len(ruler.patterns) == 3
|
||||||
|
assert len(doc.spans["ruler"]) == 3
|
||||||
|
ruler.remove("DATE3")
|
||||||
|
doc = ruler(
|
||||||
|
nlp.make_doc(
|
||||||
|
"I saw him last time we met, this time he brought some flowers, another time some chocolate."
|
||||||
|
)
|
||||||
|
)
|
||||||
|
assert len(ruler.patterns) == 2
|
||||||
|
assert len(doc.spans["ruler"]) == 2
|
||||||
|
|
||||||
|
|
||||||
|
def test_span_ruler_spans_filter(overlapping_patterns):
|
||||||
|
nlp = spacy.blank("xx")
|
||||||
|
ruler = nlp.add_pipe(
|
||||||
|
"span_ruler",
|
||||||
|
config={"spans_filter": {"@misc": "spacy.first_longest_spans_filter.v1"}},
|
||||||
|
)
|
||||||
|
ruler.add_patterns(overlapping_patterns)
|
||||||
|
doc = ruler(nlp.make_doc("foo bar baz"))
|
||||||
|
assert len(doc.spans["ruler"]) == 1
|
||||||
|
assert doc.spans["ruler"][0].label_ == "FOOBAR"
|
||||||
|
|
||||||
|
|
||||||
|
def test_span_ruler_ents_default_filter(overlapping_patterns):
|
||||||
|
nlp = spacy.blank("xx")
|
||||||
|
ruler = nlp.add_pipe("span_ruler", config={"annotate_ents": True})
|
||||||
|
ruler.add_patterns(overlapping_patterns)
|
||||||
|
doc = ruler(nlp.make_doc("foo bar baz"))
|
||||||
|
assert len(doc.ents) == 1
|
||||||
|
assert doc.ents[0].label_ == "FOOBAR"
|
||||||
|
|
||||||
|
|
||||||
|
def test_span_ruler_ents_overwrite_filter(overlapping_patterns):
|
||||||
|
nlp = spacy.blank("xx")
|
||||||
|
ruler = nlp.add_pipe(
|
||||||
|
"span_ruler",
|
||||||
|
config={
|
||||||
|
"annotate_ents": True,
|
||||||
|
"overwrite": False,
|
||||||
|
"ents_filter": {"@misc": "spacy.prioritize_new_ents_filter.v1"},
|
||||||
|
},
|
||||||
|
)
|
||||||
|
ruler.add_patterns(overlapping_patterns)
|
||||||
|
# overlapping ents are clobbered, non-overlapping ents are preserved
|
||||||
|
doc = nlp.make_doc("foo bar baz a b c")
|
||||||
|
doc.ents = [Span(doc, 1, 3, label="BARBAZ"), Span(doc, 3, 6, label="ABC")]
|
||||||
|
doc = ruler(doc)
|
||||||
|
assert len(doc.ents) == 2
|
||||||
|
assert doc.ents[0].label_ == "FOOBAR"
|
||||||
|
assert doc.ents[1].label_ == "ABC"
|
||||||
|
|
||||||
|
|
||||||
|
def test_span_ruler_ents_bad_filter(overlapping_patterns):
|
||||||
|
@registry.misc("test_pass_through_filter")
|
||||||
|
def make_pass_through_filter():
|
||||||
|
def pass_through_filter(spans1, spans2):
|
||||||
|
return spans1 + spans2
|
||||||
|
|
||||||
|
return pass_through_filter
|
||||||
|
|
||||||
|
nlp = spacy.blank("xx")
|
||||||
|
ruler = nlp.add_pipe(
|
||||||
|
"span_ruler",
|
||||||
|
config={
|
||||||
|
"annotate_ents": True,
|
||||||
|
"ents_filter": {"@misc": "test_pass_through_filter"},
|
||||||
|
},
|
||||||
|
)
|
||||||
|
ruler.add_patterns(overlapping_patterns)
|
||||||
|
with pytest.raises(ValueError):
|
||||||
|
ruler(nlp.make_doc("foo bar baz"))
|
|
@ -382,6 +382,7 @@ def test_implicit_label(name, get_examples):
|
||||||
|
|
||||||
|
|
||||||
# fmt: off
|
# fmt: off
|
||||||
|
@pytest.mark.slow
|
||||||
@pytest.mark.parametrize(
|
@pytest.mark.parametrize(
|
||||||
"name,textcat_config",
|
"name,textcat_config",
|
||||||
[
|
[
|
||||||
|
@ -390,7 +391,10 @@ def test_implicit_label(name, get_examples):
|
||||||
("textcat", {"@architectures": "spacy.TextCatBOW.v1", "exclusive_classes": True, "no_output_layer": True, "ngram_size": 3}),
|
("textcat", {"@architectures": "spacy.TextCatBOW.v1", "exclusive_classes": True, "no_output_layer": True, "ngram_size": 3}),
|
||||||
("textcat_multilabel", {"@architectures": "spacy.TextCatBOW.v1", "exclusive_classes": False, "no_output_layer": False, "ngram_size": 3}),
|
("textcat_multilabel", {"@architectures": "spacy.TextCatBOW.v1", "exclusive_classes": False, "no_output_layer": False, "ngram_size": 3}),
|
||||||
("textcat_multilabel", {"@architectures": "spacy.TextCatBOW.v1", "exclusive_classes": False, "no_output_layer": True, "ngram_size": 3}),
|
("textcat_multilabel", {"@architectures": "spacy.TextCatBOW.v1", "exclusive_classes": False, "no_output_layer": True, "ngram_size": 3}),
|
||||||
# ENSEMBLE
|
# ENSEMBLE V1
|
||||||
|
("textcat", {"@architectures": "spacy.TextCatEnsemble.v1", "exclusive_classes": False, "pretrained_vectors": None, "width": 64, "embed_size": 2000, "conv_depth": 2, "window_size": 1, "ngram_size": 1, "dropout": None}),
|
||||||
|
("textcat_multilabel", {"@architectures": "spacy.TextCatEnsemble.v1", "exclusive_classes": False, "pretrained_vectors": None, "width": 64, "embed_size": 2000, "conv_depth": 2, "window_size": 1, "ngram_size": 1, "dropout": None}),
|
||||||
|
# ENSEMBLE V2
|
||||||
("textcat", {"@architectures": "spacy.TextCatEnsemble.v2", "tok2vec": DEFAULT_TOK2VEC_MODEL, "linear_model": {"@architectures": "spacy.TextCatBOW.v1", "exclusive_classes": True, "no_output_layer": False, "ngram_size": 3}}),
|
("textcat", {"@architectures": "spacy.TextCatEnsemble.v2", "tok2vec": DEFAULT_TOK2VEC_MODEL, "linear_model": {"@architectures": "spacy.TextCatBOW.v1", "exclusive_classes": True, "no_output_layer": False, "ngram_size": 3}}),
|
||||||
("textcat", {"@architectures": "spacy.TextCatEnsemble.v2", "tok2vec": DEFAULT_TOK2VEC_MODEL, "linear_model": {"@architectures": "spacy.TextCatBOW.v1", "exclusive_classes": True, "no_output_layer": True, "ngram_size": 3}}),
|
("textcat", {"@architectures": "spacy.TextCatEnsemble.v2", "tok2vec": DEFAULT_TOK2VEC_MODEL, "linear_model": {"@architectures": "spacy.TextCatBOW.v1", "exclusive_classes": True, "no_output_layer": True, "ngram_size": 3}}),
|
||||||
("textcat_multilabel", {"@architectures": "spacy.TextCatEnsemble.v2", "tok2vec": DEFAULT_TOK2VEC_MODEL, "linear_model": {"@architectures": "spacy.TextCatBOW.v1", "exclusive_classes": False, "no_output_layer": False, "ngram_size": 3}}),
|
("textcat_multilabel", {"@architectures": "spacy.TextCatEnsemble.v2", "tok2vec": DEFAULT_TOK2VEC_MODEL, "linear_model": {"@architectures": "spacy.TextCatBOW.v1", "exclusive_classes": False, "no_output_layer": False, "ngram_size": 3}}),
|
||||||
|
@ -643,15 +647,28 @@ def test_overfitting_IO_multi():
|
||||||
|
|
||||||
|
|
||||||
# fmt: off
|
# fmt: off
|
||||||
|
@pytest.mark.slow
|
||||||
@pytest.mark.parametrize(
|
@pytest.mark.parametrize(
|
||||||
"name,train_data,textcat_config",
|
"name,train_data,textcat_config",
|
||||||
[
|
[
|
||||||
|
# BOW V1
|
||||||
|
("textcat_multilabel", TRAIN_DATA_MULTI_LABEL, {"@architectures": "spacy.TextCatBOW.v1", "exclusive_classes": False, "ngram_size": 1, "no_output_layer": False}),
|
||||||
|
("textcat", TRAIN_DATA_SINGLE_LABEL, {"@architectures": "spacy.TextCatBOW.v1", "exclusive_classes": True, "ngram_size": 4, "no_output_layer": False}),
|
||||||
|
# ENSEMBLE V1
|
||||||
|
("textcat_multilabel", TRAIN_DATA_MULTI_LABEL, {"@architectures": "spacy.TextCatEnsemble.v1", "exclusive_classes": False, "pretrained_vectors": None, "width": 64, "embed_size": 2000, "conv_depth": 2, "window_size": 1, "ngram_size": 1, "dropout": None}),
|
||||||
|
("textcat", TRAIN_DATA_SINGLE_LABEL, {"@architectures": "spacy.TextCatEnsemble.v1", "exclusive_classes": False, "pretrained_vectors": None, "width": 64, "embed_size": 2000, "conv_depth": 2, "window_size": 1, "ngram_size": 1, "dropout": None}),
|
||||||
|
# CNN V1
|
||||||
|
("textcat", TRAIN_DATA_SINGLE_LABEL, {"@architectures": "spacy.TextCatCNN.v1", "tok2vec": DEFAULT_TOK2VEC_MODEL, "exclusive_classes": True}),
|
||||||
|
("textcat_multilabel", TRAIN_DATA_MULTI_LABEL, {"@architectures": "spacy.TextCatCNN.v1", "tok2vec": DEFAULT_TOK2VEC_MODEL, "exclusive_classes": False}),
|
||||||
|
# BOW V2
|
||||||
("textcat_multilabel", TRAIN_DATA_MULTI_LABEL, {"@architectures": "spacy.TextCatBOW.v2", "exclusive_classes": False, "ngram_size": 1, "no_output_layer": False}),
|
("textcat_multilabel", TRAIN_DATA_MULTI_LABEL, {"@architectures": "spacy.TextCatBOW.v2", "exclusive_classes": False, "ngram_size": 1, "no_output_layer": False}),
|
||||||
("textcat", TRAIN_DATA_SINGLE_LABEL, {"@architectures": "spacy.TextCatBOW.v2", "exclusive_classes": True, "ngram_size": 4, "no_output_layer": False}),
|
("textcat", TRAIN_DATA_SINGLE_LABEL, {"@architectures": "spacy.TextCatBOW.v2", "exclusive_classes": True, "ngram_size": 4, "no_output_layer": False}),
|
||||||
("textcat_multilabel", TRAIN_DATA_MULTI_LABEL, {"@architectures": "spacy.TextCatBOW.v2", "exclusive_classes": False, "ngram_size": 3, "no_output_layer": True}),
|
("textcat_multilabel", TRAIN_DATA_MULTI_LABEL, {"@architectures": "spacy.TextCatBOW.v2", "exclusive_classes": False, "ngram_size": 3, "no_output_layer": True}),
|
||||||
("textcat", TRAIN_DATA_SINGLE_LABEL, {"@architectures": "spacy.TextCatBOW.v2", "exclusive_classes": True, "ngram_size": 2, "no_output_layer": True}),
|
("textcat", TRAIN_DATA_SINGLE_LABEL, {"@architectures": "spacy.TextCatBOW.v2", "exclusive_classes": True, "ngram_size": 2, "no_output_layer": True}),
|
||||||
|
# ENSEMBLE V2
|
||||||
("textcat_multilabel", TRAIN_DATA_MULTI_LABEL, {"@architectures": "spacy.TextCatEnsemble.v2", "tok2vec": DEFAULT_TOK2VEC_MODEL, "linear_model": {"@architectures": "spacy.TextCatBOW.v2", "exclusive_classes": False, "ngram_size": 1, "no_output_layer": False}}),
|
("textcat_multilabel", TRAIN_DATA_MULTI_LABEL, {"@architectures": "spacy.TextCatEnsemble.v2", "tok2vec": DEFAULT_TOK2VEC_MODEL, "linear_model": {"@architectures": "spacy.TextCatBOW.v2", "exclusive_classes": False, "ngram_size": 1, "no_output_layer": False}}),
|
||||||
("textcat", TRAIN_DATA_SINGLE_LABEL, {"@architectures": "spacy.TextCatEnsemble.v2", "tok2vec": DEFAULT_TOK2VEC_MODEL, "linear_model": {"@architectures": "spacy.TextCatBOW.v2", "exclusive_classes": True, "ngram_size": 5, "no_output_layer": False}}),
|
("textcat", TRAIN_DATA_SINGLE_LABEL, {"@architectures": "spacy.TextCatEnsemble.v2", "tok2vec": DEFAULT_TOK2VEC_MODEL, "linear_model": {"@architectures": "spacy.TextCatBOW.v2", "exclusive_classes": True, "ngram_size": 5, "no_output_layer": False}}),
|
||||||
|
# CNN V2
|
||||||
("textcat", TRAIN_DATA_SINGLE_LABEL, {"@architectures": "spacy.TextCatCNN.v2", "tok2vec": DEFAULT_TOK2VEC_MODEL, "exclusive_classes": True}),
|
("textcat", TRAIN_DATA_SINGLE_LABEL, {"@architectures": "spacy.TextCatCNN.v2", "tok2vec": DEFAULT_TOK2VEC_MODEL, "exclusive_classes": True}),
|
||||||
("textcat_multilabel", TRAIN_DATA_MULTI_LABEL, {"@architectures": "spacy.TextCatCNN.v2", "tok2vec": DEFAULT_TOK2VEC_MODEL, "exclusive_classes": False}),
|
("textcat_multilabel", TRAIN_DATA_MULTI_LABEL, {"@architectures": "spacy.TextCatCNN.v2", "tok2vec": DEFAULT_TOK2VEC_MODEL, "exclusive_classes": False}),
|
||||||
],
|
],
|
||||||
|
|
|
@ -1,13 +1,13 @@
|
||||||
import pytest
|
import pytest
|
||||||
from spacy.ml.models.tok2vec import build_Tok2Vec_model
|
from spacy.ml.models.tok2vec import build_Tok2Vec_model
|
||||||
from spacy.ml.models.tok2vec import MultiHashEmbed, CharacterEmbed
|
from spacy.ml.models.tok2vec import MultiHashEmbed, MaxoutWindowEncoder
|
||||||
from spacy.ml.models.tok2vec import MishWindowEncoder, MaxoutWindowEncoder
|
|
||||||
from spacy.pipeline.tok2vec import Tok2Vec, Tok2VecListener
|
from spacy.pipeline.tok2vec import Tok2Vec, Tok2VecListener
|
||||||
from spacy.vocab import Vocab
|
from spacy.vocab import Vocab
|
||||||
from spacy.tokens import Doc
|
from spacy.tokens import Doc
|
||||||
from spacy.training import Example
|
from spacy.training import Example
|
||||||
from spacy import util
|
from spacy import util
|
||||||
from spacy.lang.en import English
|
from spacy.lang.en import English
|
||||||
|
from spacy.util import registry
|
||||||
from thinc.api import Config, get_current_ops
|
from thinc.api import Config, get_current_ops
|
||||||
from numpy.testing import assert_array_equal
|
from numpy.testing import assert_array_equal
|
||||||
|
|
||||||
|
@ -55,24 +55,41 @@ def test_tok2vec_batch_sizes(batch_size, width, embed_size):
|
||||||
assert doc_vec.shape == (len(doc), width)
|
assert doc_vec.shape == (len(doc), width)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.slow
|
||||||
|
@pytest.mark.parametrize("width", [8])
|
||||||
@pytest.mark.parametrize(
|
@pytest.mark.parametrize(
|
||||||
"width,embed_arch,embed_config,encode_arch,encode_config",
|
"embed_arch,embed_config",
|
||||||
# fmt: off
|
# fmt: off
|
||||||
[
|
[
|
||||||
(8, MultiHashEmbed, {"rows": [100, 100], "attrs": ["SHAPE", "LOWER"], "include_static_vectors": False}, MaxoutWindowEncoder, {"window_size": 1, "maxout_pieces": 3, "depth": 2}),
|
("spacy.MultiHashEmbed.v1", {"rows": [100, 100], "attrs": ["SHAPE", "LOWER"], "include_static_vectors": False}),
|
||||||
(8, MultiHashEmbed, {"rows": [100, 20], "attrs": ["ORTH", "PREFIX"], "include_static_vectors": False}, MishWindowEncoder, {"window_size": 1, "depth": 6}),
|
("spacy.MultiHashEmbed.v1", {"rows": [100, 20], "attrs": ["ORTH", "PREFIX"], "include_static_vectors": False}),
|
||||||
(8, CharacterEmbed, {"rows": 100, "nM": 64, "nC": 8, "include_static_vectors": False}, MaxoutWindowEncoder, {"window_size": 1, "maxout_pieces": 3, "depth": 3}),
|
("spacy.CharacterEmbed.v1", {"rows": 100, "nM": 64, "nC": 8, "include_static_vectors": False}),
|
||||||
(8, CharacterEmbed, {"rows": 100, "nM": 16, "nC": 2, "include_static_vectors": False}, MishWindowEncoder, {"window_size": 1, "depth": 3}),
|
("spacy.CharacterEmbed.v1", {"rows": 100, "nM": 16, "nC": 2, "include_static_vectors": False}),
|
||||||
],
|
],
|
||||||
# fmt: on
|
# fmt: on
|
||||||
)
|
)
|
||||||
def test_tok2vec_configs(width, embed_arch, embed_config, encode_arch, encode_config):
|
@pytest.mark.parametrize(
|
||||||
|
"tok2vec_arch,encode_arch,encode_config",
|
||||||
|
# fmt: off
|
||||||
|
[
|
||||||
|
("spacy.Tok2Vec.v1", "spacy.MaxoutWindowEncoder.v1", {"window_size": 1, "maxout_pieces": 3, "depth": 2}),
|
||||||
|
("spacy.Tok2Vec.v2", "spacy.MaxoutWindowEncoder.v2", {"window_size": 1, "maxout_pieces": 3, "depth": 2}),
|
||||||
|
("spacy.Tok2Vec.v1", "spacy.MishWindowEncoder.v1", {"window_size": 1, "depth": 6}),
|
||||||
|
("spacy.Tok2Vec.v2", "spacy.MishWindowEncoder.v2", {"window_size": 1, "depth": 6}),
|
||||||
|
],
|
||||||
|
# fmt: on
|
||||||
|
)
|
||||||
|
def test_tok2vec_configs(
|
||||||
|
width, tok2vec_arch, embed_arch, embed_config, encode_arch, encode_config
|
||||||
|
):
|
||||||
|
embed = registry.get("architectures", embed_arch)
|
||||||
|
encode = registry.get("architectures", encode_arch)
|
||||||
|
tok2vec_model = registry.get("architectures", tok2vec_arch)
|
||||||
|
|
||||||
embed_config["width"] = width
|
embed_config["width"] = width
|
||||||
encode_config["width"] = width
|
encode_config["width"] = width
|
||||||
docs = get_batch(3)
|
docs = get_batch(3)
|
||||||
tok2vec = build_Tok2Vec_model(
|
tok2vec = tok2vec_model(embed(**embed_config), encode(**encode_config))
|
||||||
embed_arch(**embed_config), encode_arch(**encode_config)
|
|
||||||
)
|
|
||||||
tok2vec.initialize(docs)
|
tok2vec.initialize(docs)
|
||||||
vectors, backprop = tok2vec.begin_update(docs)
|
vectors, backprop = tok2vec.begin_update(docs)
|
||||||
assert len(vectors) == len(docs)
|
assert len(vectors) == len(docs)
|
||||||
|
|
161
spacy/tests/serialize/test_serialize_span_groups.py
Normal file
161
spacy/tests/serialize/test_serialize_span_groups.py
Normal file
|
@ -0,0 +1,161 @@
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from spacy.tokens import Span, SpanGroup
|
||||||
|
from spacy.tokens._dict_proxies import SpanGroups
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.issue(10685)
|
||||||
|
def test_issue10685(en_tokenizer):
|
||||||
|
"""Test `SpanGroups` de/serialization"""
|
||||||
|
# Start with a Doc with no SpanGroups
|
||||||
|
doc = en_tokenizer("Will it blend?")
|
||||||
|
|
||||||
|
# Test empty `SpanGroups` de/serialization:
|
||||||
|
assert len(doc.spans) == 0
|
||||||
|
doc.spans.from_bytes(doc.spans.to_bytes())
|
||||||
|
assert len(doc.spans) == 0
|
||||||
|
|
||||||
|
# Test non-empty `SpanGroups` de/serialization:
|
||||||
|
doc.spans["test"] = SpanGroup(doc, name="test", spans=[doc[0:1]])
|
||||||
|
doc.spans["test2"] = SpanGroup(doc, name="test", spans=[doc[1:2]])
|
||||||
|
|
||||||
|
def assert_spangroups():
|
||||||
|
assert len(doc.spans) == 2
|
||||||
|
assert doc.spans["test"].name == "test"
|
||||||
|
assert doc.spans["test2"].name == "test"
|
||||||
|
assert list(doc.spans["test"]) == [doc[0:1]]
|
||||||
|
assert list(doc.spans["test2"]) == [doc[1:2]]
|
||||||
|
|
||||||
|
# Sanity check the currently-expected behavior
|
||||||
|
assert_spangroups()
|
||||||
|
|
||||||
|
# Now test serialization/deserialization:
|
||||||
|
doc.spans.from_bytes(doc.spans.to_bytes())
|
||||||
|
|
||||||
|
assert_spangroups()
|
||||||
|
|
||||||
|
|
||||||
|
def test_span_groups_serialization_mismatches(en_tokenizer):
|
||||||
|
"""Test the serialization of multiple mismatching `SpanGroups` keys and `SpanGroup.name`s"""
|
||||||
|
doc = en_tokenizer("How now, brown cow?")
|
||||||
|
# Some variety:
|
||||||
|
# 1 SpanGroup where its name matches its key
|
||||||
|
# 2 SpanGroups that have the same name--which is not a key
|
||||||
|
# 2 SpanGroups that have the same name--which is a key
|
||||||
|
# 1 SpanGroup that is a value for 2 different keys (where its name is a key)
|
||||||
|
# 1 SpanGroup that is a value for 2 different keys (where its name is not a key)
|
||||||
|
groups = doc.spans
|
||||||
|
groups["key1"] = SpanGroup(doc, name="key1", spans=[doc[0:1], doc[1:2]])
|
||||||
|
groups["key2"] = SpanGroup(doc, name="too", spans=[doc[3:4], doc[4:5]])
|
||||||
|
groups["key3"] = SpanGroup(doc, name="too", spans=[doc[1:2], doc[0:1]])
|
||||||
|
groups["key4"] = SpanGroup(doc, name="key4", spans=[doc[0:1]])
|
||||||
|
groups["key5"] = SpanGroup(doc, name="key4", spans=[doc[0:1]])
|
||||||
|
sg6 = SpanGroup(doc, name="key6", spans=[doc[0:1]])
|
||||||
|
groups["key6"] = sg6
|
||||||
|
groups["key7"] = sg6
|
||||||
|
sg8 = SpanGroup(doc, name="also", spans=[doc[1:2]])
|
||||||
|
groups["key8"] = sg8
|
||||||
|
groups["key9"] = sg8
|
||||||
|
|
||||||
|
regroups = SpanGroups(doc).from_bytes(groups.to_bytes())
|
||||||
|
|
||||||
|
# Assert regroups == groups
|
||||||
|
assert regroups.keys() == groups.keys()
|
||||||
|
for key, regroup in regroups.items():
|
||||||
|
# Assert regroup == groups[key]
|
||||||
|
assert regroup.name == groups[key].name
|
||||||
|
assert list(regroup) == list(groups[key])
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"spans_bytes,doc_text,expected_spangroups,expected_warning",
|
||||||
|
# The bytestrings below were generated from an earlier version of spaCy
|
||||||
|
# that serialized `SpanGroups` as a list of SpanGroup bytes (via SpanGroups.to_bytes).
|
||||||
|
# Comments preceding the bytestrings indicate from what Doc they were created.
|
||||||
|
[
|
||||||
|
# Empty SpanGroups:
|
||||||
|
(b"\x90", "", {}, False),
|
||||||
|
# doc = nlp("Will it blend?")
|
||||||
|
# doc.spans['test'] = SpanGroup(doc, name='test', spans=[doc[0:1]])
|
||||||
|
(
|
||||||
|
b"\x91\xc4C\x83\xa4name\xa4test\xa5attrs\x80\xa5spans\x91\xc4(\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x04",
|
||||||
|
"Will it blend?",
|
||||||
|
{"test": {"name": "test", "spans": [(0, 1)]}},
|
||||||
|
False,
|
||||||
|
),
|
||||||
|
# doc = nlp("Will it blend?")
|
||||||
|
# doc.spans['test'] = SpanGroup(doc, name='test', spans=[doc[0:1]])
|
||||||
|
# doc.spans['test2'] = SpanGroup(doc, name='test', spans=[doc[1:2]])
|
||||||
|
(
|
||||||
|
b"\x92\xc4C\x83\xa4name\xa4test\xa5attrs\x80\xa5spans\x91\xc4(\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x04\xc4C\x83\xa4name\xa4test\xa5attrs\x80\xa5spans\x91\xc4(\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x02\x00\x00\x00\x05\x00\x00\x00\x07",
|
||||||
|
"Will it blend?",
|
||||||
|
# We expect only 1 SpanGroup to be in doc.spans in this example
|
||||||
|
# because there are 2 `SpanGroup`s that have the same .name. See #10685.
|
||||||
|
{"test": {"name": "test", "spans": [(1, 2)]}},
|
||||||
|
True,
|
||||||
|
),
|
||||||
|
# doc = nlp('How now, brown cow?')
|
||||||
|
# doc.spans['key1'] = SpanGroup(doc, name='key1', spans=[doc[0:1], doc[1:2]])
|
||||||
|
# doc.spans['key2'] = SpanGroup(doc, name='too', spans=[doc[3:4], doc[4:5]])
|
||||||
|
# doc.spans['key3'] = SpanGroup(doc, name='too', spans=[doc[1:2], doc[0:1]])
|
||||||
|
# doc.spans['key4'] = SpanGroup(doc, name='key4', spans=[doc[0:1]])
|
||||||
|
# doc.spans['key5'] = SpanGroup(doc, name='key4', spans=[doc[0:1]])
|
||||||
|
(
|
||||||
|
b"\x95\xc4m\x83\xa4name\xa4key1\xa5attrs\x80\xa5spans\x92\xc4(\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x03\xc4(\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x02\x00\x00\x00\x04\x00\x00\x00\x07\xc4l\x83\xa4name\xa3too\xa5attrs\x80\xa5spans\x92\xc4(\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x03\x00\x00\x00\x04\x00\x00\x00\t\x00\x00\x00\x0e\xc4(\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x04\x00\x00\x00\x05\x00\x00\x00\x0f\x00\x00\x00\x12\xc4l\x83\xa4name\xa3too\xa5attrs\x80\xa5spans\x92\xc4(\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x02\x00\x00\x00\x04\x00\x00\x00\x07\xc4(\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x03\xc4C\x83\xa4name\xa4key4\xa5attrs\x80\xa5spans\x91\xc4(\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x03\xc4C\x83\xa4name\xa4key4\xa5attrs\x80\xa5spans\x91\xc4(\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x03",
|
||||||
|
"How now, brown cow?",
|
||||||
|
{
|
||||||
|
"key1": {"name": "key1", "spans": [(0, 1), (1, 2)]},
|
||||||
|
"too": {"name": "too", "spans": [(1, 2), (0, 1)]},
|
||||||
|
"key4": {"name": "key4", "spans": [(0, 1)]},
|
||||||
|
},
|
||||||
|
True,
|
||||||
|
),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_deserialize_span_groups_compat(
|
||||||
|
en_tokenizer, spans_bytes, doc_text, expected_spangroups, expected_warning
|
||||||
|
):
|
||||||
|
"""Test backwards-compatibility of `SpanGroups` deserialization.
|
||||||
|
This uses serializations (bytes) from a prior version of spaCy (before 3.3.1).
|
||||||
|
|
||||||
|
spans_bytes (bytes): Serialized `SpanGroups` object.
|
||||||
|
doc_text (str): Doc text.
|
||||||
|
expected_spangroups (dict):
|
||||||
|
Dict mapping every expected (after deserialization) `SpanGroups` key
|
||||||
|
to a SpanGroup's "args", where a SpanGroup's args are given as a dict:
|
||||||
|
{"name": span_group.name,
|
||||||
|
"spans": [(span0.start, span0.end), ...]}
|
||||||
|
expected_warning (bool): Whether a warning is to be expected from .from_bytes()
|
||||||
|
--i.e. if more than 1 SpanGroup has the same .name within the `SpanGroups`.
|
||||||
|
"""
|
||||||
|
doc = en_tokenizer(doc_text)
|
||||||
|
|
||||||
|
if expected_warning:
|
||||||
|
with pytest.warns(UserWarning):
|
||||||
|
doc.spans.from_bytes(spans_bytes)
|
||||||
|
else:
|
||||||
|
# TODO: explicitly check for lack of a warning
|
||||||
|
doc.spans.from_bytes(spans_bytes)
|
||||||
|
|
||||||
|
assert doc.spans.keys() == expected_spangroups.keys()
|
||||||
|
for name, spangroup_args in expected_spangroups.items():
|
||||||
|
assert doc.spans[name].name == spangroup_args["name"]
|
||||||
|
spans = [Span(doc, start, end) for start, end in spangroup_args["spans"]]
|
||||||
|
assert list(doc.spans[name]) == spans
|
||||||
|
|
||||||
|
|
||||||
|
def test_span_groups_serialization(en_tokenizer):
|
||||||
|
doc = en_tokenizer("0 1 2 3 4 5 6")
|
||||||
|
span_groups = SpanGroups(doc)
|
||||||
|
spans = [doc[0:2], doc[1:3]]
|
||||||
|
sg1 = SpanGroup(doc, spans=spans)
|
||||||
|
span_groups["key1"] = sg1
|
||||||
|
span_groups["key2"] = sg1
|
||||||
|
span_groups["key3"] = []
|
||||||
|
reloaded_span_groups = SpanGroups(doc).from_bytes(span_groups.to_bytes())
|
||||||
|
assert span_groups.keys() == reloaded_span_groups.keys()
|
||||||
|
for key, value in span_groups.items():
|
||||||
|
assert all(
|
||||||
|
span == reloaded_span
|
||||||
|
for span, reloaded_span in zip(span_groups[key], reloaded_span_groups[key])
|
||||||
|
)
|
|
@ -1,4 +1,7 @@
|
||||||
import os
|
import os
|
||||||
|
import math
|
||||||
|
from random import sample
|
||||||
|
from typing import Counter
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
import srsly
|
import srsly
|
||||||
|
@ -14,6 +17,10 @@ from spacy.cli._util import substitute_project_variables
|
||||||
from spacy.cli._util import validate_project_commands
|
from spacy.cli._util import validate_project_commands
|
||||||
from spacy.cli.debug_data import _compile_gold, _get_labels_from_model
|
from spacy.cli.debug_data import _compile_gold, _get_labels_from_model
|
||||||
from spacy.cli.debug_data import _get_labels_from_spancat
|
from spacy.cli.debug_data import _get_labels_from_spancat
|
||||||
|
from spacy.cli.debug_data import _get_distribution, _get_kl_divergence
|
||||||
|
from spacy.cli.debug_data import _get_span_characteristics
|
||||||
|
from spacy.cli.debug_data import _print_span_characteristics
|
||||||
|
from spacy.cli.debug_data import _get_spans_length_freq_dist
|
||||||
from spacy.cli.download import get_compatibility, get_version
|
from spacy.cli.download import get_compatibility, get_version
|
||||||
from spacy.cli.init_config import RECOMMENDATIONS, init_config, fill_config
|
from spacy.cli.init_config import RECOMMENDATIONS, init_config, fill_config
|
||||||
from spacy.cli.package import get_third_party_dependencies
|
from spacy.cli.package import get_third_party_dependencies
|
||||||
|
@ -24,6 +31,7 @@ from spacy.lang.nl import Dutch
|
||||||
from spacy.language import Language
|
from spacy.language import Language
|
||||||
from spacy.schemas import ProjectConfigSchema, RecommendationSchema, validate
|
from spacy.schemas import ProjectConfigSchema, RecommendationSchema, validate
|
||||||
from spacy.tokens import Doc
|
from spacy.tokens import Doc
|
||||||
|
from spacy.tokens.span import Span
|
||||||
from spacy.training import Example, docs_to_json, offsets_to_biluo_tags
|
from spacy.training import Example, docs_to_json, offsets_to_biluo_tags
|
||||||
from spacy.training.converters import conll_ner_to_docs, conllu_to_docs
|
from spacy.training.converters import conll_ner_to_docs, conllu_to_docs
|
||||||
from spacy.training.converters import iob_to_docs
|
from spacy.training.converters import iob_to_docs
|
||||||
|
@ -341,6 +349,7 @@ def test_project_config_validation_full():
|
||||||
"assets": [
|
"assets": [
|
||||||
{
|
{
|
||||||
"dest": "x",
|
"dest": "x",
|
||||||
|
"extra": True,
|
||||||
"url": "https://example.com",
|
"url": "https://example.com",
|
||||||
"checksum": "63373dd656daa1fd3043ce166a59474c",
|
"checksum": "63373dd656daa1fd3043ce166a59474c",
|
||||||
},
|
},
|
||||||
|
@ -352,6 +361,12 @@ def test_project_config_validation_full():
|
||||||
"path": "y",
|
"path": "y",
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
"dest": "z",
|
||||||
|
"extra": False,
|
||||||
|
"url": "https://example.com",
|
||||||
|
"checksum": "63373dd656daa1fd3043ce166a59474c",
|
||||||
|
},
|
||||||
],
|
],
|
||||||
"commands": [
|
"commands": [
|
||||||
{
|
{
|
||||||
|
@ -733,3 +748,110 @@ def test_debug_data_compile_gold():
|
||||||
eg = Example(pred, ref)
|
eg = Example(pred, ref)
|
||||||
data = _compile_gold([eg], ["ner"], nlp, True)
|
data = _compile_gold([eg], ["ner"], nlp, True)
|
||||||
assert data["boundary_cross_ents"] == 1
|
assert data["boundary_cross_ents"] == 1
|
||||||
|
|
||||||
|
|
||||||
|
def test_debug_data_compile_gold_for_spans():
|
||||||
|
nlp = English()
|
||||||
|
spans_key = "sc"
|
||||||
|
|
||||||
|
pred = Doc(nlp.vocab, words=["Welcome", "to", "the", "Bank", "of", "China", "."])
|
||||||
|
pred.spans[spans_key] = [Span(pred, 3, 6, "ORG"), Span(pred, 5, 6, "GPE")]
|
||||||
|
ref = Doc(nlp.vocab, words=["Welcome", "to", "the", "Bank", "of", "China", "."])
|
||||||
|
ref.spans[spans_key] = [Span(ref, 3, 6, "ORG"), Span(ref, 5, 6, "GPE")]
|
||||||
|
eg = Example(pred, ref)
|
||||||
|
|
||||||
|
data = _compile_gold([eg], ["spancat"], nlp, True)
|
||||||
|
|
||||||
|
assert data["spancat"][spans_key] == Counter({"ORG": 1, "GPE": 1})
|
||||||
|
assert data["spans_length"][spans_key] == {"ORG": [3], "GPE": [1]}
|
||||||
|
assert data["spans_per_type"][spans_key] == {
|
||||||
|
"ORG": [Span(ref, 3, 6, "ORG")],
|
||||||
|
"GPE": [Span(ref, 5, 6, "GPE")],
|
||||||
|
}
|
||||||
|
assert data["sb_per_type"][spans_key] == {
|
||||||
|
"ORG": {"start": [ref[2:3]], "end": [ref[6:7]]},
|
||||||
|
"GPE": {"start": [ref[4:5]], "end": [ref[6:7]]},
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def test_frequency_distribution_is_correct():
|
||||||
|
nlp = English()
|
||||||
|
docs = [
|
||||||
|
Doc(nlp.vocab, words=["Bank", "of", "China"]),
|
||||||
|
Doc(nlp.vocab, words=["China"]),
|
||||||
|
]
|
||||||
|
|
||||||
|
expected = Counter({"china": 0.5, "bank": 0.25, "of": 0.25})
|
||||||
|
freq_distribution = _get_distribution(docs, normalize=True)
|
||||||
|
assert freq_distribution == expected
|
||||||
|
|
||||||
|
|
||||||
|
def test_kl_divergence_computation_is_correct():
|
||||||
|
p = Counter({"a": 0.5, "b": 0.25})
|
||||||
|
q = Counter({"a": 0.25, "b": 0.50, "c": 0.15, "d": 0.10})
|
||||||
|
result = _get_kl_divergence(p, q)
|
||||||
|
expected = 0.1733
|
||||||
|
assert math.isclose(result, expected, rel_tol=1e-3)
|
||||||
|
|
||||||
|
|
||||||
|
def test_get_span_characteristics_return_value():
|
||||||
|
nlp = English()
|
||||||
|
spans_key = "sc"
|
||||||
|
|
||||||
|
pred = Doc(nlp.vocab, words=["Welcome", "to", "the", "Bank", "of", "China", "."])
|
||||||
|
pred.spans[spans_key] = [Span(pred, 3, 6, "ORG"), Span(pred, 5, 6, "GPE")]
|
||||||
|
ref = Doc(nlp.vocab, words=["Welcome", "to", "the", "Bank", "of", "China", "."])
|
||||||
|
ref.spans[spans_key] = [Span(ref, 3, 6, "ORG"), Span(ref, 5, 6, "GPE")]
|
||||||
|
eg = Example(pred, ref)
|
||||||
|
|
||||||
|
examples = [eg]
|
||||||
|
data = _compile_gold(examples, ["spancat"], nlp, True)
|
||||||
|
span_characteristics = _get_span_characteristics(
|
||||||
|
examples=examples, compiled_gold=data, spans_key=spans_key
|
||||||
|
)
|
||||||
|
|
||||||
|
assert {"sd", "bd", "lengths"}.issubset(span_characteristics.keys())
|
||||||
|
assert span_characteristics["min_length"] == 1
|
||||||
|
assert span_characteristics["max_length"] == 3
|
||||||
|
|
||||||
|
|
||||||
|
def test_ensure_print_span_characteristics_wont_fail():
|
||||||
|
"""Test if interface between two methods aren't destroyed if refactored"""
|
||||||
|
nlp = English()
|
||||||
|
spans_key = "sc"
|
||||||
|
|
||||||
|
pred = Doc(nlp.vocab, words=["Welcome", "to", "the", "Bank", "of", "China", "."])
|
||||||
|
pred.spans[spans_key] = [Span(pred, 3, 6, "ORG"), Span(pred, 5, 6, "GPE")]
|
||||||
|
ref = Doc(nlp.vocab, words=["Welcome", "to", "the", "Bank", "of", "China", "."])
|
||||||
|
ref.spans[spans_key] = [Span(ref, 3, 6, "ORG"), Span(ref, 5, 6, "GPE")]
|
||||||
|
eg = Example(pred, ref)
|
||||||
|
|
||||||
|
examples = [eg]
|
||||||
|
data = _compile_gold(examples, ["spancat"], nlp, True)
|
||||||
|
span_characteristics = _get_span_characteristics(
|
||||||
|
examples=examples, compiled_gold=data, spans_key=spans_key
|
||||||
|
)
|
||||||
|
_print_span_characteristics(span_characteristics)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("threshold", [70, 80, 85, 90, 95])
|
||||||
|
def test_span_length_freq_dist_threshold_must_be_correct(threshold):
|
||||||
|
sample_span_lengths = {
|
||||||
|
"span_type_1": [1, 4, 4, 5],
|
||||||
|
"span_type_2": [5, 3, 3, 2],
|
||||||
|
"span_type_3": [3, 1, 3, 3],
|
||||||
|
}
|
||||||
|
span_freqs = _get_spans_length_freq_dist(sample_span_lengths, threshold)
|
||||||
|
assert sum(span_freqs.values()) >= threshold
|
||||||
|
|
||||||
|
|
||||||
|
def test_span_length_freq_dist_output_must_be_correct():
|
||||||
|
sample_span_lengths = {
|
||||||
|
"span_type_1": [1, 4, 4, 5],
|
||||||
|
"span_type_2": [5, 3, 3, 2],
|
||||||
|
"span_type_3": [3, 1, 3, 3],
|
||||||
|
}
|
||||||
|
threshold = 90
|
||||||
|
span_freqs = _get_spans_length_freq_dist(sample_span_lengths, threshold)
|
||||||
|
assert sum(span_freqs.values()) >= threshold
|
||||||
|
assert list(span_freqs.keys()) == [3, 1, 4, 5, 2]
|
||||||
|
|
|
@ -1,7 +1,13 @@
|
||||||
import pytest
|
|
||||||
import re
|
import re
|
||||||
from spacy.util import get_lang_class
|
import string
|
||||||
|
|
||||||
|
import hypothesis
|
||||||
|
import hypothesis.strategies
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
import spacy
|
||||||
from spacy.tokenizer import Tokenizer
|
from spacy.tokenizer import Tokenizer
|
||||||
|
from spacy.util import get_lang_class
|
||||||
|
|
||||||
# Only include languages with no external dependencies
|
# Only include languages with no external dependencies
|
||||||
# "is" seems to confuse importlib, so we're also excluding it for now
|
# "is" seems to confuse importlib, so we're also excluding it for now
|
||||||
|
@ -77,3 +83,46 @@ def test_tokenizer_explain_special_matcher(en_vocab):
|
||||||
tokens = [t.text for t in tokenizer("a/a.")]
|
tokens = [t.text for t in tokenizer("a/a.")]
|
||||||
explain_tokens = [t[1] for t in tokenizer.explain("a/a.")]
|
explain_tokens = [t[1] for t in tokenizer.explain("a/a.")]
|
||||||
assert tokens == explain_tokens
|
assert tokens == explain_tokens
|
||||||
|
|
||||||
|
|
||||||
|
@hypothesis.strategies.composite
|
||||||
|
def sentence_strategy(draw: hypothesis.strategies.DrawFn, max_n_words: int = 4) -> str:
|
||||||
|
"""
|
||||||
|
Composite strategy for fuzzily generating sentence with varying interpunctation.
|
||||||
|
|
||||||
|
draw (hypothesis.strategies.DrawFn): Protocol for drawing function allowing to fuzzily pick from hypothesis'
|
||||||
|
strategies.
|
||||||
|
max_n_words (int): Max. number of words in generated sentence.
|
||||||
|
RETURNS (str): Fuzzily generated sentence.
|
||||||
|
"""
|
||||||
|
|
||||||
|
punctuation_and_space_regex = "|".join(
|
||||||
|
[*[re.escape(p) for p in string.punctuation], r"\s"]
|
||||||
|
)
|
||||||
|
sentence = [
|
||||||
|
[
|
||||||
|
draw(hypothesis.strategies.text(min_size=1)),
|
||||||
|
draw(hypothesis.strategies.from_regex(punctuation_and_space_regex)),
|
||||||
|
]
|
||||||
|
for _ in range(
|
||||||
|
draw(hypothesis.strategies.integers(min_value=2, max_value=max_n_words))
|
||||||
|
)
|
||||||
|
]
|
||||||
|
|
||||||
|
return " ".join([token for token_pair in sentence for token in token_pair])
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.xfail
|
||||||
|
@pytest.mark.parametrize("lang", LANGUAGES)
|
||||||
|
@hypothesis.given(sentence=sentence_strategy())
|
||||||
|
def test_tokenizer_explain_fuzzy(lang: str, sentence: str) -> None:
|
||||||
|
"""
|
||||||
|
Tests whether output of tokenizer.explain() matches tokenizer output. Input generated by hypothesis.
|
||||||
|
lang (str): Language to test.
|
||||||
|
text (str): Fuzzily generated sentence to tokenize.
|
||||||
|
"""
|
||||||
|
|
||||||
|
tokenizer: Tokenizer = spacy.blank(lang).tokenizer
|
||||||
|
tokens = [t.text for t in tokenizer(sentence) if not t.is_space]
|
||||||
|
debug_tokens = [t[1] for t in tokenizer.explain(sentence)]
|
||||||
|
assert tokens == debug_tokens, f"{tokens}, {debug_tokens}, {sentence}"
|
||||||
|
|
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue
Block a user