diff --git a/.github/azure-steps.yml b/.github/azure-steps.yml index 80c88b0b8..aae08c7f3 100644 --- a/.github/azure-steps.yml +++ b/.github/azure-steps.yml @@ -27,7 +27,6 @@ steps: - script: python -m mypy spacy displayName: 'Run mypy' - condition: ne(variables['python_version'], '3.10') - task: DeleteFiles@1 inputs: @@ -41,7 +40,7 @@ steps: - bash: | ${{ parameters.prefix }} SDIST=$(python -c "import os;print(os.listdir('./dist')[-1])" 2>&1) - ${{ parameters.prefix }} python -m pip install dist/$SDIST + ${{ parameters.prefix }} SPACY_NUM_BUILD_JOBS=2 python -m pip install dist/$SDIST displayName: "Install from sdist" - script: | @@ -111,7 +110,7 @@ steps: condition: eq(variables['python_version'], '3.8') - script: | - ${{ parameters.prefix }} python -m pip install thinc-apple-ops + ${{ parameters.prefix }} python -m pip install --pre thinc-apple-ops ${{ parameters.prefix }} python -m pytest --pyargs spacy displayName: "Run CPU tests with thinc-apple-ops" - condition: and(startsWith(variables['imageName'], 'macos'), eq(variables['python.version'], '3.9')) + condition: and(startsWith(variables['imageName'], 'macos'), eq(variables['python.version'], '3.10')) diff --git a/.github/contributors/Lucaterre.md b/.github/contributors/Lucaterre.md new file mode 100644 index 000000000..5da763b22 --- /dev/null +++ b/.github/contributors/Lucaterre.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- |---------------| +| Name | Lucas Terriel | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 2022-06-20 | +| GitHub username | Lucaterre | +| Website (optional) | | \ No newline at end of file diff --git a/.github/spacy_universe_alert.py b/.github/spacy_universe_alert.py new file mode 100644 index 000000000..99ffabe93 --- /dev/null +++ b/.github/spacy_universe_alert.py @@ -0,0 +1,67 @@ +import os +import sys +import json +from datetime import datetime + +from slack_sdk.web.client import WebClient + +CHANNEL = "#alerts-universe" +SLACK_TOKEN = os.environ.get("SLACK_BOT_TOKEN", "ENV VAR not available!") +DATETIME_FORMAT = "%Y-%m-%dT%H:%M:%SZ" + +client = WebClient(SLACK_TOKEN) +github_context = json.loads(sys.argv[1]) + +event = github_context['event'] +pr_title = event['pull_request']["title"] +pr_link = event['pull_request']["patch_url"].replace(".patch", "") +pr_author_url = event['sender']["html_url"] +pr_author_name = pr_author_url.rsplit('/')[-1] +pr_created_at_dt = datetime.strptime( + event['pull_request']["created_at"], + DATETIME_FORMAT +) +pr_created_at = pr_created_at_dt.strftime("%c") +pr_updated_at_dt = datetime.strptime( + event['pull_request']["updated_at"], + DATETIME_FORMAT +) +pr_updated_at = pr_updated_at_dt.strftime("%c") + +blocks = [ + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": "📣 New spaCy Universe Project Alert ✨" + } + }, + { + "type": "section", + "fields": [ + { + "type": "mrkdwn", + "text": f"*Pull Request:*\n<{pr_link}|{pr_title}>" + }, + { + "type": "mrkdwn", + "text": f"*Author:*\n<{pr_author_url}|{pr_author_name}>" + }, + { + "type": "mrkdwn", + "text": f"*Created at:*\n {pr_created_at}" + }, + { + "type": "mrkdwn", + "text": f"*Last Updated:*\n {pr_updated_at}" + } + ] + } + ] + + +client.chat_postMessage( + channel=CHANNEL, + text="spaCy universe project PR alert", + blocks=blocks +) diff --git a/.github/workflows/explosionbot.yml b/.github/workflows/explosionbot.yml index e29ce8fe8..d585ecd9c 100644 --- a/.github/workflows/explosionbot.yml +++ b/.github/workflows/explosionbot.yml @@ -23,5 +23,5 @@ jobs: env: INPUT_TOKEN: ${{ secrets.EXPLOSIONBOT_TOKEN }} INPUT_BK_TOKEN: ${{ secrets.BUILDKITE_SECRET }} - ENABLED_COMMANDS: "test_gpu,test_slow" + ENABLED_COMMANDS: "test_gpu,test_slow,test_slow_gpu" ALLOWED_TEAMS: "spaCy" diff --git a/.github/workflows/gputests.yml b/.github/workflows/gputests.yml index bb7f51d29..66e0707e0 100644 --- a/.github/workflows/gputests.yml +++ b/.github/workflows/gputests.yml @@ -10,6 +10,7 @@ jobs: fail-fast: false matrix: branch: [master, v4] + if: github.repository_owner == 'explosion' runs-on: ubuntu-latest steps: - name: Trigger buildkite build diff --git a/.github/workflows/slowtests.yml b/.github/workflows/slowtests.yml index 1a99c751c..38ceb18c6 100644 --- a/.github/workflows/slowtests.yml +++ b/.github/workflows/slowtests.yml @@ -10,6 +10,7 @@ jobs: fail-fast: false matrix: branch: [master, v4] + if: github.repository_owner == 'explosion' runs-on: ubuntu-latest steps: - name: Checkout diff --git a/.github/workflows/spacy_universe_alert.yml b/.github/workflows/spacy_universe_alert.yml new file mode 100644 index 000000000..cbbf14c6e --- /dev/null +++ b/.github/workflows/spacy_universe_alert.yml @@ -0,0 +1,30 @@ +name: spaCy universe project alert + +on: + pull_request_target: + paths: + - "website/meta/universe.json" + +jobs: + build: + runs-on: ubuntu-latest + + steps: + - name: Dump GitHub context + env: + GITHUB_CONTEXT: ${{ toJson(github) }} + PR_NUMBER: ${{github.event.number}} + run: | + echo "$GITHUB_CONTEXT" + + - uses: actions/checkout@v1 + - uses: actions/setup-python@v1 + - name: Install Bernadette app dependency and send an alert + env: + SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} + GITHUB_CONTEXT: ${{ toJson(github) }} + CHANNEL: "#alerts-universe" + run: | + pip install slack-sdk==3.17.2 aiohttp==3.8.1 + echo "$CHANNEL" + python .github/spacy_universe_alert.py "$GITHUB_CONTEXT" diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ddd833be1..1f396bd71 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -271,7 +271,8 @@ except: # noqa: E722 ### Python conventions -All Python code must be written **compatible with Python 3.6+**. +All Python code must be written **compatible with Python 3.6+**. More detailed +code conventions can be found in the [developer docs](https://github.com/explosion/spaCy/blob/master/extra/DEVELOPER_DOCS/Code%20Conventions.md). #### I/O and handling paths diff --git a/MANIFEST.in b/MANIFEST.in index b7826e456..8ded6f808 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,4 +1,4 @@ -recursive-include spacy *.pyi *.pyx *.pxd *.txt *.cfg *.jinja *.toml +recursive-include spacy *.pyi *.pyx *.pxd *.txt *.cfg *.jinja *.toml *.hh include LICENSE include README.md include pyproject.toml diff --git a/README.md b/README.md index 05c912ffa..d9ef83e01 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ production-ready [**training system**](https://spacy.io/usage/training) and easy model packaging, deployment and workflow management. spaCy is commercial open-source software, released under the MIT license. -💫 **Version 3.2 out now!** +💫 **Version 3.4.0 out now!** [Check out the release notes here.](https://github.com/explosion/spaCy/releases) [![Azure Pipelines](https://img.shields.io/azure-devops/build/explosion-ai/public/8/master.svg?logo=azure-pipelines&style=flat-square&label=build)](https://dev.azure.com/explosion-ai/public/_build?definitionId=8) diff --git a/build-constraints.txt b/build-constraints.txt index cf5fe3284..956973abf 100644 --- a/build-constraints.txt +++ b/build-constraints.txt @@ -1,6 +1,8 @@ # build version constraints for use with wheelwright + multibuild -numpy==1.15.0; python_version<='3.7' -numpy==1.17.3; python_version=='3.8' +numpy==1.15.0; python_version<='3.7' and platform_machine!='aarch64' +numpy==1.19.2; python_version<='3.7' and platform_machine=='aarch64' +numpy==1.17.3; python_version=='3.8' and platform_machine!='aarch64' +numpy==1.19.2; python_version=='3.8' and platform_machine=='aarch64' numpy==1.19.3; python_version=='3.9' numpy==1.21.3; python_version=='3.10' numpy; python_version>='3.11' diff --git a/extra/DEVELOPER_DOCS/Code Conventions.md b/extra/DEVELOPER_DOCS/Code Conventions.md index 37cd8ff27..31a87d362 100644 --- a/extra/DEVELOPER_DOCS/Code Conventions.md +++ b/extra/DEVELOPER_DOCS/Code Conventions.md @@ -455,6 +455,10 @@ Regression tests are tests that refer to bugs reported in specific issues. They The test suite also provides [fixtures](https://github.com/explosion/spaCy/blob/master/spacy/tests/conftest.py) for different language tokenizers that can be used as function arguments of the same name and will be passed in automatically. Those should only be used for tests related to those specific languages. We also have [test utility functions](https://github.com/explosion/spaCy/blob/master/spacy/tests/util.py) for common operations, like creating a temporary file. +### Testing Cython Code + +If you're developing Cython code (`.pyx` files), those extensions will need to be built before the test runner can test that code - otherwise it's going to run the tests with stale code from the last time the extension was built. You can build the extensions locally with `python setup.py build_ext -i`. + ### Constructing objects and state Test functions usually follow the same simple structure: they set up some state, perform the operation you want to test and `assert` conditions that you expect to be true, usually before and after the operation. diff --git a/extra/DEVELOPER_DOCS/ExplosionBot.md b/extra/DEVELOPER_DOCS/ExplosionBot.md index eebec1a06..606fe93a0 100644 --- a/extra/DEVELOPER_DOCS/ExplosionBot.md +++ b/extra/DEVELOPER_DOCS/ExplosionBot.md @@ -16,21 +16,41 @@ To summon the robot, write a github comment on the issue/PR you wish to test. Th Some things to note: -* The `@explosion-bot please` must be the beginning of the command - you cannot add anything in front of this or else the robot won't know how to parse it. Adding anything at the end aside from the test name will also confuse the robot, so keep it simple! -* The command name (such as `test_gpu`) must be one of the tests that the bot knows how to run. The available commands are documented in the bot's [workflow config](https://github.com/explosion/spaCy/blob/master/.github/workflows/explosionbot.yml#L26) and must match exactly one of the commands listed there. -* The robot can't do multiple things at once, so if you want it to run multiple tests, you'll have to summon it with one comment per test. -* For the `test_gpu` command, you can specify an optional thinc branch (from the spaCy repo) or a spaCy branch (from the thinc repo) with either the `--thinc-branch` or `--spacy-branch` flags. By default, the bot will pull in the PR branch from the repo where the command was issued, and the main branch of the other repository. However, if you need to run against another branch, you can say (for example): +- The `@explosion-bot please` must be the beginning of the command - you cannot add anything in front of this or else the robot won't know how to parse it. Adding anything at the end aside from the test name will also confuse the robot, so keep it simple! +- The command name (such as `test_gpu`) must be one of the tests that the bot knows how to run. The available commands are documented in the bot's [workflow config](https://github.com/explosion/spaCy/blob/master/.github/workflows/explosionbot.yml#L26) and must match exactly one of the commands listed there. +- The robot can't do multiple things at once, so if you want it to run multiple tests, you'll have to summon it with one comment per test. -``` -@explosion-bot please test_gpu --thinc-branch develop -``` -You can also specify a branch from an unmerged PR: -``` -@explosion-bot please test_gpu --thinc-branch refs/pull/633/head -``` +### Examples + +- Execute spaCy slow GPU tests with a custom thinc branch from a spaCy PR: + + ``` + @explosion-bot please test_slow_gpu --thinc-branch + ``` + + `branch_name` can either be a named branch, e.g: `develop`, or an unmerged PR, e.g: `refs/pull//head`. + +- Execute spaCy Transformers GPU tests from a spaCy PR: + + ``` + @explosion-bot please test_gpu --run-on spacy-transformers --run-on-branch master --spacy-branch current_pr + ``` + + This will launch the GPU pipeline for the `spacy-transformers` repo on its `master` branch, using the current spaCy PR's branch to build spaCy. The name of the repository passed to `--run-on` is case-sensitive, e.g: use `spaCy` instead of `spacy`. + +- General info about supported commands. + + ``` + @explosion-bot please info + ``` + +- Help text for a specific command + ``` + @explosion-bot please --help + ``` ## Troubleshooting -If the robot isn't responding to commands as expected, you can check its logs in the [Github Action](https://github.com/explosion/spaCy/actions/workflows/explosionbot.yml). +If the robot isn't responding to commands as expected, you can check its logs in the [Github Action](https://github.com/explosion/spaCy/actions/workflows/explosionbot.yml). For each command sent to the bot, there should be a run of the `explosion-bot` workflow. In the `Install and run explosion-bot` step, towards the ends of the logs you should see info about the configuration that the bot was run with, as well as any errors that the bot encountered. diff --git a/pyproject.toml b/pyproject.toml index a43b4c814..317c5fdbe 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,8 +5,7 @@ requires = [ "cymem>=2.0.2,<2.1.0", "preshed>=3.0.2,<3.1.0", "murmurhash>=0.28.0,<1.1.0", - "thinc>=8.0.14,<8.1.0", - "blis>=0.4.0,<0.8.0", + "thinc>=8.1.0,<8.2.0", "pathy", "numpy>=1.15.0", ] diff --git a/requirements.txt b/requirements.txt index 619d35ebc..437dd415a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,8 +3,7 @@ spacy-legacy>=3.0.9,<3.1.0 spacy-loggers>=1.0.0,<2.0.0 cymem>=2.0.2,<2.1.0 preshed>=3.0.2,<3.1.0 -thinc>=8.0.14,<8.1.0 -blis>=0.4.0,<0.8.0 +thinc>=8.1.0,<8.2.0 ml_datasets>=0.2.0,<0.3.0 murmurhash>=0.28.0,<1.1.0 wasabi>=0.9.1,<1.1.0 @@ -16,13 +15,13 @@ pathy>=0.3.5 numpy>=1.15.0 requests>=2.13.0,<3.0.0 tqdm>=4.38.0,<5.0.0 -pydantic>=1.7.4,!=1.8,!=1.8.1,<1.9.0 +pydantic>=1.7.4,!=1.8,!=1.8.1,<1.10.0 jinja2 langcodes>=3.2.0,<4.0.0 # Official Python utilities setuptools packaging>=20.0 -typing_extensions>=3.7.4.1,<4.0.0.0; python_version < "3.8" +typing_extensions>=3.7.4.1,<4.2.0; python_version < "3.8" # Development dependencies pre-commit>=2.13.0 cython>=0.25,<3.0 @@ -31,7 +30,7 @@ pytest-timeout>=1.3.0,<2.0.0 mock>=2.0.0,<3.0.0 flake8>=3.8.0,<3.10.0 hypothesis>=3.27.0,<7.0.0 -mypy==0.910 +mypy>=0.910,<0.970; platform_machine!='aarch64' types-dataclasses>=0.1.3; python_version < "3.7" types-mock>=0.1.1 types-requests diff --git a/setup.cfg b/setup.cfg index 2626de87e..708300b04 100644 --- a/setup.cfg +++ b/setup.cfg @@ -38,7 +38,7 @@ setup_requires = cymem>=2.0.2,<2.1.0 preshed>=3.0.2,<3.1.0 murmurhash>=0.28.0,<1.1.0 - thinc>=8.0.14,<8.1.0 + thinc>=8.1.0,<8.2.0 install_requires = # Our libraries spacy-legacy>=3.0.9,<3.1.0 @@ -46,8 +46,7 @@ install_requires = murmurhash>=0.28.0,<1.1.0 cymem>=2.0.2,<2.1.0 preshed>=3.0.2,<3.1.0 - thinc>=8.0.14,<8.1.0 - blis>=0.4.0,<0.8.0 + thinc>=8.1.0,<8.2.0 wasabi>=0.9.1,<1.1.0 srsly>=2.4.3,<3.0.0 catalogue>=2.0.6,<2.1.0 @@ -57,12 +56,12 @@ install_requires = tqdm>=4.38.0,<5.0.0 numpy>=1.15.0 requests>=2.13.0,<3.0.0 - pydantic>=1.7.4,!=1.8,!=1.8.1,<1.9.0 + pydantic>=1.7.4,!=1.8,!=1.8.1,<1.10.0 jinja2 # Official Python utilities setuptools packaging>=20.0 - typing_extensions>=3.7.4,<4.0.0.0; python_version < "3.8" + typing_extensions>=3.7.4,<4.2.0; python_version < "3.8" langcodes>=3.2.0,<4.0.0 [options.entry_points] @@ -104,14 +103,18 @@ cuda114 = cupy-cuda114>=5.0.0b4,<11.0.0 cuda115 = cupy-cuda115>=5.0.0b4,<11.0.0 +cuda116 = + cupy-cuda116>=5.0.0b4,<11.0.0 +cuda117 = + cupy-cuda117>=5.0.0b4,<11.0.0 apple = - thinc-apple-ops>=0.0.4,<1.0.0 + thinc-apple-ops>=0.1.0.dev0,<1.0.0 # Language tokenizers with external dependencies ja = sudachipy>=0.5.2,!=0.6.1 sudachidict_core>=20211220 ko = - natto-py==0.9.0 + natto-py>=0.9.0 th = pythainlp>=2.0 diff --git a/setup.py b/setup.py index 9023b9fa3..ec1bd35fa 100755 --- a/setup.py +++ b/setup.py @@ -126,6 +126,8 @@ class build_ext_options: class build_ext_subclass(build_ext, build_ext_options): def build_extensions(self): + if self.parallel is None and os.environ.get("SPACY_NUM_BUILD_JOBS") is not None: + self.parallel = int(os.environ.get("SPACY_NUM_BUILD_JOBS")) build_ext_options.build_options(self) build_ext.build_extensions(self) @@ -206,7 +208,11 @@ def setup_package(): for name in MOD_NAMES: mod_path = name.replace(".", "/") + ".pyx" ext = Extension( - name, [mod_path], language="c++", include_dirs=include_dirs, extra_compile_args=["-std=c++11"] + name, + [mod_path], + language="c++", + include_dirs=include_dirs, + extra_compile_args=["-std=c++11"], ) ext_modules.append(ext) print("Cythonizing sources") diff --git a/spacy/__init__.py b/spacy/__init__.py index ca47edc94..069215fda 100644 --- a/spacy/__init__.py +++ b/spacy/__init__.py @@ -32,6 +32,7 @@ def load( *, vocab: Union[Vocab, bool] = True, disable: Iterable[str] = util.SimpleFrozenList(), + enable: Iterable[str] = util.SimpleFrozenList(), exclude: Iterable[str] = util.SimpleFrozenList(), config: Union[Dict[str, Any], Config] = util.SimpleFrozenDict(), ) -> Language: @@ -42,6 +43,8 @@ def load( disable (Iterable[str]): Names of pipeline components to disable. Disabled pipes will be loaded but they won't be run unless you explicitly enable them by calling nlp.enable_pipe. + enable (Iterable[str]): Names of pipeline components to enable. All other + pipes will be disabled (but can be enabled later using nlp.enable_pipe). exclude (Iterable[str]): Names of pipeline components to exclude. Excluded components won't be loaded. config (Dict[str, Any] / Config): Config overrides as nested dict or dict @@ -49,7 +52,12 @@ def load( RETURNS (Language): The loaded nlp object. """ return util.load_model( - name, vocab=vocab, disable=disable, exclude=exclude, config=config + name, + vocab=vocab, + disable=disable, + enable=enable, + exclude=exclude, + config=config, ) diff --git a/spacy/about.py b/spacy/about.py index 03eabc2e9..843c15aba 100644 --- a/spacy/about.py +++ b/spacy/about.py @@ -1,6 +1,6 @@ # fmt: off __title__ = "spacy" -__version__ = "3.3.0" +__version__ = "3.4.1" __download_url__ = "https://github.com/explosion/spacy-models/releases/download" __compatibility__ = "https://raw.githubusercontent.com/explosion/spacy-models/master/compatibility.json" __projects__ = "https://github.com/explosion/projects" diff --git a/spacy/cli/_util.py b/spacy/cli/_util.py index df98e711f..ae43b991b 100644 --- a/spacy/cli/_util.py +++ b/spacy/cli/_util.py @@ -12,7 +12,7 @@ from click.parser import split_arg_string from typer.main import get_command from contextlib import contextmanager from thinc.api import Config, ConfigValidationError, require_gpu -from thinc.util import has_cupy, gpu_is_available +from thinc.util import gpu_is_available from configparser import InterpolationError import os @@ -462,6 +462,23 @@ def git_sparse_checkout(repo, subpath, dest, branch): shutil.move(str(source_path), str(dest)) +def git_repo_branch_exists(repo: str, branch: str) -> bool: + """Uses 'git ls-remote' to check if a repository and branch exists + + repo (str): URL to get repo. + branch (str): Branch on repo to check. + RETURNS (bool): True if repo:branch exists. + """ + get_git_version() + cmd = f"git ls-remote {repo} {branch}" + # We might be tempted to use `--exit-code` with `git ls-remote`, but + # `run_command` handles the `returncode` for us, so we'll rely on + # the fact that stdout returns '' if the requested branch doesn't exist + ret = run_command(cmd, capture=True) + exists = ret.stdout != "" + return exists + + def get_git_version( error: str = "Could not run 'git'. Make sure it's installed and the executable is available.", ) -> Tuple[int, int]: @@ -554,5 +571,5 @@ def setup_gpu(use_gpu: int, silent=None) -> None: require_gpu(use_gpu) else: local_msg.info("Using CPU") - if has_cupy and gpu_is_available(): + if gpu_is_available(): local_msg.info("To switch to GPU 0, use the option: --gpu-id 0") diff --git a/spacy/cli/debug_data.py b/spacy/cli/debug_data.py index f94319d1d..bd05471b1 100644 --- a/spacy/cli/debug_data.py +++ b/spacy/cli/debug_data.py @@ -6,10 +6,11 @@ import sys import srsly from wasabi import Printer, MESSAGES, msg import typer +import math from ._util import app, Arg, Opt, show_validation_error, parse_config_overrides from ._util import import_code, debug_cli -from ..training import Example +from ..training import Example, remove_bilu_prefix from ..training.initialize import get_sourced_components from ..schemas import ConfigSchemaTraining from ..pipeline._parser_internals import nonproj @@ -30,6 +31,12 @@ DEP_LABEL_THRESHOLD = 20 # Minimum number of expected examples to train a new pipeline BLANK_MODEL_MIN_THRESHOLD = 100 BLANK_MODEL_THRESHOLD = 2000 +# Arbitrary threshold where SpanCat performs well +SPAN_DISTINCT_THRESHOLD = 1 +# Arbitrary threshold where SpanCat performs well +BOUNDARY_DISTINCT_THRESHOLD = 1 +# Arbitrary threshold for filtering span lengths during reporting (percentage) +SPAN_LENGTH_THRESHOLD_PERCENTAGE = 90 @debug_cli.command( @@ -247,6 +254,69 @@ def debug_data( msg.warn(f"No examples for texts WITHOUT new label '{label}'") has_no_neg_warning = True + with msg.loading("Obtaining span characteristics..."): + span_characteristics = _get_span_characteristics( + train_dataset, gold_train_data, spans_key + ) + + msg.info(f"Span characteristics for spans_key '{spans_key}'") + msg.info("SD = Span Distinctiveness, BD = Boundary Distinctiveness") + _print_span_characteristics(span_characteristics) + + _span_freqs = _get_spans_length_freq_dist( + gold_train_data["spans_length"][spans_key] + ) + _filtered_span_freqs = _filter_spans_length_freq_dist( + _span_freqs, threshold=SPAN_LENGTH_THRESHOLD_PERCENTAGE + ) + + msg.info( + f"Over {SPAN_LENGTH_THRESHOLD_PERCENTAGE}% of spans have lengths of 1 -- " + f"{max(_filtered_span_freqs.keys())} " + f"(min={span_characteristics['min_length']}, max={span_characteristics['max_length']}). " + f"The most common span lengths are: {_format_freqs(_filtered_span_freqs)}. " + "If you are using the n-gram suggester, note that omitting " + "infrequent n-gram lengths can greatly improve speed and " + "memory usage." + ) + + msg.text( + f"Full distribution of span lengths: {_format_freqs(_span_freqs)}", + show=verbose, + ) + + # Add report regarding span characteristics + if span_characteristics["avg_sd"] < SPAN_DISTINCT_THRESHOLD: + msg.warn("Spans may not be distinct from the rest of the corpus") + else: + msg.good("Spans are distinct from the rest of the corpus") + + p_spans = span_characteristics["p_spans"].values() + all_span_tokens: Counter = sum(p_spans, Counter()) + most_common_spans = [w for w, _ in all_span_tokens.most_common(10)] + msg.text( + "10 most common span tokens: {}".format( + _format_labels(most_common_spans) + ), + show=verbose, + ) + + # Add report regarding span boundary characteristics + if span_characteristics["avg_bd"] < BOUNDARY_DISTINCT_THRESHOLD: + msg.warn("Boundary tokens are not distinct from the rest of the corpus") + else: + msg.good("Boundary tokens are distinct from the rest of the corpus") + + p_bounds = span_characteristics["p_bounds"].values() + all_span_bound_tokens: Counter = sum(p_bounds, Counter()) + most_common_bounds = [w for w, _ in all_span_bound_tokens.most_common(10)] + msg.text( + "10 most common span boundary tokens: {}".format( + _format_labels(most_common_bounds) + ), + show=verbose, + ) + if has_low_data_warning: msg.text( f"To train a new span type, your data should include at " @@ -291,7 +361,7 @@ def debug_data( if label != "-" ] labels_with_counts = _format_labels(labels_with_counts, counts=True) - msg.text(f"Labels in train data: {_format_labels(labels)}", show=verbose) + msg.text(f"Labels in train data: {labels_with_counts}", show=verbose) missing_labels = model_labels - labels if missing_labels: msg.warn( @@ -647,6 +717,9 @@ def _compile_gold( "words": Counter(), "roots": Counter(), "spancat": dict(), + "spans_length": dict(), + "spans_per_type": dict(), + "sb_per_type": dict(), "ws_ents": 0, "boundary_cross_ents": 0, "n_words": 0, @@ -685,21 +758,66 @@ def _compile_gold( # "Illegal" whitespace entity data["ws_ents"] += 1 if label.startswith(("B-", "U-")): - combined_label = label.split("-")[1] + combined_label = remove_bilu_prefix(label) data["ner"][combined_label] += 1 - if sent_starts[i] == True and label.startswith(("I-", "L-")): + if sent_starts[i] and label.startswith(("I-", "L-")): data["boundary_cross_ents"] += 1 elif label == "-": data["ner"]["-"] += 1 if "spancat" in factory_names: - for span_key in list(eg.reference.spans.keys()): - if span_key not in data["spancat"]: - data["spancat"][span_key] = Counter() - for i, span in enumerate(eg.reference.spans[span_key]): + for spans_key in list(eg.reference.spans.keys()): + # Obtain the span frequency + if spans_key not in data["spancat"]: + data["spancat"][spans_key] = Counter() + for i, span in enumerate(eg.reference.spans[spans_key]): if span.label_ is None: continue else: - data["spancat"][span_key][span.label_] += 1 + data["spancat"][spans_key][span.label_] += 1 + + # Obtain the span length + if spans_key not in data["spans_length"]: + data["spans_length"][spans_key] = dict() + for span in gold.spans[spans_key]: + if span.label_ is None: + continue + if span.label_ not in data["spans_length"][spans_key]: + data["spans_length"][spans_key][span.label_] = [] + data["spans_length"][spans_key][span.label_].append(len(span)) + + # Obtain spans per span type + if spans_key not in data["spans_per_type"]: + data["spans_per_type"][spans_key] = dict() + for span in gold.spans[spans_key]: + if span.label_ not in data["spans_per_type"][spans_key]: + data["spans_per_type"][spans_key][span.label_] = [] + data["spans_per_type"][spans_key][span.label_].append(span) + + # Obtain boundary tokens per span type + window_size = 1 + if spans_key not in data["sb_per_type"]: + data["sb_per_type"][spans_key] = dict() + for span in gold.spans[spans_key]: + if span.label_ not in data["sb_per_type"][spans_key]: + # Creating a data structure that holds the start and + # end tokens for each span type + data["sb_per_type"][spans_key][span.label_] = { + "start": [], + "end": [], + } + for offset in range(window_size): + sb_start_idx = span.start - (offset + 1) + if sb_start_idx >= 0: + data["sb_per_type"][spans_key][span.label_]["start"].append( + gold[sb_start_idx : sb_start_idx + 1] + ) + + sb_end_idx = span.end + (offset + 1) + if sb_end_idx <= len(gold): + data["sb_per_type"][spans_key][span.label_]["end"].append( + gold[sb_end_idx - 1 : sb_end_idx] + ) + if "textcat" in factory_names or "textcat_multilabel" in factory_names: data["cats"].update(gold.cats) if any(val not in (0, 1) for val in gold.cats.values()): @@ -770,6 +888,16 @@ def _format_labels( return ", ".join([f"'{l}'" for l in cast(Iterable[str], labels)]) +def _format_freqs(freqs: Dict[int, float], sort: bool = True) -> str: + if sort: + freqs = dict(sorted(freqs.items())) + + _freqs = [(str(k), v) for k, v in freqs.items()] + return ", ".join( + [f"{l} ({c}%)" for l, c in cast(Iterable[Tuple[str, float]], _freqs)] + ) + + def _get_examples_without_label( data: Sequence[Example], label: str, @@ -780,7 +908,7 @@ def _get_examples_without_label( for eg in data: if component == "ner": labels = [ - label.split("-")[1] + remove_bilu_prefix(label) for label in eg.get_aligned_ner() if label not in ("O", "-", None) ] @@ -824,3 +952,158 @@ def _get_labels_from_spancat(nlp: Language) -> Dict[str, Set[str]]: labels[pipe.key] = set() labels[pipe.key].update(pipe.labels) return labels + + +def _gmean(l: List) -> float: + """Compute geometric mean of a list""" + return math.exp(math.fsum(math.log(i) for i in l) / len(l)) + + +def _wgt_average(metric: Dict[str, float], frequencies: Counter) -> float: + total = sum(value * frequencies[span_type] for span_type, value in metric.items()) + return total / sum(frequencies.values()) + + +def _get_distribution(docs, normalize: bool = True) -> Counter: + """Get the frequency distribution given a set of Docs""" + word_counts: Counter = Counter() + for doc in docs: + for token in doc: + # Normalize the text + t = token.text.lower().replace("``", '"').replace("''", '"') + word_counts[t] += 1 + if normalize: + total = sum(word_counts.values(), 0.0) + word_counts = Counter({k: v / total for k, v in word_counts.items()}) + return word_counts + + +def _get_kl_divergence(p: Counter, q: Counter) -> float: + """Compute the Kullback-Leibler divergence from two frequency distributions""" + total = 0.0 + for word, p_word in p.items(): + total += p_word * math.log(p_word / q[word]) + return total + + +def _format_span_row(span_data: List[Dict], labels: List[str]) -> List[Any]: + """Compile into one list for easier reporting""" + d = { + label: [label] + list(round(d[label], 2) for d in span_data) for label in labels + } + return list(d.values()) + + +def _get_span_characteristics( + examples: List[Example], compiled_gold: Dict[str, Any], spans_key: str +) -> Dict[str, Any]: + """Obtain all span characteristics""" + data_labels = compiled_gold["spancat"][spans_key] + # Get lengths + span_length = { + label: _gmean(l) + for label, l in compiled_gold["spans_length"][spans_key].items() + } + min_lengths = [min(l) for l in compiled_gold["spans_length"][spans_key].values()] + max_lengths = [max(l) for l in compiled_gold["spans_length"][spans_key].values()] + + # Get relevant distributions: corpus, spans, span boundaries + p_corpus = _get_distribution([eg.reference for eg in examples], normalize=True) + p_spans = { + label: _get_distribution(spans, normalize=True) + for label, spans in compiled_gold["spans_per_type"][spans_key].items() + } + p_bounds = { + label: _get_distribution(sb["start"] + sb["end"], normalize=True) + for label, sb in compiled_gold["sb_per_type"][spans_key].items() + } + + # Compute for actual span characteristics + span_distinctiveness = { + label: _get_kl_divergence(freq_dist, p_corpus) + for label, freq_dist in p_spans.items() + } + sb_distinctiveness = { + label: _get_kl_divergence(freq_dist, p_corpus) + for label, freq_dist in p_bounds.items() + } + + return { + "sd": span_distinctiveness, + "bd": sb_distinctiveness, + "lengths": span_length, + "min_length": min(min_lengths), + "max_length": max(max_lengths), + "avg_sd": _wgt_average(span_distinctiveness, data_labels), + "avg_bd": _wgt_average(sb_distinctiveness, data_labels), + "avg_length": _wgt_average(span_length, data_labels), + "labels": list(data_labels.keys()), + "p_spans": p_spans, + "p_bounds": p_bounds, + } + + +def _print_span_characteristics(span_characteristics: Dict[str, Any]): + """Print all span characteristics into a table""" + headers = ("Span Type", "Length", "SD", "BD") + # Prepare table data with all span characteristics + table_data = [ + span_characteristics["lengths"], + span_characteristics["sd"], + span_characteristics["bd"], + ] + table = _format_span_row( + span_data=table_data, labels=span_characteristics["labels"] + ) + # Prepare table footer with weighted averages + footer_data = [ + span_characteristics["avg_length"], + span_characteristics["avg_sd"], + span_characteristics["avg_bd"], + ] + footer = ["Wgt. Average"] + [str(round(f, 2)) for f in footer_data] + msg.table(table, footer=footer, header=headers, divider=True) + + +def _get_spans_length_freq_dist( + length_dict: Dict, threshold=SPAN_LENGTH_THRESHOLD_PERCENTAGE +) -> Dict[int, float]: + """Get frequency distribution of spans length under a certain threshold""" + all_span_lengths = [] + for _, lengths in length_dict.items(): + all_span_lengths.extend(lengths) + + freq_dist: Counter = Counter() + for i in all_span_lengths: + if freq_dist.get(i): + freq_dist[i] += 1 + else: + freq_dist[i] = 1 + + # We will be working with percentages instead of raw counts + freq_dist_percentage = {} + for span_length, count in freq_dist.most_common(): + percentage = (count / len(all_span_lengths)) * 100.0 + percentage = round(percentage, 2) + freq_dist_percentage[span_length] = percentage + + return freq_dist_percentage + + +def _filter_spans_length_freq_dist( + freq_dist: Dict[int, float], threshold: int +) -> Dict[int, float]: + """Filter frequency distribution with respect to a threshold + + We're going to filter all the span lengths that fall + around a percentage threshold when summed. + """ + total = 0.0 + filtered_freq_dist = {} + for span_length, dist in freq_dist.items(): + if total >= threshold: + break + else: + filtered_freq_dist[span_length] = dist + total += dist + return filtered_freq_dist diff --git a/spacy/cli/download.py b/spacy/cli/download.py index 4ea9a8f0e..b7de88729 100644 --- a/spacy/cli/download.py +++ b/spacy/cli/download.py @@ -7,6 +7,7 @@ import typer from ._util import app, Arg, Opt, WHEEL_SUFFIX, SDIST_SUFFIX from .. import about from ..util import is_package, get_minor_version, run_command +from ..util import is_prerelease_version from ..errors import OLD_MODEL_SHORTCUTS @@ -74,7 +75,10 @@ def download(model: str, direct: bool = False, sdist: bool = False, *pip_args) - def get_compatibility() -> dict: - version = get_minor_version(about.__version__) + if is_prerelease_version(about.__version__): + version: Optional[str] = about.__version__ + else: + version = get_minor_version(about.__version__) r = requests.get(about.__compatibility__) if r.status_code != 200: msg.fail( diff --git a/spacy/cli/init_config.py b/spacy/cli/init_config.py index d4cd939c2..b634caa4c 100644 --- a/spacy/cli/init_config.py +++ b/spacy/cli/init_config.py @@ -10,6 +10,7 @@ from jinja2 import Template from .. import util from ..language import DEFAULT_CONFIG_PRETRAIN_PATH from ..schemas import RecommendationSchema +from ..util import SimpleFrozenList from ._util import init_cli, Arg, Opt, show_validation_error, COMMAND from ._util import string_to_list, import_code @@ -24,16 +25,30 @@ class Optimizations(str, Enum): accuracy = "accuracy" +class InitValues: + """ + Default values for initialization. Dedicated class to allow synchronized default values for init_config_cli() and + init_config(), i.e. initialization calls via CLI respectively Python. + """ + + lang = "en" + pipeline = SimpleFrozenList(["tagger", "parser", "ner"]) + optimize = Optimizations.efficiency + gpu = False + pretraining = False + force_overwrite = False + + @init_cli.command("config") def init_config_cli( # fmt: off output_file: Path = Arg(..., help="File to save the config to or - for stdout (will only output config and no additional logging info)", allow_dash=True), - lang: str = Opt("en", "--lang", "-l", help="Two-letter code of the language to use"), - pipeline: str = Opt("tagger,parser,ner", "--pipeline", "-p", help="Comma-separated names of trainable pipeline components to include (without 'tok2vec' or 'transformer')"), - optimize: Optimizations = Opt(Optimizations.efficiency.value, "--optimize", "-o", help="Whether to optimize for efficiency (faster inference, smaller model, lower memory consumption) or higher accuracy (potentially larger and slower model). This will impact the choice of architecture, pretrained weights and related hyperparameters."), - gpu: bool = Opt(False, "--gpu", "-G", help="Whether the model can run on GPU. This will impact the choice of architecture, pretrained weights and related hyperparameters."), - pretraining: bool = Opt(False, "--pretraining", "-pt", help="Include config for pretraining (with 'spacy pretrain')"), - force_overwrite: bool = Opt(False, "--force", "-F", help="Force overwriting the output file"), + lang: str = Opt(InitValues.lang, "--lang", "-l", help="Two-letter code of the language to use"), + pipeline: str = Opt(",".join(InitValues.pipeline), "--pipeline", "-p", help="Comma-separated names of trainable pipeline components to include (without 'tok2vec' or 'transformer')"), + optimize: Optimizations = Opt(InitValues.optimize, "--optimize", "-o", help="Whether to optimize for efficiency (faster inference, smaller model, lower memory consumption) or higher accuracy (potentially larger and slower model). This will impact the choice of architecture, pretrained weights and related hyperparameters."), + gpu: bool = Opt(InitValues.gpu, "--gpu", "-G", help="Whether the model can run on GPU. This will impact the choice of architecture, pretrained weights and related hyperparameters."), + pretraining: bool = Opt(InitValues.pretraining, "--pretraining", "-pt", help="Include config for pretraining (with 'spacy pretrain')"), + force_overwrite: bool = Opt(InitValues.force_overwrite, "--force", "-F", help="Force overwriting the output file"), # fmt: on ): """ @@ -133,11 +148,11 @@ def fill_config( def init_config( *, - lang: str, - pipeline: List[str], - optimize: str, - gpu: bool, - pretraining: bool = False, + lang: str = InitValues.lang, + pipeline: List[str] = InitValues.pipeline, + optimize: str = InitValues.optimize, + gpu: bool = InitValues.gpu, + pretraining: bool = InitValues.pretraining, silent: bool = True, ) -> Config: msg = Printer(no_print=silent) diff --git a/spacy/cli/pretrain.py b/spacy/cli/pretrain.py index fe3ce0dad..381d589cf 100644 --- a/spacy/cli/pretrain.py +++ b/spacy/cli/pretrain.py @@ -61,7 +61,7 @@ def pretrain_cli( # TODO: What's the solution here? How do we handle optional blocks? msg.fail("The [pretraining] block in your config is empty", exits=1) if not output_dir.exists(): - output_dir.mkdir() + output_dir.mkdir(parents=True) msg.good(f"Created output directory: {output_dir}") # Save non-interpolated config raw_config.to_disk(output_dir / "config.cfg") diff --git a/spacy/cli/project/assets.py b/spacy/cli/project/assets.py index 5e0cdfdf2..61438d1a8 100644 --- a/spacy/cli/project/assets.py +++ b/spacy/cli/project/assets.py @@ -12,6 +12,9 @@ from .._util import project_cli, Arg, Opt, PROJECT_FILE, load_project_config from .._util import get_checksum, download_file, git_checkout, get_git_version from .._util import SimpleFrozenDict, parse_config_overrides +# Whether assets are extra if `extra` is not set. +EXTRA_DEFAULT = False + @project_cli.command( "assets", @@ -21,7 +24,8 @@ def project_assets_cli( # fmt: off ctx: typer.Context, # This is only used to read additional arguments project_dir: Path = Arg(Path.cwd(), help="Path to cloned project. Defaults to current working directory.", exists=True, file_okay=False), - sparse_checkout: bool = Opt(False, "--sparse", "-S", help="Use sparse checkout for assets provided via Git, to only check out and clone the files needed. Requires Git v22.2+.") + sparse_checkout: bool = Opt(False, "--sparse", "-S", help="Use sparse checkout for assets provided via Git, to only check out and clone the files needed. Requires Git v22.2+."), + extra: bool = Opt(False, "--extra", "-e", help="Download all assets, including those marked as 'extra'.") # fmt: on ): """Fetch project assets like datasets and pretrained weights. Assets are @@ -32,7 +36,12 @@ def project_assets_cli( DOCS: https://spacy.io/api/cli#project-assets """ overrides = parse_config_overrides(ctx.args) - project_assets(project_dir, overrides=overrides, sparse_checkout=sparse_checkout) + project_assets( + project_dir, + overrides=overrides, + sparse_checkout=sparse_checkout, + extra=extra, + ) def project_assets( @@ -40,17 +49,29 @@ def project_assets( *, overrides: Dict[str, Any] = SimpleFrozenDict(), sparse_checkout: bool = False, + extra: bool = False, ) -> None: """Fetch assets for a project using DVC if possible. project_dir (Path): Path to project directory. + sparse_checkout (bool): Use sparse checkout for assets provided via Git, to only check out and clone the files + needed. + extra (bool): Whether to download all assets, including those marked as 'extra'. """ project_path = ensure_path(project_dir) config = load_project_config(project_path, overrides=overrides) - assets = config.get("assets", {}) + assets = [ + asset + for asset in config.get("assets", []) + if extra or not asset.get("extra", EXTRA_DEFAULT) + ] if not assets: - msg.warn(f"No assets specified in {PROJECT_FILE}", exits=0) + msg.warn( + f"No assets specified in {PROJECT_FILE} (if assets are marked as extra, download them with --extra)", + exits=0, + ) msg.info(f"Fetching {len(assets)} asset(s)") + for asset in assets: dest = (project_dir / asset["dest"]).resolve() checksum = asset.get("checksum") diff --git a/spacy/cli/project/clone.py b/spacy/cli/project/clone.py index 360ee3428..14b4ed9b5 100644 --- a/spacy/cli/project/clone.py +++ b/spacy/cli/project/clone.py @@ -7,11 +7,11 @@ import re from ... import about from ...util import ensure_path from .._util import project_cli, Arg, Opt, COMMAND, PROJECT_FILE -from .._util import git_checkout, get_git_version +from .._util import git_checkout, get_git_version, git_repo_branch_exists DEFAULT_REPO = about.__projects__ DEFAULT_PROJECTS_BRANCH = about.__projects_branch__ -DEFAULT_BRANCH = "master" +DEFAULT_BRANCHES = ["main", "master"] @project_cli.command("clone") @@ -20,7 +20,7 @@ def project_clone_cli( name: str = Arg(..., help="The name of the template to clone"), dest: Optional[Path] = Arg(None, help="Where to clone the project. Defaults to current working directory", exists=False), repo: str = Opt(DEFAULT_REPO, "--repo", "-r", help="The repository to clone from"), - branch: Optional[str] = Opt(None, "--branch", "-b", help="The branch to clone from"), + branch: Optional[str] = Opt(None, "--branch", "-b", help=f"The branch to clone from. If not provided, will attempt {', '.join(DEFAULT_BRANCHES)}"), sparse_checkout: bool = Opt(False, "--sparse", "-S", help="Use sparse Git checkout to only check out and clone the files needed. Requires Git v22.2+.") # fmt: on ): @@ -33,9 +33,25 @@ def project_clone_cli( """ if dest is None: dest = Path.cwd() / Path(name).parts[-1] + if repo == DEFAULT_REPO and branch is None: + branch = DEFAULT_PROJECTS_BRANCH + if branch is None: - # If it's a user repo, we want to default to other branch - branch = DEFAULT_PROJECTS_BRANCH if repo == DEFAULT_REPO else DEFAULT_BRANCH + for default_branch in DEFAULT_BRANCHES: + if git_repo_branch_exists(repo, default_branch): + branch = default_branch + break + if branch is None: + default_branches_msg = ", ".join(f"'{b}'" for b in DEFAULT_BRANCHES) + msg.fail( + "No branch provided and attempted default " + f"branches {default_branches_msg} do not exist.", + exits=1, + ) + else: + if not git_repo_branch_exists(repo, branch): + msg.fail(f"repo: {repo} (branch: {branch}) does not exist.", exits=1) + assert isinstance(branch, str) project_clone(name, dest, repo=repo, branch=branch, sparse_checkout=sparse_checkout) @@ -61,9 +77,9 @@ def project_clone( try: git_checkout(repo, name, dest, branch=branch, sparse=sparse_checkout) except subprocess.CalledProcessError: - err = f"Could not clone '{name}' from repo '{repo_name}'" + err = f"Could not clone '{name}' from repo '{repo_name}' (branch '{branch}')" msg.fail(err, exits=1) - msg.good(f"Cloned '{name}' from {repo_name}", project_dir) + msg.good(f"Cloned '{name}' from '{repo_name}' (branch '{branch}')", project_dir) if not (project_dir / PROJECT_FILE).exists(): msg.warn(f"No {PROJECT_FILE} found in directory") else: diff --git a/spacy/displacy/render.py b/spacy/displacy/render.py index 247ad996b..50dc3466c 100644 --- a/spacy/displacy/render.py +++ b/spacy/displacy/render.py @@ -64,8 +64,11 @@ class SpanRenderer: # Set up how the text and labels will be rendered self.direction = DEFAULT_DIR self.lang = DEFAULT_LANG + # These values are in px self.top_offset = options.get("top_offset", 40) - self.top_offset_step = options.get("top_offset_step", 17) + # This is how far under the top offset the span labels appear + self.span_label_offset = options.get("span_label_offset", 20) + self.offset_step = options.get("top_offset_step", 17) # Set up which templates will be used template = options.get("template") @@ -127,26 +130,56 @@ class SpanRenderer: title (str / None): Document title set in Doc.user_data['title']. """ per_token_info = [] + # we must sort so that we can correctly describe when spans need to "stack" + # which is determined by their start token, then span length (longer spans on top), + # then break any remaining ties with the span label + spans = sorted( + spans, + key=lambda s: ( + s["start_token"], + -(s["end_token"] - s["start_token"]), + s["label"], + ), + ) + for s in spans: + # this is the vertical 'slot' that the span will be rendered in + # vertical_position = span_label_offset + (offset_step * (slot - 1)) + s["render_slot"] = 0 for idx, token in enumerate(tokens): # Identify if a token belongs to a Span (and which) and if it's a # start token of said Span. We'll use this for the final HTML render token_markup: Dict[str, Any] = {} token_markup["text"] = token + concurrent_spans = 0 entities = [] for span in spans: ent = {} if span["start_token"] <= idx < span["end_token"]: + concurrent_spans += 1 + span_start = idx == span["start_token"] ent["label"] = span["label"] - ent["is_start"] = True if idx == span["start_token"] else False + ent["is_start"] = span_start + if span_start: + # When the span starts, we need to know how many other + # spans are on the 'span stack' and will be rendered. + # This value becomes the vertical render slot for this entire span + span["render_slot"] = concurrent_spans + ent["render_slot"] = span["render_slot"] kb_id = span.get("kb_id", "") kb_url = span.get("kb_url", "#") ent["kb_link"] = ( TPL_KB_LINK.format(kb_id=kb_id, kb_url=kb_url) if kb_id else "" ) entities.append(ent) + else: + # We don't specifically need to do this since we loop + # over tokens and spans sorted by their start_token, + # so we'll never use a span again after the last token it appears in, + # but if we were to use these spans again we'd want to make sure + # this value was reset correctly. + span["render_slot"] = 0 token_markup["entities"] = entities per_token_info.append(token_markup) - markup = self._render_markup(per_token_info) markup = TPL_SPANS.format(content=markup, dir=self.direction) if title: @@ -157,12 +190,24 @@ class SpanRenderer: """Render the markup from per-token information""" markup = "" for token in per_token_info: - entities = sorted(token["entities"], key=lambda d: d["label"]) - if entities: + entities = sorted(token["entities"], key=lambda d: d["render_slot"]) + # Whitespace tokens disrupt the vertical space (no line height) so that the + # span indicators get misaligned. We don't render them as individual + # tokens anyway, so we'll just not display a span indicator either. + is_whitespace = token["text"].strip() == "" + if entities and not is_whitespace: slices = self._get_span_slices(token["entities"]) starts = self._get_span_starts(token["entities"]) + total_height = ( + self.top_offset + + self.span_label_offset + + (self.offset_step * (len(entities) - 1)) + ) markup += self.span_template.format( - text=token["text"], span_slices=slices, span_starts=starts + text=token["text"], + span_slices=slices, + span_starts=starts, + total_height=total_height, ) else: markup += escape_html(token["text"] + " ") @@ -171,10 +216,18 @@ class SpanRenderer: def _get_span_slices(self, entities: List[Dict]) -> str: """Get the rendered markup of all Span slices""" span_slices = [] - for entity, step in zip(entities, itertools.count(step=self.top_offset_step)): + for entity in entities: + # rather than iterate over multiples of offset_step, we use entity['render_slot'] + # to determine the vertical position, since that tells where + # the span starts vertically so we can extend it horizontally, + # past other spans that might have already ended color = self.colors.get(entity["label"].upper(), self.default_color) + top_offset = self.top_offset + ( + self.offset_step * (entity["render_slot"] - 1) + ) span_slice = self.span_slice_template.format( - bg=color, top_offset=self.top_offset + step + bg=color, + top_offset=top_offset, ) span_slices.append(span_slice) return "".join(span_slices) @@ -182,12 +235,15 @@ class SpanRenderer: def _get_span_starts(self, entities: List[Dict]) -> str: """Get the rendered markup of all Span start tokens""" span_starts = [] - for entity, step in zip(entities, itertools.count(step=self.top_offset_step)): + for entity in entities: color = self.colors.get(entity["label"].upper(), self.default_color) + top_offset = self.top_offset + ( + self.offset_step * (entity["render_slot"] - 1) + ) span_start = ( self.span_start_template.format( bg=color, - top_offset=self.top_offset + step, + top_offset=top_offset, label=entity["label"], kb_link=entity["kb_link"], ) diff --git a/spacy/displacy/templates.py b/spacy/displacy/templates.py index ff81e7a1d..40f5376b1 100644 --- a/spacy/displacy/templates.py +++ b/spacy/displacy/templates.py @@ -67,7 +67,7 @@ TPL_SPANS = """ """ TPL_SPAN = """ - + {text} {span_slices} {span_starts} diff --git a/spacy/errors.py b/spacy/errors.py index b01afcb80..fd412a4da 100644 --- a/spacy/errors.py +++ b/spacy/errors.py @@ -1,4 +1,5 @@ import warnings +from .compat import Literal class ErrorsWithCodes(type): @@ -26,7 +27,10 @@ def setup_default_warnings(): filter_warning("once", error_msg="[W114]") -def filter_warning(action: str, error_msg: str): +def filter_warning( + action: Literal["default", "error", "ignore", "always", "module", "once"], + error_msg: str, +): """Customize how spaCy should handle a certain warning. error_msg (str): e.g. "W006", or a full error message @@ -199,6 +203,15 @@ class Warnings(metaclass=ErrorsWithCodes): W118 = ("Term '{term}' not found in glossary. It may however be explained in documentation " "for the corpora used to train the language. Please check " "`nlp.meta[\"sources\"]` for any relevant links.") + W119 = ("Overriding pipe name in `config` is not supported. Ignoring override '{name_in_config}'.") + W120 = ("Unable to load all spans in Doc.spans: more than one span group " + "with the name '{group_name}' was found in the saved spans data. " + "Only the last span group will be loaded under " + "Doc.spans['{group_name}']. Skipping span group with values: " + "{group_values}") + W121 = ("Attempting to trace non-existent method '{method}' in pipe '{pipe}'") + W122 = ("Couldn't trace method '{method}' in pipe '{pipe}'. This can happen if the pipe class " + "is a Cython extension type.") class Errors(metaclass=ErrorsWithCodes): @@ -444,10 +457,10 @@ class Errors(metaclass=ErrorsWithCodes): "same, but found '{nlp}' and '{vocab}' respectively.") E152 = ("The attribute {attr} is not supported for token patterns. " "Please use the option `validate=True` with the Matcher, PhraseMatcher, " - "or EntityRuler for more details.") + "EntityRuler or AttributeRuler for more details.") E153 = ("The value type {vtype} is not supported for token patterns. " "Please use the option validate=True with Matcher, PhraseMatcher, " - "or EntityRuler for more details.") + "EntityRuler or AttributeRuler for more details.") E154 = ("One of the attributes or values is not supported for token " "patterns. Please use the option `validate=True` with the Matcher, " "PhraseMatcher, or EntityRuler for more details.") @@ -527,6 +540,8 @@ class Errors(metaclass=ErrorsWithCodes): E202 = ("Unsupported {name} mode '{mode}'. Supported modes: {modes}.") # New errors added in v3.x + E854 = ("Unable to set doc.ents. Check that the 'ents_filter' does not " + "permit overlapping spans.") E855 = ("Invalid {obj}: {obj} is not from the same doc.") E856 = ("Error accessing span at position {i}: out of bounds in span group " "of length {length}.") @@ -898,8 +913,8 @@ class Errors(metaclass=ErrorsWithCodes): E1022 = ("Words must be of type str or int, but input is of type '{wtype}'") E1023 = ("Couldn't read EntityRuler from the {path}. This file doesn't " "exist.") - E1024 = ("A pattern with ID \"{ent_id}\" is not present in EntityRuler " - "patterns.") + E1024 = ("A pattern with {attr_type} '{label}' is not present in " + "'{component}' patterns.") E1025 = ("Cannot intify the value '{value}' as an IOB string. The only " "supported values are: 'I', 'O', 'B' and ''") E1026 = ("Edit tree has an invalid format:\n{errors}") @@ -913,6 +928,17 @@ class Errors(metaclass=ErrorsWithCodes): E1034 = ("Node index {i} out of bounds ({length})") E1035 = ("Token index {i} out of bounds ({length})") E1036 = ("Cannot index into NoneNode") + E1037 = ("Invalid attribute value '{attr}'.") + E1038 = ("Invalid JSON input: {message}") + E1039 = ("The {obj} start or end annotations (start: {start}, end: {end}) " + "could not be aligned to token boundaries.") + E1040 = ("Doc.from_json requires all tokens to have the same attributes. " + "Some tokens do not contain annotation for: {partial_attrs}") + E1041 = ("Expected a string, Doc, or bytes as input, but got: {type}") + E1042 = ("Function was called with `{arg1}`={arg1_values} and " + "`{arg2}`={arg2_values} but these arguments are conflicting.") + E1043 = ("Expected None or a value in range [{range_start}, {range_end}] for entity linker threshold, but got " + "{value}.") # Deprecated model shortcuts, only used in errors and warnings diff --git a/spacy/glossary.py b/spacy/glossary.py index 25c00d3ed..d2240fbba 100644 --- a/spacy/glossary.py +++ b/spacy/glossary.py @@ -273,6 +273,7 @@ GLOSSARY = { "relcl": "relative clause modifier", "reparandum": "overridden disfluency", "root": "root", + "ROOT": "root", "vocative": "vocative", "xcomp": "open clausal complement", # Dependency labels (German) diff --git a/spacy/kb.pyx b/spacy/kb.pyx index 9a765c8e4..ae1983a8d 100644 --- a/spacy/kb.pyx +++ b/spacy/kb.pyx @@ -93,14 +93,14 @@ cdef class KnowledgeBase: self.vocab = vocab self._create_empty_vectors(dummy_hash=self.vocab.strings[""]) - def initialize_entities(self, int64_t nr_entities): + def _initialize_entities(self, int64_t nr_entities): self._entry_index = PreshMap(nr_entities + 1) self._entries = entry_vec(nr_entities + 1) - def initialize_vectors(self, int64_t nr_entities): + def _initialize_vectors(self, int64_t nr_entities): self._vectors_table = float_matrix(nr_entities + 1) - def initialize_aliases(self, int64_t nr_aliases): + def _initialize_aliases(self, int64_t nr_aliases): self._alias_index = PreshMap(nr_aliases + 1) self._aliases_table = alias_vec(nr_aliases + 1) @@ -155,8 +155,8 @@ cdef class KnowledgeBase: raise ValueError(Errors.E140) nr_entities = len(set(entity_list)) - self.initialize_entities(nr_entities) - self.initialize_vectors(nr_entities) + self._initialize_entities(nr_entities) + self._initialize_vectors(nr_entities) i = 0 cdef KBEntryC entry @@ -388,9 +388,9 @@ cdef class KnowledgeBase: nr_entities = header[0] nr_aliases = header[1] entity_vector_length = header[2] - self.initialize_entities(nr_entities) - self.initialize_vectors(nr_entities) - self.initialize_aliases(nr_aliases) + self._initialize_entities(nr_entities) + self._initialize_vectors(nr_entities) + self._initialize_aliases(nr_aliases) self.entity_vector_length = entity_vector_length def deserialize_vectors(b): @@ -512,8 +512,8 @@ cdef class KnowledgeBase: cdef int64_t entity_vector_length reader.read_header(&nr_entities, &entity_vector_length) - self.initialize_entities(nr_entities) - self.initialize_vectors(nr_entities) + self._initialize_entities(nr_entities) + self._initialize_vectors(nr_entities) self.entity_vector_length = entity_vector_length # STEP 1: load entity vectors @@ -552,7 +552,7 @@ cdef class KnowledgeBase: # STEP 3: load aliases cdef int64_t nr_aliases reader.read_alias_length(&nr_aliases) - self.initialize_aliases(nr_aliases) + self._initialize_aliases(nr_aliases) cdef int64_t nr_candidates cdef vector[int64_t] entry_indices diff --git a/spacy/lang/bg/__init__.py b/spacy/lang/bg/__init__.py index 559cc34c4..c9176b946 100644 --- a/spacy/lang/bg/__init__.py +++ b/spacy/lang/bg/__init__.py @@ -2,7 +2,8 @@ from .stop_words import STOP_WORDS from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS from .lex_attrs import LEX_ATTRS from ..tokenizer_exceptions import BASE_EXCEPTIONS - +from ..punctuation import COMBINING_DIACRITICS_TOKENIZER_INFIXES +from ..punctuation import COMBINING_DIACRITICS_TOKENIZER_SUFFIXES from ...language import Language, BaseDefaults from ...attrs import LANG from ...util import update_exc @@ -16,6 +17,8 @@ class BulgarianDefaults(BaseDefaults): stop_words = STOP_WORDS tokenizer_exceptions = update_exc(BASE_EXCEPTIONS, TOKENIZER_EXCEPTIONS) + suffixes = COMBINING_DIACRITICS_TOKENIZER_SUFFIXES + infixes = COMBINING_DIACRITICS_TOKENIZER_INFIXES class Bulgarian(Language): diff --git a/spacy/lang/char_classes.py b/spacy/lang/char_classes.py index b15bb3cf3..1d204c46c 100644 --- a/spacy/lang/char_classes.py +++ b/spacy/lang/char_classes.py @@ -258,6 +258,10 @@ ALPHA = group_chars( ALPHA_LOWER = group_chars(_lower + _uncased) ALPHA_UPPER = group_chars(_upper + _uncased) +_combining_diacritics = r"\u0300-\u036f" + +COMBINING_DIACRITICS = _combining_diacritics + _units = ( "km km² km³ m m² m³ dm dm² dm³ cm cm² cm³ mm mm² mm³ ha µm nm yd in ft " "kg g mg µg t lb oz m/s km/h kmh mph hPa Pa mbar mb MB kb KB gb GB tb " diff --git a/spacy/lang/en/tokenizer_exceptions.py b/spacy/lang/en/tokenizer_exceptions.py index 2c20b8c27..7886e28cb 100644 --- a/spacy/lang/en/tokenizer_exceptions.py +++ b/spacy/lang/en/tokenizer_exceptions.py @@ -35,7 +35,7 @@ for pron in ["i"]: _exc[orth + "m"] = [ {ORTH: orth, NORM: pron}, - {ORTH: "m", "tenspect": 1, "number": 1}, + {ORTH: "m"}, ] _exc[orth + "'ma"] = [ @@ -139,26 +139,27 @@ for pron in ["he", "she", "it"]: # W-words, relative pronouns, prepositions etc. -for word in [ - "who", - "what", - "when", - "where", - "why", - "how", - "there", - "that", - "this", - "these", - "those", +for word, morph in [ + ("who", None), + ("what", None), + ("when", None), + ("where", None), + ("why", None), + ("how", None), + ("there", None), + ("that", "Number=Sing|Person=3"), + ("this", "Number=Sing|Person=3"), + ("these", "Number=Plur|Person=3"), + ("those", "Number=Plur|Person=3"), ]: for orth in [word, word.title()]: - _exc[orth + "'s"] = [ - {ORTH: orth, NORM: word}, - {ORTH: "'s", NORM: "'s"}, - ] + if morph != "Number=Plur|Person=3": + _exc[orth + "'s"] = [ + {ORTH: orth, NORM: word}, + {ORTH: "'s", NORM: "'s"}, + ] - _exc[orth + "s"] = [{ORTH: orth, NORM: word}, {ORTH: "s"}] + _exc[orth + "s"] = [{ORTH: orth, NORM: word}, {ORTH: "s"}] _exc[orth + "'ll"] = [ {ORTH: orth, NORM: word}, @@ -182,25 +183,26 @@ for word in [ {ORTH: "ve", NORM: "have"}, ] - _exc[orth + "'re"] = [ - {ORTH: orth, NORM: word}, - {ORTH: "'re", NORM: "are"}, - ] + if morph != "Number=Sing|Person=3": + _exc[orth + "'re"] = [ + {ORTH: orth, NORM: word}, + {ORTH: "'re", NORM: "are"}, + ] - _exc[orth + "re"] = [ - {ORTH: orth, NORM: word}, - {ORTH: "re", NORM: "are"}, - ] + _exc[orth + "re"] = [ + {ORTH: orth, NORM: word}, + {ORTH: "re", NORM: "are"}, + ] - _exc[orth + "'ve"] = [ - {ORTH: orth, NORM: word}, - {ORTH: "'ve"}, - ] + _exc[orth + "'ve"] = [ + {ORTH: orth, NORM: word}, + {ORTH: "'ve"}, + ] - _exc[orth + "ve"] = [ - {ORTH: orth}, - {ORTH: "ve", NORM: "have"}, - ] + _exc[orth + "ve"] = [ + {ORTH: orth}, + {ORTH: "ve", NORM: "have"}, + ] _exc[orth + "'d"] = [ {ORTH: orth, NORM: word}, diff --git a/spacy/lang/punctuation.py b/spacy/lang/punctuation.py index e712e71d6..a1cfe6224 100644 --- a/spacy/lang/punctuation.py +++ b/spacy/lang/punctuation.py @@ -1,5 +1,5 @@ from .char_classes import LIST_PUNCT, LIST_ELLIPSES, LIST_QUOTES, LIST_CURRENCY -from .char_classes import LIST_ICONS, HYPHENS, CURRENCY, UNITS +from .char_classes import LIST_ICONS, HYPHENS, CURRENCY, UNITS, COMBINING_DIACRITICS from .char_classes import CONCAT_QUOTES, ALPHA_LOWER, ALPHA_UPPER, ALPHA, PUNCT @@ -44,3 +44,23 @@ TOKENIZER_INFIXES = ( r"(?<=[{a}0-9])[:<>=/](?=[{a}])".format(a=ALPHA), ] ) + + +# Some languages e.g. written with the Cyrillic alphabet permit the use of diacritics +# to mark stressed syllables in words where stress is distinctive. Such languages +# should use the COMBINING_DIACRITICS... suffix and infix regex lists in +# place of the standard ones. +COMBINING_DIACRITICS_TOKENIZER_SUFFIXES = list(TOKENIZER_SUFFIXES) + [ + r"(?<=[{a}][{d}])\.".format(a=ALPHA, d=COMBINING_DIACRITICS), +] + +COMBINING_DIACRITICS_TOKENIZER_INFIXES = list(TOKENIZER_INFIXES) + [ + r"(?<=[{al}][{d}])\.(?=[{au}{q}])".format( + al=ALPHA_LOWER, au=ALPHA_UPPER, q=CONCAT_QUOTES, d=COMBINING_DIACRITICS + ), + r"(?<=[{a}][{d}]),(?=[{a}])".format(a=ALPHA, d=COMBINING_DIACRITICS), + r"(?<=[{a}][{d}])(?:{h})(?=[{a}])".format( + a=ALPHA, d=COMBINING_DIACRITICS, h=HYPHENS + ), + r"(?<=[{a}][{d}])[:<>=/](?=[{a}])".format(a=ALPHA, d=COMBINING_DIACRITICS), +] diff --git a/spacy/lang/ru/__init__.py b/spacy/lang/ru/__init__.py index 5d31d8ea2..c118c26ff 100644 --- a/spacy/lang/ru/__init__.py +++ b/spacy/lang/ru/__init__.py @@ -5,6 +5,8 @@ from .stop_words import STOP_WORDS from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS from .lex_attrs import LEX_ATTRS from .lemmatizer import RussianLemmatizer +from ..punctuation import COMBINING_DIACRITICS_TOKENIZER_INFIXES +from ..punctuation import COMBINING_DIACRITICS_TOKENIZER_SUFFIXES from ...language import Language, BaseDefaults @@ -12,6 +14,8 @@ class RussianDefaults(BaseDefaults): tokenizer_exceptions = TOKENIZER_EXCEPTIONS lex_attr_getters = LEX_ATTRS stop_words = STOP_WORDS + suffixes = COMBINING_DIACRITICS_TOKENIZER_SUFFIXES + infixes = COMBINING_DIACRITICS_TOKENIZER_INFIXES class Russian(Language): diff --git a/spacy/lang/uk/__init__.py b/spacy/lang/uk/__init__.py index 21f9649f2..737243b66 100644 --- a/spacy/lang/uk/__init__.py +++ b/spacy/lang/uk/__init__.py @@ -6,6 +6,8 @@ from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS from .stop_words import STOP_WORDS from .lex_attrs import LEX_ATTRS from .lemmatizer import UkrainianLemmatizer +from ..punctuation import COMBINING_DIACRITICS_TOKENIZER_INFIXES +from ..punctuation import COMBINING_DIACRITICS_TOKENIZER_SUFFIXES from ...language import Language, BaseDefaults @@ -13,6 +15,8 @@ class UkrainianDefaults(BaseDefaults): tokenizer_exceptions = TOKENIZER_EXCEPTIONS lex_attr_getters = LEX_ATTRS stop_words = STOP_WORDS + suffixes = COMBINING_DIACRITICS_TOKENIZER_SUFFIXES + infixes = COMBINING_DIACRITICS_TOKENIZER_INFIXES class Ukrainian(Language): diff --git a/spacy/language.py b/spacy/language.py index bab403f0e..816bd6531 100644 --- a/spacy/language.py +++ b/spacy/language.py @@ -1,4 +1,4 @@ -from typing import Iterator, Optional, Any, Dict, Callable, Iterable +from typing import Iterator, Optional, Any, Dict, Callable, Iterable, Collection from typing import Union, Tuple, List, Set, Pattern, Sequence from typing import NoReturn, TYPE_CHECKING, TypeVar, cast, overload @@ -774,6 +774,9 @@ class Language: name = name if name is not None else factory_name if name in self.component_names: raise ValueError(Errors.E007.format(name=name, opts=self.component_names)) + # Overriding pipe name in the config is not supported and will be ignored. + if "name" in config: + warnings.warn(Warnings.W119.format(name_in_config=config.pop("name"))) if source is not None: # We're loading the component from a model. After loading the # component, we know its real factory name @@ -1087,16 +1090,21 @@ class Language: ) return self.tokenizer(text) - def _ensure_doc(self, doc_like: Union[str, Doc]) -> Doc: - """Create a Doc if need be, or raise an error if the input is not a Doc or a string.""" + def _ensure_doc(self, doc_like: Union[str, Doc, bytes]) -> Doc: + """Create a Doc if need be, or raise an error if the input is not + a Doc, string, or a byte array (generated by Doc.to_bytes()).""" if isinstance(doc_like, Doc): return doc_like if isinstance(doc_like, str): return self.make_doc(doc_like) - raise ValueError(Errors.E866.format(type=type(doc_like))) + if isinstance(doc_like, bytes): + return Doc(self.vocab).from_bytes(doc_like) + raise ValueError(Errors.E1041.format(type=type(doc_like))) - def _ensure_doc_with_context(self, doc_like: Union[str, Doc], context: Any) -> Doc: - """Create a Doc if need be and add as_tuples context, or raise an error if the input is not a Doc or a string.""" + def _ensure_doc_with_context( + self, doc_like: Union[str, Doc, bytes], context: _AnyContext + ) -> Doc: + """Call _ensure_doc to generate a Doc and set its context object.""" doc = self._ensure_doc(doc_like) doc._context = context return doc @@ -1516,7 +1524,6 @@ class Language: DOCS: https://spacy.io/api/language#pipe """ - # Handle texts with context as tuples if as_tuples: texts = cast(Iterable[Tuple[Union[str, Doc], _AnyContext]], texts) docs_with_contexts = ( @@ -1594,8 +1601,21 @@ class Language: n_process: int, batch_size: int, ) -> Iterator[Doc]: + def prepare_input( + texts: Iterable[Union[str, Doc]] + ) -> Iterable[Tuple[Union[str, bytes], _AnyContext]]: + # Serialize Doc inputs to bytes to avoid incurring pickling + # overhead when they are passed to child processes. Also yield + # any context objects they might have separately (as they are not serialized). + for doc_like in texts: + if isinstance(doc_like, Doc): + yield (doc_like.to_bytes(), cast(_AnyContext, doc_like._context)) + else: + yield (doc_like, cast(_AnyContext, None)) + + serialized_texts_with_ctx = prepare_input(texts) # type: ignore # raw_texts is used later to stop iteration. - texts, raw_texts = itertools.tee(texts) + texts, raw_texts = itertools.tee(serialized_texts_with_ctx) # type: ignore # for sending texts to worker texts_q: List[mp.Queue] = [mp.Queue() for _ in range(n_process)] # for receiving byte-encoded docs from worker @@ -1615,7 +1635,13 @@ class Language: procs = [ mp.Process( target=_apply_pipes, - args=(self._ensure_doc, pipes, rch, sch, Underscore.get_state()), + args=( + self._ensure_doc_with_context, + pipes, + rch, + sch, + Underscore.get_state(), + ), ) for rch, sch in zip(texts_q, bytedocs_send_ch) ] @@ -1628,12 +1654,12 @@ class Language: recv.recv() for recv in cycle(bytedocs_recv_ch) ) try: - for i, (_, (byte_doc, byte_context, byte_error)) in enumerate( + for i, (_, (byte_doc, context, byte_error)) in enumerate( zip(raw_texts, byte_tuples), 1 ): if byte_doc is not None: doc = Doc(self.vocab).from_bytes(byte_doc) - doc._context = byte_context + doc._context = context yield doc elif byte_error is not None: error = srsly.msgpack_loads(byte_error) @@ -1668,6 +1694,7 @@ class Language: *, vocab: Union[Vocab, bool] = True, disable: Iterable[str] = SimpleFrozenList(), + enable: Iterable[str] = SimpleFrozenList(), exclude: Iterable[str] = SimpleFrozenList(), meta: Dict[str, Any] = SimpleFrozenDict(), auto_fill: bool = True, @@ -1682,6 +1709,8 @@ class Language: disable (Iterable[str]): Names of pipeline components to disable. Disabled pipes will be loaded but they won't be run unless you explicitly enable them by calling nlp.enable_pipe. + enable (Iterable[str]): Names of pipeline components to enable. All other + pipes will be disabled (and can be enabled using `nlp.enable_pipe`). exclude (Iterable[str]): Names of pipeline components to exclude. Excluded components won't be loaded. meta (Dict[str, Any]): Meta overrides for nlp.meta. @@ -1835,8 +1864,15 @@ class Language: # Restore the original vocab after sourcing if necessary if vocab_b is not None: nlp.vocab.from_bytes(vocab_b) - disabled_pipes = [*config["nlp"]["disabled"], *disable] + + # Resolve disabled/enabled settings. + disabled_pipes = cls._resolve_component_status( + [*config["nlp"]["disabled"], *disable], + [*config["nlp"].get("enabled", []), *enable], + config["nlp"]["pipeline"], + ) nlp._disabled = set(p for p in disabled_pipes if p not in exclude) + nlp.batch_size = config["nlp"]["batch_size"] nlp.config = filled if auto_fill else config if after_pipeline_creation is not None: @@ -1988,6 +2024,42 @@ class Language: serializers["vocab"] = lambda p: self.vocab.to_disk(p, exclude=exclude) util.to_disk(path, serializers, exclude) + @staticmethod + def _resolve_component_status( + disable: Iterable[str], enable: Iterable[str], pipe_names: Collection[str] + ) -> Tuple[str, ...]: + """Derives whether (1) `disable` and `enable` values are consistent and (2) + resolves those to a single set of disabled components. Raises an error in + case of inconsistency. + + disable (Iterable[str]): Names of components or serialization fields to disable. + enable (Iterable[str]): Names of pipeline components to enable. + pipe_names (Iterable[str]): Names of all pipeline components. + + RETURNS (Tuple[str, ...]): Names of components to exclude from pipeline w.r.t. + specified includes and excludes. + """ + + if disable is not None and isinstance(disable, str): + disable = [disable] + to_disable = disable + + if enable: + to_disable = [ + pipe_name for pipe_name in pipe_names if pipe_name not in enable + ] + if disable and disable != to_disable: + raise ValueError( + Errors.E1042.format( + arg1="enable", + arg2="disable", + arg1_values=enable, + arg2_values=disable, + ) + ) + + return tuple(to_disable) + def from_disk( self, path: Union[str, Path], @@ -2160,7 +2232,7 @@ def _copy_examples(examples: Iterable[Example]) -> List[Example]: def _apply_pipes( - ensure_doc: Callable[[Union[str, Doc]], Doc], + ensure_doc: Callable[[Union[str, Doc, bytes], _AnyContext], Doc], pipes: Iterable[Callable[..., Iterator[Doc]]], receiver, sender, @@ -2181,17 +2253,19 @@ def _apply_pipes( Underscore.load_state(underscore_state) while True: try: - texts = receiver.get() - docs = (ensure_doc(text) for text in texts) + texts_with_ctx = receiver.get() + docs = ( + ensure_doc(doc_like, context) for doc_like, context in texts_with_ctx + ) for pipe in pipes: docs = pipe(docs) # type: ignore[arg-type, assignment] # Connection does not accept unpickable objects, so send list. byte_docs = [(doc.to_bytes(), doc._context, None) for doc in docs] - padding = [(None, None, None)] * (len(texts) - len(byte_docs)) + padding = [(None, None, None)] * (len(texts_with_ctx) - len(byte_docs)) sender.send(byte_docs + padding) # type: ignore[operator] except Exception: error_msg = [(None, None, srsly.msgpack_dumps(traceback.format_exc()))] - padding = [(None, None, None)] * (len(texts) - 1) + padding = [(None, None, None)] * (len(texts_with_ctx) - 1) sender.send(error_msg + padding) diff --git a/spacy/lookups.py b/spacy/lookups.py index b2f3dc15e..d7cc44fb3 100644 --- a/spacy/lookups.py +++ b/spacy/lookups.py @@ -85,7 +85,7 @@ class Table(OrderedDict): value: The value to set. """ key = get_string_id(key) - OrderedDict.__setitem__(self, key, value) + OrderedDict.__setitem__(self, key, value) # type:ignore[assignment] self.bloom.add(key) def set(self, key: Union[str, int], value: Any) -> None: @@ -104,7 +104,7 @@ class Table(OrderedDict): RETURNS: The value. """ key = get_string_id(key) - return OrderedDict.__getitem__(self, key) + return OrderedDict.__getitem__(self, key) # type:ignore[index] def get(self, key: Union[str, int], default: Optional[Any] = None) -> Any: """Get the value for a given key. String keys will be hashed. @@ -114,7 +114,7 @@ class Table(OrderedDict): RETURNS: The value. """ key = get_string_id(key) - return OrderedDict.get(self, key, default) + return OrderedDict.get(self, key, default) # type:ignore[arg-type] def __contains__(self, key: Union[str, int]) -> bool: # type: ignore[override] """Check whether a key is in the table. String keys will be hashed. diff --git a/spacy/matcher/dependencymatcher.pyx b/spacy/matcher/dependencymatcher.pyx index a602ba737..74c2d002f 100644 --- a/spacy/matcher/dependencymatcher.pyx +++ b/spacy/matcher/dependencymatcher.pyx @@ -82,6 +82,10 @@ cdef class DependencyMatcher: "$-": self._imm_left_sib, "$++": self._right_sib, "$--": self._left_sib, + ">++": self._right_child, + ">--": self._left_child, + "<++": self._right_parent, + "<--": self._left_parent, } def __reduce__(self): @@ -423,6 +427,22 @@ cdef class DependencyMatcher: def _left_sib(self, doc, node): return [doc[child.i] for child in doc[node].head.children if child.i < node] + def _right_child(self, doc, node): + return [doc[child.i] for child in doc[node].children if child.i > node] + + def _left_child(self, doc, node): + return [doc[child.i] for child in doc[node].children if child.i < node] + + def _right_parent(self, doc, node): + if doc[node].head.i > node: + return [doc[node].head] + return [] + + def _left_parent(self, doc, node): + if doc[node].head.i < node: + return [doc[node].head] + return [] + def _normalize_key(self, key): if isinstance(key, str): return self.vocab.strings.add(key) diff --git a/spacy/matcher/matcher.pyx b/spacy/matcher/matcher.pyx index e43583e30..5105f69ed 100644 --- a/spacy/matcher/matcher.pyx +++ b/spacy/matcher/matcher.pyx @@ -86,10 +86,14 @@ cdef class Matcher: is a dictionary mapping attribute IDs to values, and optionally a quantifier operator under the key "op". The available quantifiers are: - '!': Negate the pattern, by requiring it to match exactly 0 times. - '?': Make the pattern optional, by allowing it to match 0 or 1 times. - '+': Require the pattern to match 1 or more times. - '*': Allow the pattern to zero or more times. + '!': Negate the pattern, by requiring it to match exactly 0 times. + '?': Make the pattern optional, by allowing it to match 0 or 1 times. + '+': Require the pattern to match 1 or more times. + '*': Allow the pattern to zero or more times. + '{n}': Require the pattern to match exactly _n_ times. + '{n,m}': Require the pattern to match at least _n_ but not more than _m_ times. + '{n,}': Require the pattern to match at least _n_ times. + '{,m}': Require the pattern to match at most _m_ times. The + and * operators return all possible matches (not just the greedy ones). However, the "greedy" argument can filter the final matches @@ -786,6 +790,7 @@ def _preprocess_pattern(token_specs, vocab, extensions_table, extra_predicates): def _get_attr_values(spec, string_store): attr_values = [] for attr, value in spec.items(): + input_attr = attr if isinstance(attr, str): attr = attr.upper() if attr == '_': @@ -814,7 +819,7 @@ def _get_attr_values(spec, string_store): attr_values.append((attr, value)) else: # should be caught in validation - raise ValueError(Errors.E152.format(attr=attr)) + raise ValueError(Errors.E152.format(attr=input_attr)) return attr_values @@ -1003,8 +1008,29 @@ def _get_operators(spec): return (ONE,) elif spec["OP"] in lookup: return lookup[spec["OP"]] + #Min_max {n,m} + elif spec["OP"].startswith("{") and spec["OP"].endswith("}"): + # {n} --> {n,n} exactly n ONE,(n) + # {n,m}--> {n,m} min of n, max of m ONE,(n),ZERO_ONE,(m) + # {,m} --> {0,m} min of zero, max of m ZERO_ONE,(m) + # {n,} --> {n,∞} min of n, max of inf ONE,(n),ZERO_PLUS + + min_max = spec["OP"][1:-1] + min_max = min_max if "," in min_max else f"{min_max},{min_max}" + n, m = min_max.split(",") + + #1. Either n or m is a blank string and the other is numeric -->isdigit + #2. Both are numeric and n <= m + if (not n.isdecimal() and not m.isdecimal()) or (n.isdecimal() and m.isdecimal() and int(n) > int(m)): + keys = ", ".join(lookup.keys()) + ", {n}, {n,m}, {n,}, {,m} where n and m are integers and n <= m " + raise ValueError(Errors.E011.format(op=spec["OP"], opts=keys)) + + # if n is empty string, zero would be used + head = tuple(ONE for __ in range(int(n or 0))) + tail = tuple(ZERO_ONE for __ in range(int(m) - int(n or 0))) if m else (ZERO_PLUS,) + return head + tail else: - keys = ", ".join(lookup.keys()) + keys = ", ".join(lookup.keys()) + ", {n}, {n,m}, {n,}, {,m} where n and m are integers and n <= m " raise ValueError(Errors.E011.format(op=spec["OP"], opts=keys)) diff --git a/spacy/matcher/phrasematcher.pyx b/spacy/matcher/phrasematcher.pyx index 2ff5105ad..382029872 100644 --- a/spacy/matcher/phrasematcher.pyx +++ b/spacy/matcher/phrasematcher.pyx @@ -118,6 +118,8 @@ cdef class PhraseMatcher: # if token is not found, break out of the loop current_node = NULL break + path_nodes.push_back(current_node) + path_keys.push_back(self._terminal_hash) # remove the tokens from trie node if there are no other # keywords with them result = map_get(current_node, self._terminal_hash) diff --git a/spacy/ml/_precomputable_affine.py b/spacy/ml/_precomputable_affine.py index b99de2d2b..1c20c622b 100644 --- a/spacy/ml/_precomputable_affine.py +++ b/spacy/ml/_precomputable_affine.py @@ -22,9 +22,15 @@ def forward(model, X, is_train): nP = model.get_dim("nP") nI = model.get_dim("nI") W = model.get_param("W") - Yf = model.ops.gemm(X, W.reshape((nF * nO * nP, nI)), trans2=True) + # Preallocate array for layer output, including padding. + Yf = model.ops.alloc2f(X.shape[0] + 1, nF * nO * nP, zeros=False) + model.ops.gemm(X, W.reshape((nF * nO * nP, nI)), trans2=True, out=Yf[1:]) Yf = Yf.reshape((Yf.shape[0], nF, nO, nP)) - Yf = model.ops.xp.vstack((model.get_param("pad"), Yf)) + + # Set padding. Padding has shape (1, nF, nO, nP). Unfortunately, we cannot + # change its shape to (nF, nO, nP) without breaking existing models. So + # we'll squeeze the first dimension here. + Yf[0] = model.ops.xp.squeeze(model.get_param("pad"), 0) def backward(dY_ids): # This backprop is particularly tricky, because we get back a different diff --git a/spacy/ml/callbacks.py b/spacy/ml/callbacks.py index b0d088182..18290b947 100644 --- a/spacy/ml/callbacks.py +++ b/spacy/ml/callbacks.py @@ -1,9 +1,14 @@ -from functools import partial -from typing import Type, Callable, TYPE_CHECKING +from typing import Type, Callable, Dict, TYPE_CHECKING, List, Optional, Set +import functools +import inspect +import types +import warnings from thinc.layers import with_nvtx_range from thinc.model import Model, wrap_model_recursive +from thinc.util import use_nvtx_range +from ..errors import Warnings from ..util import registry if TYPE_CHECKING: @@ -11,29 +16,106 @@ if TYPE_CHECKING: from ..language import Language # noqa: F401 -@registry.callbacks("spacy.models_with_nvtx_range.v1") -def create_models_with_nvtx_range( - forward_color: int = -1, backprop_color: int = -1 -) -> Callable[["Language"], "Language"]: - def models_with_nvtx_range(nlp): - pipes = [ - pipe - for _, pipe in nlp.components - if hasattr(pipe, "is_trainable") and pipe.is_trainable - ] +DEFAULT_NVTX_ANNOTATABLE_PIPE_METHODS = [ + "pipe", + "predict", + "set_annotations", + "update", + "rehearse", + "get_loss", + "initialize", + "begin_update", + "finish_update", + "update", +] - # We need process all models jointly to avoid wrapping callbacks twice. - models = Model( - "wrap_with_nvtx_range", - forward=lambda model, X, is_train: ..., - layers=[pipe.model for pipe in pipes], - ) - for node in models.walk(): +def models_with_nvtx_range(nlp, forward_color: int, backprop_color: int): + pipes = [ + pipe + for _, pipe in nlp.components + if hasattr(pipe, "is_trainable") and pipe.is_trainable + ] + + seen_models: Set[int] = set() + for pipe in pipes: + for node in pipe.model.walk(): + if id(node) in seen_models: + continue + seen_models.add(id(node)) with_nvtx_range( node, forward_color=forward_color, backprop_color=backprop_color ) + return nlp + + +@registry.callbacks("spacy.models_with_nvtx_range.v1") +def create_models_with_nvtx_range( + forward_color: int = -1, backprop_color: int = -1 +) -> Callable[["Language"], "Language"]: + return functools.partial( + models_with_nvtx_range, + forward_color=forward_color, + backprop_color=backprop_color, + ) + + +def nvtx_range_wrapper_for_pipe_method(self, func, *args, **kwargs): + if isinstance(func, functools.partial): + return func(*args, **kwargs) + else: + with use_nvtx_range(f"{self.name} {func.__name__}"): + return func(*args, **kwargs) + + +def pipes_with_nvtx_range( + nlp, additional_pipe_functions: Optional[Dict[str, List[str]]] +): + for _, pipe in nlp.components: + if additional_pipe_functions: + extra_funcs = additional_pipe_functions.get(pipe.name, []) + else: + extra_funcs = [] + + for name in DEFAULT_NVTX_ANNOTATABLE_PIPE_METHODS + extra_funcs: + func = getattr(pipe, name, None) + if func is None: + if name in extra_funcs: + warnings.warn(Warnings.W121.format(method=name, pipe=pipe.name)) + continue + + wrapped_func = functools.partial( + types.MethodType(nvtx_range_wrapper_for_pipe_method, pipe), func + ) + + # Try to preserve the original function signature. + try: + wrapped_func.__signature__ = inspect.signature(func) # type: ignore + except: + pass + + try: + setattr( + pipe, + name, + wrapped_func, + ) + except AttributeError: + warnings.warn(Warnings.W122.format(method=name, pipe=pipe.name)) + + return nlp + + +@registry.callbacks("spacy.models_and_pipes_with_nvtx_range.v1") +def create_models_and_pipes_with_nvtx_range( + forward_color: int = -1, + backprop_color: int = -1, + additional_pipe_functions: Optional[Dict[str, List[str]]] = None, +) -> Callable[["Language"], "Language"]: + def inner(nlp): + nlp = models_with_nvtx_range(nlp, forward_color, backprop_color) + nlp = pipes_with_nvtx_range(nlp, additional_pipe_functions) return nlp - return models_with_nvtx_range + return inner diff --git a/spacy/ml/models/entity_linker.py b/spacy/ml/models/entity_linker.py index 0149bea89..d847342a3 100644 --- a/spacy/ml/models/entity_linker.py +++ b/spacy/ml/models/entity_linker.py @@ -23,7 +23,7 @@ def build_nel_encoder( ((tok2vec >> list2ragged()) & build_span_maker()) >> extract_spans() >> reduce_mean() - >> residual(Maxout(nO=token_width, nI=token_width, nP=2, dropout=0.0)) # type: ignore[arg-type] + >> residual(Maxout(nO=token_width, nI=token_width, nP=2, dropout=0.0)) # type: ignore >> output_layer ) model.set_ref("output_layer", output_layer) diff --git a/spacy/ml/models/parser.py b/spacy/ml/models/parser.py index 63284e766..a70d84dea 100644 --- a/spacy/ml/models/parser.py +++ b/spacy/ml/models/parser.py @@ -72,7 +72,7 @@ def build_tb_parser_model( t2v_width = tok2vec.get_dim("nO") if tok2vec.has_dim("nO") else None tok2vec = chain( tok2vec, - cast(Model[List["Floats2d"], Floats2d], list2array()), + list2array(), Linear(hidden_width, t2v_width), ) tok2vec.set_dim("nO", hidden_width) diff --git a/spacy/ml/models/textcat.py b/spacy/ml/models/textcat.py index c8c146f02..9c7e607fe 100644 --- a/spacy/ml/models/textcat.py +++ b/spacy/ml/models/textcat.py @@ -1,5 +1,5 @@ +from typing import Optional, List, cast from functools import partial -from typing import Optional, List from thinc.types import Floats2d from thinc.api import Model, reduce_mean, Linear, list2ragged, Logistic @@ -59,7 +59,8 @@ def build_simple_cnn_text_classifier( resizable_layer=resizable_layer, ) model.set_ref("tok2vec", tok2vec) - model.set_dim("nO", nO) # type: ignore # TODO: remove type ignore once Thinc has been updated + if nO is not None: + model.set_dim("nO", cast(int, nO)) model.attrs["multi_label"] = not exclusive_classes return model @@ -85,7 +86,7 @@ def build_bow_text_classifier( if not no_output_layer: fill_defaults["b"] = NEG_VALUE output_layer = softmax_activation() if exclusive_classes else Logistic() - resizable_layer = resizable( # type: ignore[var-annotated] + resizable_layer: Model[Floats2d, Floats2d] = resizable( sparse_linear, resize_layer=partial(resize_linear_weighted, fill_defaults=fill_defaults), ) @@ -93,7 +94,8 @@ def build_bow_text_classifier( model = with_cpu(model, model.ops) if output_layer: model = model >> with_cpu(output_layer, output_layer.ops) - model.set_dim("nO", nO) # type: ignore[arg-type] + if nO is not None: + model.set_dim("nO", cast(int, nO)) model.set_ref("output_layer", sparse_linear) model.attrs["multi_label"] = not exclusive_classes model.attrs["resize_output"] = partial( @@ -129,8 +131,8 @@ def build_text_classifier_v2( output_layer = Linear(nO=nO, nI=nO_double) >> Logistic() model = (linear_model | cnn_model) >> output_layer model.set_ref("tok2vec", tok2vec) - if model.has_dim("nO") is not False: - model.set_dim("nO", nO) # type: ignore[arg-type] + if model.has_dim("nO") is not False and nO is not None: + model.set_dim("nO", cast(int, nO)) model.set_ref("output_layer", linear_model.get_ref("output_layer")) model.set_ref("attention_layer", attention_layer) model.set_ref("maxout_layer", maxout_layer) @@ -164,7 +166,7 @@ def build_text_classifier_lowdata( >> list2ragged() >> ParametricAttention(width) >> reduce_sum() - >> residual(Relu(width, width)) ** 2 # type: ignore[arg-type] + >> residual(Relu(width, width)) ** 2 >> Linear(nO, width) ) if dropout: diff --git a/spacy/ml/models/tok2vec.py b/spacy/ml/models/tok2vec.py index ecdf6be27..30c7360ff 100644 --- a/spacy/ml/models/tok2vec.py +++ b/spacy/ml/models/tok2vec.py @@ -1,5 +1,5 @@ from typing import Optional, List, Union, cast -from thinc.types import Floats2d, Ints2d, Ragged +from thinc.types import Floats2d, Ints2d, Ragged, Ints1d from thinc.api import chain, clone, concatenate, with_array, with_padded from thinc.api import Model, noop, list2ragged, ragged2list, HashEmbed from thinc.api import expand_window, residual, Maxout, Mish, PyTorchLSTM @@ -159,7 +159,7 @@ def MultiHashEmbed( embeddings = [make_hash_embed(i) for i in range(len(attrs))] concat_size = width * (len(embeddings) + include_static_vectors) max_out: Model[Ragged, Ragged] = with_array( - Maxout(width, concat_size, nP=3, dropout=0.0, normalize=True) # type: ignore + Maxout(width, concat_size, nP=3, dropout=0.0, normalize=True) ) if include_static_vectors: feature_extractor: Model[List[Doc], Ragged] = chain( @@ -173,7 +173,7 @@ def MultiHashEmbed( StaticVectors(width, dropout=0.0), ), max_out, - cast(Model[Ragged, List[Floats2d]], ragged2list()), + ragged2list(), ) else: model = chain( @@ -181,7 +181,7 @@ def MultiHashEmbed( cast(Model[List[Ints2d], Ragged], list2ragged()), with_array(concatenate(*embeddings)), max_out, - cast(Model[Ragged, List[Floats2d]], ragged2list()), + ragged2list(), ) return model @@ -232,12 +232,12 @@ def CharacterEmbed( feature_extractor: Model[List[Doc], Ragged] = chain( FeatureExtractor([feature]), cast(Model[List[Ints2d], Ragged], list2ragged()), - with_array(HashEmbed(nO=width, nV=rows, column=0, seed=5)), # type: ignore + with_array(HashEmbed(nO=width, nV=rows, column=0, seed=5)), # type: ignore[misc] ) max_out: Model[Ragged, Ragged] if include_static_vectors: max_out = with_array( - Maxout(width, nM * nC + (2 * width), nP=3, normalize=True, dropout=0.0) # type: ignore + Maxout(width, nM * nC + (2 * width), nP=3, normalize=True, dropout=0.0) ) model = chain( concatenate( @@ -246,11 +246,11 @@ def CharacterEmbed( StaticVectors(width, dropout=0.0), ), max_out, - cast(Model[Ragged, List[Floats2d]], ragged2list()), + ragged2list(), ) else: max_out = with_array( - Maxout(width, nM * nC + width, nP=3, normalize=True, dropout=0.0) # type: ignore + Maxout(width, nM * nC + width, nP=3, normalize=True, dropout=0.0) ) model = chain( concatenate( @@ -258,7 +258,7 @@ def CharacterEmbed( feature_extractor, ), max_out, - cast(Model[Ragged, List[Floats2d]], ragged2list()), + ragged2list(), ) return model @@ -289,10 +289,10 @@ def MaxoutWindowEncoder( normalize=True, ), ) - model = clone(residual(cnn), depth) # type: ignore[arg-type] + model = clone(residual(cnn), depth) model.set_dim("nO", width) receptive_field = window_size * depth - return with_array(model, pad=receptive_field) # type: ignore[arg-type] + return with_array(model, pad=receptive_field) @registry.architectures("spacy.MishWindowEncoder.v2") @@ -313,9 +313,9 @@ def MishWindowEncoder( expand_window(window_size=window_size), Mish(nO=width, nI=width * ((window_size * 2) + 1), dropout=0.0, normalize=True), ) - model = clone(residual(cnn), depth) # type: ignore[arg-type] + model = clone(residual(cnn), depth) model.set_dim("nO", width) - return with_array(model) # type: ignore[arg-type] + return with_array(model) @registry.architectures("spacy.TorchBiLSTMEncoder.v1") diff --git a/spacy/ml/parser_model.pxd b/spacy/ml/parser_model.pxd index 6582b3468..8def6cea5 100644 --- a/spacy/ml/parser_model.pxd +++ b/spacy/ml/parser_model.pxd @@ -1,4 +1,5 @@ from libc.string cimport memset, memcpy +from thinc.backends.cblas cimport CBlas from ..typedefs cimport weight_t, hash_t from ..pipeline._parser_internals._state cimport StateC @@ -38,7 +39,7 @@ cdef ActivationsC alloc_activations(SizesC n) nogil cdef void free_activations(const ActivationsC* A) nogil -cdef void predict_states(ActivationsC* A, StateC** states, +cdef void predict_states(CBlas cblas, ActivationsC* A, StateC** states, const WeightsC* W, SizesC n) nogil cdef int arg_max_if_valid(const weight_t* scores, const int* is_valid, int n) nogil diff --git a/spacy/ml/parser_model.pyx b/spacy/ml/parser_model.pyx index 4e854178d..961bf4d70 100644 --- a/spacy/ml/parser_model.pyx +++ b/spacy/ml/parser_model.pyx @@ -4,11 +4,11 @@ from libc.math cimport exp from libc.string cimport memset, memcpy from libc.stdlib cimport calloc, free, realloc from thinc.backends.linalg cimport Vec, VecVec -cimport blis.cy +from thinc.backends.cblas cimport saxpy, sgemm import numpy import numpy.random -from thinc.api import Model, CupyOps, NumpyOps +from thinc.api import Model, CupyOps, NumpyOps, get_ops from .. import util from ..errors import Errors @@ -91,7 +91,7 @@ cdef void resize_activations(ActivationsC* A, SizesC n) nogil: A._curr_size = n.states -cdef void predict_states(ActivationsC* A, StateC** states, +cdef void predict_states(CBlas cblas, ActivationsC* A, StateC** states, const WeightsC* W, SizesC n) nogil: cdef double one = 1.0 resize_activations(A, n) @@ -99,7 +99,7 @@ cdef void predict_states(ActivationsC* A, StateC** states, states[i].set_context_tokens(&A.token_ids[i*n.feats], n.feats) memset(A.unmaxed, 0, n.states * n.hiddens * n.pieces * sizeof(float)) memset(A.hiddens, 0, n.states * n.hiddens * sizeof(float)) - sum_state_features(A.unmaxed, + sum_state_features(cblas, A.unmaxed, W.feat_weights, A.token_ids, n.states, n.feats, n.hiddens * n.pieces) for i in range(n.states): VecVec.add_i(&A.unmaxed[i*n.hiddens*n.pieces], @@ -113,12 +113,10 @@ cdef void predict_states(ActivationsC* A, StateC** states, memcpy(A.scores, A.hiddens, n.states * n.classes * sizeof(float)) else: # Compute hidden-to-output - blis.cy.gemm(blis.cy.NO_TRANSPOSE, blis.cy.TRANSPOSE, - n.states, n.classes, n.hiddens, one, - A.hiddens, n.hiddens, 1, - W.hidden_weights, n.hiddens, 1, - one, - A.scores, n.classes, 1) + sgemm(cblas)(False, True, n.states, n.classes, n.hiddens, + 1.0, A.hiddens, n.hiddens, + W.hidden_weights, n.hiddens, + 0.0, A.scores, n.classes) # Add bias for i in range(n.states): VecVec.add_i(&A.scores[i*n.classes], @@ -135,7 +133,7 @@ cdef void predict_states(ActivationsC* A, StateC** states, A.scores[i*n.classes+j] = min_ -cdef void sum_state_features(float* output, +cdef void sum_state_features(CBlas cblas, float* output, const float* cached, const int* token_ids, int B, int F, int O) nogil: cdef int idx, b, f, i cdef const float* feature @@ -150,9 +148,7 @@ cdef void sum_state_features(float* output, else: idx = token_ids[f] * id_stride + f*O feature = &cached[idx] - blis.cy.axpyv(blis.cy.NO_CONJUGATE, O, one, - feature, 1, - &output[b*O], 1) + saxpy(cblas)(O, one, feature, 1, &output[b*O], 1) token_ids += F @@ -443,9 +439,15 @@ cdef class precompute_hiddens: # - Output from backward on GPU bp_hiddens = self._bp_hiddens + cdef CBlas cblas + if isinstance(self.ops, CupyOps): + cblas = NUMPY_OPS.cblas() + else: + cblas = self.ops.cblas() + feat_weights = self.get_feat_weights() cdef int[:, ::1] ids = token_ids - sum_state_features(state_vector.data, + sum_state_features(cblas, state_vector.data, feat_weights, &ids[0,0], token_ids.shape[0], self.nF, self.nO*self.nP) state_vector += self.bias diff --git a/spacy/ml/staticvectors.py b/spacy/ml/staticvectors.py index 8d9b1af9b..04cfe912d 100644 --- a/spacy/ml/staticvectors.py +++ b/spacy/ml/staticvectors.py @@ -40,17 +40,15 @@ def forward( if not token_count: return _handle_empty(model.ops, model.get_dim("nO")) key_attr: int = model.attrs["key_attr"] - keys: Ints1d = model.ops.flatten( - cast(Sequence, [doc.to_array(key_attr) for doc in docs]) - ) + keys = model.ops.flatten([cast(Ints1d, doc.to_array(key_attr)) for doc in docs]) vocab: Vocab = docs[0].vocab W = cast(Floats2d, model.ops.as_contig(model.get_param("W"))) if vocab.vectors.mode == Mode.default: - V = cast(Floats2d, model.ops.asarray(vocab.vectors.data)) + V = model.ops.asarray(vocab.vectors.data) rows = vocab.vectors.find(keys=keys) V = model.ops.as_contig(V[rows]) elif vocab.vectors.mode == Mode.floret: - V = cast(Floats2d, vocab.vectors.get_batch(keys)) + V = vocab.vectors.get_batch(keys) V = model.ops.as_contig(V) else: raise RuntimeError(Errors.E896) @@ -62,9 +60,7 @@ def forward( # Convert negative indices to 0-vectors # TODO: more options for UNK tokens vectors_data[rows < 0] = 0 - output = Ragged( - vectors_data, model.ops.asarray([len(doc) for doc in docs], dtype="i") # type: ignore - ) + output = Ragged(vectors_data, model.ops.asarray1i([len(doc) for doc in docs])) mask = None if is_train: mask = _get_drop_mask(model.ops, W.shape[0], model.attrs.get("dropout_rate")) @@ -77,7 +73,9 @@ def forward( model.inc_grad( "W", model.ops.gemm( - cast(Floats2d, d_output.data), model.ops.as_contig(V), trans1=True + cast(Floats2d, d_output.data), + cast(Floats2d, model.ops.as_contig(V)), + trans1=True, ), ) return [] diff --git a/spacy/pipeline/__init__.py b/spacy/pipeline/__init__.py index 938ab08c6..26931606b 100644 --- a/spacy/pipeline/__init__.py +++ b/spacy/pipeline/__init__.py @@ -13,6 +13,7 @@ from .sentencizer import Sentencizer from .tagger import Tagger from .textcat import TextCategorizer from .spancat import SpanCategorizer +from .span_ruler import SpanRuler from .textcat_multilabel import MultiLabel_TextCategorizer from .tok2vec import Tok2Vec from .functions import merge_entities, merge_noun_chunks, merge_subtokens @@ -30,6 +31,7 @@ __all__ = [ "SentenceRecognizer", "Sentencizer", "SpanCategorizer", + "SpanRuler", "Tagger", "TextCategorizer", "Tok2Vec", diff --git a/spacy/pipeline/_parser_internals/arc_eager.pyx b/spacy/pipeline/_parser_internals/arc_eager.pyx index d60f1c3e6..257b5ef8a 100644 --- a/spacy/pipeline/_parser_internals/arc_eager.pyx +++ b/spacy/pipeline/_parser_internals/arc_eager.pyx @@ -10,6 +10,7 @@ from ...strings cimport hash_string from ...structs cimport TokenC from ...tokens.doc cimport Doc, set_children_from_heads from ...tokens.token cimport MISSING_DEP +from ...training import split_bilu_label from ...training.example cimport Example from .stateclass cimport StateClass from ._state cimport StateC, ArcC @@ -687,7 +688,7 @@ cdef class ArcEager(TransitionSystem): return self.c[name_or_id] name = name_or_id if '-' in name: - move_str, label_str = name.split('-', 1) + move_str, label_str = split_bilu_label(name) label = self.strings[label_str] else: move_str = name diff --git a/spacy/pipeline/_parser_internals/ner.pyx b/spacy/pipeline/_parser_internals/ner.pyx index 3edeff19a..fab872f00 100644 --- a/spacy/pipeline/_parser_internals/ner.pyx +++ b/spacy/pipeline/_parser_internals/ner.pyx @@ -13,6 +13,7 @@ from ...typedefs cimport weight_t, attr_t from ...lexeme cimport Lexeme from ...attrs cimport IS_SPACE from ...structs cimport TokenC, SpanC +from ...training import split_bilu_label from ...training.example cimport Example from .stateclass cimport StateClass from ._state cimport StateC @@ -182,7 +183,7 @@ cdef class BiluoPushDown(TransitionSystem): if name == '-' or name == '' or name is None: return Transition(clas=0, move=MISSING, label=0, score=0) elif '-' in name: - move_str, label_str = name.split('-', 1) + move_str, label_str = split_bilu_label(name) # Deprecated, hacky way to denote 'not this entity' if label_str.startswith('!'): raise ValueError(Errors.E869.format(label=name)) diff --git a/spacy/pipeline/_parser_internals/nonproj.hh b/spacy/pipeline/_parser_internals/nonproj.hh new file mode 100644 index 000000000..071fd57b4 --- /dev/null +++ b/spacy/pipeline/_parser_internals/nonproj.hh @@ -0,0 +1,11 @@ +#ifndef NONPROJ_HH +#define NONPROJ_HH + +#include +#include + +void raise_domain_error(std::string const &msg) { + throw std::domain_error(msg); +} + +#endif // NONPROJ_HH diff --git a/spacy/pipeline/_parser_internals/nonproj.pxd b/spacy/pipeline/_parser_internals/nonproj.pxd index e69de29bb..aabdf7ebe 100644 --- a/spacy/pipeline/_parser_internals/nonproj.pxd +++ b/spacy/pipeline/_parser_internals/nonproj.pxd @@ -0,0 +1,4 @@ +from libcpp.string cimport string + +cdef extern from "nonproj.hh": + cdef void raise_domain_error(const string& msg) nogil except + diff --git a/spacy/pipeline/_parser_internals/nonproj.pyx b/spacy/pipeline/_parser_internals/nonproj.pyx index 36163fcc3..d1b6e7066 100644 --- a/spacy/pipeline/_parser_internals/nonproj.pyx +++ b/spacy/pipeline/_parser_internals/nonproj.pyx @@ -4,10 +4,13 @@ for doing pseudo-projective parsing implementation uses the HEAD decoration scheme. """ from copy import copy +from cython.operator cimport preincrement as incr, dereference as deref from libc.limits cimport INT_MAX from libc.stdlib cimport abs from libcpp cimport bool +from libcpp.string cimport string, to_string from libcpp.vector cimport vector +from libcpp.unordered_set cimport unordered_set from ...tokens.doc cimport Doc, set_children_from_heads @@ -49,7 +52,7 @@ def is_nonproj_arc(tokenid, heads): return _is_nonproj_arc(tokenid, c_heads) -cdef bool _is_nonproj_arc(int tokenid, const vector[int]& heads) nogil: +cdef bool _is_nonproj_arc(int tokenid, const vector[int]& heads) nogil except *: # definition (e.g. Havelka 2007): an arc h -> d, h < d is non-projective # if there is a token k, h < k < d such that h is not # an ancestor of k. Same for h -> d, h > d @@ -58,32 +61,56 @@ cdef bool _is_nonproj_arc(int tokenid, const vector[int]& heads) nogil: return False elif head < 0: # unattached tokens cannot be non-projective return False - + cdef int start, end if head < tokenid: start, end = (head+1, tokenid) else: start, end = (tokenid+1, head) for k in range(start, end): - if _has_head_as_ancestor(k, head, heads): - continue - else: # head not in ancestors: d -> h is non-projective + if not _has_head_as_ancestor(k, head, heads): return True return False -cdef bool _has_head_as_ancestor(int tokenid, int head, const vector[int]& heads) nogil: +cdef bool _has_head_as_ancestor(int tokenid, int head, const vector[int]& heads) nogil except *: ancestor = tokenid - cnt = 0 - while cnt < heads.size(): + cdef unordered_set[int] seen_tokens + seen_tokens.insert(ancestor) + while True: + # Reached the head or a disconnected node if heads[ancestor] == head or heads[ancestor] < 0: return True + # Reached the root + if heads[ancestor] == ancestor: + return False ancestor = heads[ancestor] - cnt += 1 + result = seen_tokens.insert(ancestor) + # Found cycle + if not result.second: + raise_domain_error(heads_to_string(heads)) return False +cdef string heads_to_string(const vector[int]& heads) nogil: + cdef vector[int].const_iterator citer + cdef string cycle_str + + cycle_str.append("Found cycle in dependency graph: [") + + # FIXME: Rewrite using ostringstream when available in Cython. + citer = heads.const_begin() + while citer != heads.const_end(): + if citer != heads.const_begin(): + cycle_str.append(", ") + cycle_str.append(to_string(deref(citer))) + incr(citer) + cycle_str.append("]") + + return cycle_str + + def is_nonproj_tree(heads): cdef vector[int] c_heads = _heads_to_c(heads) # a tree is non-projective if at least one arc is non-projective @@ -176,11 +203,12 @@ def get_smallest_nonproj_arc_slow(heads): return _get_smallest_nonproj_arc(c_heads) -cdef int _get_smallest_nonproj_arc(const vector[int]& heads) nogil: +cdef int _get_smallest_nonproj_arc(const vector[int]& heads) nogil except -2: # return the smallest non-proj arc or None # where size is defined as the distance between dep and head # and ties are broken left to right cdef int smallest_size = INT_MAX + # -1 means its already projective. cdef int smallest_np_arc = -1 cdef int size cdef int tokenid diff --git a/spacy/pipeline/dep_parser.pyx b/spacy/pipeline/dep_parser.pyx index 50c57ee5b..e5f686158 100644 --- a/spacy/pipeline/dep_parser.pyx +++ b/spacy/pipeline/dep_parser.pyx @@ -12,6 +12,7 @@ from ..language import Language from ._parser_internals import nonproj from ._parser_internals.nonproj import DELIMITER from ..scorer import Scorer +from ..training import remove_bilu_prefix from ..util import registry @@ -314,7 +315,7 @@ cdef class DependencyParser(Parser): # Get the labels from the model by looking at the available moves for move in self.move_names: if "-" in move: - label = move.split("-")[1] + label = remove_bilu_prefix(move) if DELIMITER in label: label = label.split(DELIMITER)[1] labels.add(label) diff --git a/spacy/pipeline/edit_tree_lemmatizer.py b/spacy/pipeline/edit_tree_lemmatizer.py index 54a7030dc..b7d615f6d 100644 --- a/spacy/pipeline/edit_tree_lemmatizer.py +++ b/spacy/pipeline/edit_tree_lemmatizer.py @@ -138,7 +138,7 @@ class EditTreeLemmatizer(TrainablePipe): truths.append(eg_truths) - d_scores, loss = loss_func(scores, truths) # type: ignore + d_scores, loss = loss_func(scores, truths) if self.model.ops.xp.isnan(loss): raise ValueError(Errors.E910.format(name=self.name)) diff --git a/spacy/pipeline/entity_linker.py b/spacy/pipeline/entity_linker.py index 89e7576bf..73a90b268 100644 --- a/spacy/pipeline/entity_linker.py +++ b/spacy/pipeline/entity_linker.py @@ -56,6 +56,7 @@ DEFAULT_NEL_MODEL = Config().from_str(default_model_config)["model"] "overwrite": True, "scorer": {"@scorers": "spacy.entity_linker_scorer.v1"}, "use_gold_ents": True, + "threshold": None, }, default_score_weights={ "nel_micro_f": 1.0, @@ -77,6 +78,7 @@ def make_entity_linker( overwrite: bool, scorer: Optional[Callable], use_gold_ents: bool, + threshold: Optional[float] = None, ): """Construct an EntityLinker component. @@ -91,6 +93,10 @@ def make_entity_linker( get_candidates (Callable[[KnowledgeBase, "Span"], Iterable[Candidate]]): Function that produces a list of candidates, given a certain knowledge base and a textual mention. scorer (Optional[Callable]): The scoring method. + use_gold_ents (bool): Whether to copy entities from gold docs or not. If false, another + component must provide entity annotations. + threshold (Optional[float]): Confidence threshold for entity predictions. If confidence is below the threshold, + prediction is discarded. If None, predictions are not filtered by any threshold. """ if not model.attrs.get("include_span_maker", False): @@ -121,6 +127,7 @@ def make_entity_linker( overwrite=overwrite, scorer=scorer, use_gold_ents=use_gold_ents, + threshold=threshold, ) @@ -156,6 +163,7 @@ class EntityLinker(TrainablePipe): overwrite: bool = BACKWARD_OVERWRITE, scorer: Optional[Callable] = entity_linker_score, use_gold_ents: bool, + threshold: Optional[float] = None, ) -> None: """Initialize an entity linker. @@ -174,9 +182,20 @@ class EntityLinker(TrainablePipe): Scorer.score_links. use_gold_ents (bool): Whether to copy entities from gold docs or not. If false, another component must provide entity annotations. - + threshold (Optional[float]): Confidence threshold for entity predictions. If confidence is below the + threshold, prediction is discarded. If None, predictions are not filtered by any threshold. DOCS: https://spacy.io/api/entitylinker#init """ + + if threshold is not None and not (0 <= threshold <= 1): + raise ValueError( + Errors.E1043.format( + range_start=0, + range_end=1, + value=threshold, + ) + ) + self.vocab = vocab self.model = model self.name = name @@ -192,6 +211,7 @@ class EntityLinker(TrainablePipe): self.kb = empty_kb(entity_vector_length)(self.vocab) self.scorer = scorer self.use_gold_ents = use_gold_ents + self.threshold = threshold def set_kb(self, kb_loader: Callable[[Vocab], KnowledgeBase]): """Define the KB of this pipe by providing a function that will @@ -234,10 +254,11 @@ class EntityLinker(TrainablePipe): nO = self.kb.entity_vector_length doc_sample = [] vector_sample = [] - for example in islice(get_examples(), 10): - doc = example.x + for eg in islice(get_examples(), 10): + doc = eg.x if self.use_gold_ents: - doc.ents = example.y.ents + ents, _ = eg.get_aligned_ents_and_ner() + doc.ents = ents doc_sample.append(doc) vector_sample.append(self.model.ops.alloc1f(nO)) assert len(doc_sample) > 0, Errors.E923.format(name=self.name) @@ -312,7 +333,8 @@ class EntityLinker(TrainablePipe): for doc, ex in zip(docs, examples): if self.use_gold_ents: - doc.ents = ex.reference.ents + ents, _ = ex.get_aligned_ents_and_ner() + doc.ents = ents else: # only keep matching ents doc.ents = ex.get_matching_ents() @@ -345,7 +367,7 @@ class EntityLinker(TrainablePipe): for eg in examples: kb_ids = eg.get_aligned("ENT_KB_ID", as_string=True) - for ent in eg.reference.ents: + for ent in eg.get_matching_ents(): kb_id = kb_ids[ent.start] if kb_id: entity_encoding = self.kb.get_vector(kb_id) @@ -353,22 +375,25 @@ class EntityLinker(TrainablePipe): keep_ents.append(eidx) eidx += 1 - entity_encodings = self.model.ops.asarray(entity_encodings, dtype="float32") + entity_encodings = self.model.ops.asarray2f(entity_encodings, dtype="float32") selected_encodings = sentence_encodings[keep_ents] - # If the entity encodings list is empty, then + # if there are no matches, short circuit + if not keep_ents: + out = self.model.ops.alloc2f(*sentence_encodings.shape) + return 0, out + if selected_encodings.shape != entity_encodings.shape: err = Errors.E147.format( method="get_loss", msg="gold entities do not match up" ) raise RuntimeError(err) - # TODO: fix typing issue here - gradients = self.distance.get_grad(selected_encodings, entity_encodings) # type: ignore + gradients = self.distance.get_grad(selected_encodings, entity_encodings) # to match the input size, we need to give a zero gradient for items not in the kb out = self.model.ops.alloc2f(*sentence_encodings.shape) out[keep_ents] = gradients - loss = self.distance.get_loss(selected_encodings, entity_encodings) # type: ignore + loss = self.distance.get_loss(selected_encodings, entity_encodings) loss = loss / len(entity_encodings) return float(loss), out @@ -385,18 +410,21 @@ class EntityLinker(TrainablePipe): self.validate_kb() entity_count = 0 final_kb_ids: List[str] = [] + xp = self.model.ops.xp if not docs: return final_kb_ids if isinstance(docs, Doc): docs = [docs] for i, doc in enumerate(docs): + if len(doc) == 0: + continue sentences = [s for s in doc.sents] - if len(doc) > 0: - # Looping through each entity (TODO: rewrite) - for ent in doc.ents: - sent = ent.sent - sent_index = sentences.index(sent) - assert sent_index >= 0 + # Looping through each entity (TODO: rewrite) + for ent in doc.ents: + sent_index = sentences.index(ent.sent) + assert sent_index >= 0 + + if self.incl_context: # get n_neighbour sentences, clipped to the length of the document start_sentence = max(0, sent_index - self.n_sents) end_sentence = min(len(sentences) - 1, sent_index + self.n_sents) @@ -404,55 +432,53 @@ class EntityLinker(TrainablePipe): end_token = sentences[end_sentence].end sent_doc = doc[start_token:end_token].as_doc() # currently, the context is the same for each entity in a sentence (should be refined) - xp = self.model.ops.xp - if self.incl_context: - sentence_encoding = self.model.predict([sent_doc])[0] - sentence_encoding_t = sentence_encoding.T - sentence_norm = xp.linalg.norm(sentence_encoding_t) - entity_count += 1 - if ent.label_ in self.labels_discard: - # ignoring this entity - setting to NIL + sentence_encoding = self.model.predict([sent_doc])[0] + sentence_encoding_t = sentence_encoding.T + sentence_norm = xp.linalg.norm(sentence_encoding_t) + entity_count += 1 + if ent.label_ in self.labels_discard: + # ignoring this entity - setting to NIL + final_kb_ids.append(self.NIL) + else: + candidates = list(self.get_candidates(self.kb, ent)) + if not candidates: + # no prediction possible for this entity - setting to NIL final_kb_ids.append(self.NIL) + elif len(candidates) == 1 and self.threshold is None: + # shortcut for efficiency reasons: take the 1 candidate + final_kb_ids.append(candidates[0].entity_) else: - candidates = list(self.get_candidates(self.kb, ent)) - if not candidates: - # no prediction possible for this entity - setting to NIL - final_kb_ids.append(self.NIL) - elif len(candidates) == 1: - # shortcut for efficiency reasons: take the 1 candidate - # TODO: thresholding - final_kb_ids.append(candidates[0].entity_) - else: - random.shuffle(candidates) - # set all prior probabilities to 0 if incl_prior=False - prior_probs = xp.asarray([c.prior_prob for c in candidates]) - if not self.incl_prior: - prior_probs = xp.asarray([0.0 for _ in candidates]) - scores = prior_probs - # add in similarity from the context - if self.incl_context: - entity_encodings = xp.asarray( - [c.entity_vector for c in candidates] - ) - entity_norm = xp.linalg.norm(entity_encodings, axis=1) - if len(entity_encodings) != len(prior_probs): - raise RuntimeError( - Errors.E147.format( - method="predict", - msg="vectors not of equal length", - ) + random.shuffle(candidates) + # set all prior probabilities to 0 if incl_prior=False + prior_probs = xp.asarray([c.prior_prob for c in candidates]) + if not self.incl_prior: + prior_probs = xp.asarray([0.0 for _ in candidates]) + scores = prior_probs + # add in similarity from the context + if self.incl_context: + entity_encodings = xp.asarray( + [c.entity_vector for c in candidates] + ) + entity_norm = xp.linalg.norm(entity_encodings, axis=1) + if len(entity_encodings) != len(prior_probs): + raise RuntimeError( + Errors.E147.format( + method="predict", + msg="vectors not of equal length", ) - # cosine similarity - sims = xp.dot(entity_encodings, sentence_encoding_t) / ( - sentence_norm * entity_norm ) - if sims.shape != prior_probs.shape: - raise ValueError(Errors.E161) - scores = prior_probs + sims - (prior_probs * sims) - # TODO: thresholding - best_index = scores.argmax().item() - best_candidate = candidates[best_index] - final_kb_ids.append(best_candidate.entity_) + # cosine similarity + sims = xp.dot(entity_encodings, sentence_encoding_t) / ( + sentence_norm * entity_norm + ) + if sims.shape != prior_probs.shape: + raise ValueError(Errors.E161) + scores = prior_probs + sims - (prior_probs * sims) + final_kb_ids.append( + candidates[scores.argmax().item()].entity_ + if self.threshold is None or scores.max() >= self.threshold + else EntityLinker.NIL + ) if not (len(final_kb_ids) == entity_count): err = Errors.E147.format( method="predict", msg="result variables not of equal length" diff --git a/spacy/pipeline/entityruler.py b/spacy/pipeline/entityruler.py index 614d71f41..3cb1ca676 100644 --- a/spacy/pipeline/entityruler.py +++ b/spacy/pipeline/entityruler.py @@ -159,10 +159,8 @@ class EntityRuler(Pipe): self._require_patterns() with warnings.catch_warnings(): warnings.filterwarnings("ignore", message="\\[W036") - matches = cast( - List[Tuple[int, int, int]], - list(self.matcher(doc)) + list(self.phrase_matcher(doc)), - ) + matches = list(self.matcher(doc)) + list(self.phrase_matcher(doc)) + final_matches = set( [(m_id, start, end) for m_id, start, end in matches if start != end] ) @@ -182,10 +180,7 @@ class EntityRuler(Pipe): if start not in seen_tokens and end - 1 not in seen_tokens: if match_id in self._ent_ids: label, ent_id = self._ent_ids[match_id] - span = Span(doc, start, end, label=label) - if ent_id: - for token in span: - token.ent_id_ = ent_id + span = Span(doc, start, end, label=label, span_id=ent_id) else: span = Span(doc, start, end, label=match_id) new_entities.append(span) @@ -359,7 +354,9 @@ class EntityRuler(Pipe): (label, eid) for (label, eid) in self._ent_ids.values() if eid == ent_id ] if not label_id_pairs: - raise ValueError(Errors.E1024.format(ent_id=ent_id)) + raise ValueError( + Errors.E1024.format(attr_type="ID", label=ent_id, component=self.name) + ) created_labels = [ self._create_label(label, eid) for (label, eid) in label_id_pairs ] diff --git a/spacy/pipeline/legacy/entity_linker.py b/spacy/pipeline/legacy/entity_linker.py index 6440c18e5..2f8a1f8ea 100644 --- a/spacy/pipeline/legacy/entity_linker.py +++ b/spacy/pipeline/legacy/entity_linker.py @@ -7,7 +7,7 @@ from pathlib import Path from itertools import islice import srsly import random -from thinc.api import CosineDistance, Model, Optimizer, Config +from thinc.api import CosineDistance, Model, Optimizer from thinc.api import set_dropout_rate import warnings @@ -20,7 +20,7 @@ from ...language import Language from ...vocab import Vocab from ...training import Example, validate_examples, validate_get_examples from ...errors import Errors, Warnings -from ...util import SimpleFrozenList, registry +from ...util import SimpleFrozenList from ... import util from ...scorer import Scorer @@ -70,7 +70,6 @@ class EntityLinker_v1(TrainablePipe): produces a list of candidates, given a certain knowledge base and a textual mention. scorer (Optional[Callable]): The scoring method. Defaults to Scorer.score_links. - DOCS: https://spacy.io/api/entitylinker#init """ self.vocab = vocab @@ -213,15 +212,14 @@ class EntityLinker_v1(TrainablePipe): if kb_id: entity_encoding = self.kb.get_vector(kb_id) entity_encodings.append(entity_encoding) - entity_encodings = self.model.ops.asarray(entity_encodings, dtype="float32") + entity_encodings = self.model.ops.asarray2f(entity_encodings) if sentence_encodings.shape != entity_encodings.shape: err = Errors.E147.format( method="get_loss", msg="gold entities do not match up" ) raise RuntimeError(err) - # TODO: fix typing issue here - gradients = self.distance.get_grad(sentence_encodings, entity_encodings) # type: ignore - loss = self.distance.get_loss(sentence_encodings, entity_encodings) # type: ignore + gradients = self.distance.get_grad(sentence_encodings, entity_encodings) + loss = self.distance.get_loss(sentence_encodings, entity_encodings) loss = loss / len(entity_encodings) return float(loss), gradients @@ -273,7 +271,6 @@ class EntityLinker_v1(TrainablePipe): final_kb_ids.append(self.NIL) elif len(candidates) == 1: # shortcut for efficiency reasons: take the 1 candidate - # TODO: thresholding final_kb_ids.append(candidates[0].entity_) else: random.shuffle(candidates) @@ -302,7 +299,6 @@ class EntityLinker_v1(TrainablePipe): if sims.shape != prior_probs.shape: raise ValueError(Errors.E161) scores = prior_probs + sims - (prior_probs * sims) - # TODO: thresholding best_index = scores.argmax().item() best_candidate = candidates[best_index] final_kb_ids.append(best_candidate.entity_) diff --git a/spacy/pipeline/ner.pyx b/spacy/pipeline/ner.pyx index 4835a8c4b..25f48c9f8 100644 --- a/spacy/pipeline/ner.pyx +++ b/spacy/pipeline/ner.pyx @@ -6,10 +6,10 @@ from thinc.api import Model, Config from ._parser_internals.transition_system import TransitionSystem from .transition_parser cimport Parser from ._parser_internals.ner cimport BiluoPushDown - from ..language import Language from ..scorer import get_ner_prf, PRFScore from ..util import registry +from ..training import remove_bilu_prefix default_model_config = """ @@ -242,7 +242,7 @@ cdef class EntityRecognizer(Parser): def labels(self): # Get the labels from the model by looking at the available moves, e.g. # B-PERSON, I-PERSON, L-PERSON, U-PERSON - labels = set(move.split("-")[1] for move in self.move_names + labels = set(remove_bilu_prefix(move) for move in self.move_names if move[0] in ("B", "I", "L", "U")) return tuple(sorted(labels)) diff --git a/spacy/pipeline/pipe.pyx b/spacy/pipeline/pipe.pyx index d24e4d574..4e3ae1cf0 100644 --- a/spacy/pipeline/pipe.pyx +++ b/spacy/pipeline/pipe.pyx @@ -31,7 +31,7 @@ cdef class Pipe: and returned. This usually happens under the hood when the nlp object is called on a text and all components are applied to the Doc. - docs (Doc): The Doc to process. + doc (Doc): The Doc to process. RETURNS (Doc): The processed Doc. DOCS: https://spacy.io/api/pipe#call diff --git a/spacy/pipeline/span_ruler.py b/spacy/pipeline/span_ruler.py new file mode 100644 index 000000000..807a4ffe5 --- /dev/null +++ b/spacy/pipeline/span_ruler.py @@ -0,0 +1,569 @@ +from typing import Optional, Union, List, Dict, Tuple, Iterable, Any, Callable +from typing import Sequence, Set, cast +import warnings +from functools import partial +from pathlib import Path +import srsly + +from .pipe import Pipe +from ..training import Example +from ..language import Language +from ..errors import Errors, Warnings +from ..util import ensure_path, SimpleFrozenList, registry +from ..tokens import Doc, Span +from ..scorer import Scorer +from ..matcher import Matcher, PhraseMatcher +from .. import util + +PatternType = Dict[str, Union[str, List[Dict[str, Any]]]] +DEFAULT_SPANS_KEY = "ruler" + + +@Language.factory( + "future_entity_ruler", + assigns=["doc.ents"], + default_config={ + "phrase_matcher_attr": None, + "validate": False, + "overwrite_ents": False, + "scorer": {"@scorers": "spacy.entity_ruler_scorer.v1"}, + "ent_id_sep": "__unused__", + }, + default_score_weights={ + "ents_f": 1.0, + "ents_p": 0.0, + "ents_r": 0.0, + "ents_per_type": None, + }, +) +def make_entity_ruler( + nlp: Language, + name: str, + phrase_matcher_attr: Optional[Union[int, str]], + validate: bool, + overwrite_ents: bool, + scorer: Optional[Callable], + ent_id_sep: str, +): + if overwrite_ents: + ents_filter = prioritize_new_ents_filter + else: + ents_filter = prioritize_existing_ents_filter + return SpanRuler( + nlp, + name, + spans_key=None, + spans_filter=None, + annotate_ents=True, + ents_filter=ents_filter, + phrase_matcher_attr=phrase_matcher_attr, + validate=validate, + overwrite=False, + scorer=scorer, + ) + + +@Language.factory( + "span_ruler", + assigns=["doc.spans"], + default_config={ + "spans_key": DEFAULT_SPANS_KEY, + "spans_filter": None, + "annotate_ents": False, + "ents_filter": {"@misc": "spacy.first_longest_spans_filter.v1"}, + "phrase_matcher_attr": None, + "validate": False, + "overwrite": True, + "scorer": { + "@scorers": "spacy.overlapping_labeled_spans_scorer.v1", + "spans_key": DEFAULT_SPANS_KEY, + }, + }, + default_score_weights={ + f"spans_{DEFAULT_SPANS_KEY}_f": 1.0, + f"spans_{DEFAULT_SPANS_KEY}_p": 0.0, + f"spans_{DEFAULT_SPANS_KEY}_r": 0.0, + f"spans_{DEFAULT_SPANS_KEY}_per_type": None, + }, +) +def make_span_ruler( + nlp: Language, + name: str, + spans_key: Optional[str], + spans_filter: Optional[Callable[[Iterable[Span], Iterable[Span]], Iterable[Span]]], + annotate_ents: bool, + ents_filter: Callable[[Iterable[Span], Iterable[Span]], Iterable[Span]], + phrase_matcher_attr: Optional[Union[int, str]], + validate: bool, + overwrite: bool, + scorer: Optional[Callable], +): + return SpanRuler( + nlp, + name, + spans_key=spans_key, + spans_filter=spans_filter, + annotate_ents=annotate_ents, + ents_filter=ents_filter, + phrase_matcher_attr=phrase_matcher_attr, + validate=validate, + overwrite=overwrite, + scorer=scorer, + ) + + +def prioritize_new_ents_filter( + entities: Iterable[Span], spans: Iterable[Span] +) -> List[Span]: + """Merge entities and spans into one list without overlaps by allowing + spans to overwrite any entities that they overlap with. Intended to + replicate the overwrite_ents=True behavior from the EntityRuler. + + entities (Iterable[Span]): The entities, already filtered for overlaps. + spans (Iterable[Span]): The spans to merge, may contain overlaps. + RETURNS (List[Span]): Filtered list of non-overlapping spans. + """ + get_sort_key = lambda span: (span.end - span.start, -span.start) + spans = sorted(spans, key=get_sort_key, reverse=True) + entities = list(entities) + new_entities = [] + seen_tokens: Set[int] = set() + for span in spans: + start = span.start + end = span.end + if all(token.i not in seen_tokens for token in span): + new_entities.append(span) + entities = [e for e in entities if not (e.start < end and e.end > start)] + seen_tokens.update(range(start, end)) + return entities + new_entities + + +@registry.misc("spacy.prioritize_new_ents_filter.v1") +def make_prioritize_new_ents_filter(): + return prioritize_new_ents_filter + + +def prioritize_existing_ents_filter( + entities: Iterable[Span], spans: Iterable[Span] +) -> List[Span]: + """Merge entities and spans into one list without overlaps by prioritizing + existing entities. Intended to replicate the overwrite_ents=False behavior + from the EntityRuler. + + entities (Iterable[Span]): The entities, already filtered for overlaps. + spans (Iterable[Span]): The spans to merge, may contain overlaps. + RETURNS (List[Span]): Filtered list of non-overlapping spans. + """ + get_sort_key = lambda span: (span.end - span.start, -span.start) + spans = sorted(spans, key=get_sort_key, reverse=True) + entities = list(entities) + new_entities = [] + seen_tokens: Set[int] = set() + seen_tokens.update(*(range(ent.start, ent.end) for ent in entities)) + for span in spans: + start = span.start + end = span.end + if all(token.i not in seen_tokens for token in span): + new_entities.append(span) + seen_tokens.update(range(start, end)) + return entities + new_entities + + +@registry.misc("spacy.prioritize_existing_ents_filter.v1") +def make_preverse_existing_ents_filter(): + return prioritize_existing_ents_filter + + +def overlapping_labeled_spans_score( + examples: Iterable[Example], *, spans_key=DEFAULT_SPANS_KEY, **kwargs +) -> Dict[str, Any]: + kwargs = dict(kwargs) + attr_prefix = f"spans_" + kwargs.setdefault("attr", f"{attr_prefix}{spans_key}") + kwargs.setdefault("allow_overlap", True) + kwargs.setdefault("labeled", True) + kwargs.setdefault( + "getter", lambda doc, key: doc.spans.get(key[len(attr_prefix) :], []) + ) + kwargs.setdefault("has_annotation", lambda doc: spans_key in doc.spans) + return Scorer.score_spans(examples, **kwargs) + + +@registry.scorers("spacy.overlapping_labeled_spans_scorer.v1") +def make_overlapping_labeled_spans_scorer(spans_key: str = DEFAULT_SPANS_KEY): + return partial(overlapping_labeled_spans_score, spans_key=spans_key) + + +class SpanRuler(Pipe): + """The SpanRuler lets you add spans to the `Doc.spans` using token-based + rules or exact phrase matches. + + DOCS: https://spacy.io/api/spanruler + USAGE: https://spacy.io/usage/rule-based-matching#spanruler + """ + + def __init__( + self, + nlp: Language, + name: str = "span_ruler", + *, + spans_key: Optional[str] = DEFAULT_SPANS_KEY, + spans_filter: Optional[ + Callable[[Iterable[Span], Iterable[Span]], Iterable[Span]] + ] = None, + annotate_ents: bool = False, + ents_filter: Callable[ + [Iterable[Span], Iterable[Span]], Iterable[Span] + ] = util.filter_chain_spans, + phrase_matcher_attr: Optional[Union[int, str]] = None, + validate: bool = False, + overwrite: bool = False, + scorer: Optional[Callable] = partial( + overlapping_labeled_spans_score, spans_key=DEFAULT_SPANS_KEY + ), + ) -> None: + """Initialize the span ruler. If patterns are supplied here, they + need to be a list of dictionaries with a `"label"` and `"pattern"` + key. A pattern can either be a token pattern (list) or a phrase pattern + (string). For example: `{'label': 'ORG', 'pattern': 'Apple'}`. + + nlp (Language): The shared nlp object to pass the vocab to the matchers + and process phrase patterns. + name (str): Instance name of the current pipeline component. Typically + passed in automatically from the factory when the component is + added. Used to disable the current span ruler while creating + phrase patterns with the nlp object. + spans_key (Optional[str]): The spans key to save the spans under. If + `None`, no spans are saved. Defaults to "ruler". + spans_filter (Optional[Callable[[Iterable[Span], Iterable[Span]], List[Span]]): + The optional method to filter spans before they are assigned to + doc.spans. Defaults to `None`. + annotate_ents (bool): Whether to save spans to doc.ents. Defaults to + `False`. + ents_filter (Callable[[Iterable[Span], Iterable[Span]], List[Span]]): + The method to filter spans before they are assigned to doc.ents. + Defaults to `util.filter_chain_spans`. + phrase_matcher_attr (Optional[Union[int, str]]): Token attribute to + match on, passed to the internal PhraseMatcher as `attr`. Defaults + to `None`. + validate (bool): Whether patterns should be validated, passed to + Matcher and PhraseMatcher as `validate`. + overwrite (bool): Whether to remove any existing spans under this spans + key if `spans_key` is set, and/or to remove any ents under `doc.ents` if + `annotate_ents` is set. Defaults to `True`. + scorer (Optional[Callable]): The scoring method. Defaults to + spacy.pipeline.span_ruler.overlapping_labeled_spans_score. + + DOCS: https://spacy.io/api/spanruler#init + """ + self.nlp = nlp + self.name = name + self.spans_key = spans_key + self.annotate_ents = annotate_ents + self.phrase_matcher_attr = phrase_matcher_attr + self.validate = validate + self.overwrite = overwrite + self.spans_filter = spans_filter + self.ents_filter = ents_filter + self.scorer = scorer + self._match_label_id_map: Dict[int, Dict[str, str]] = {} + self.clear() + + def __len__(self) -> int: + """The number of all labels added to the span ruler.""" + return len(self._patterns) + + def __contains__(self, label: str) -> bool: + """Whether a label is present in the patterns.""" + for label_id in self._match_label_id_map.values(): + if label_id["label"] == label: + return True + return False + + @property + def key(self) -> Optional[str]: + """Key of the doc.spans dict to save the spans under.""" + return self.spans_key + + def __call__(self, doc: Doc) -> Doc: + """Find matches in document and add them as entities. + + doc (Doc): The Doc object in the pipeline. + RETURNS (Doc): The Doc with added entities, if available. + + DOCS: https://spacy.io/api/spanruler#call + """ + error_handler = self.get_error_handler() + try: + matches = self.match(doc) + self.set_annotations(doc, matches) + return doc + except Exception as e: + return error_handler(self.name, self, [doc], e) + + def match(self, doc: Doc): + self._require_patterns() + with warnings.catch_warnings(): + warnings.filterwarnings("ignore", message="\\[W036") + matches = cast( + List[Tuple[int, int, int]], + list(self.matcher(doc)) + list(self.phrase_matcher(doc)), + ) + deduplicated_matches = set( + Span( + doc, + start, + end, + label=self._match_label_id_map[m_id]["label"], + span_id=self._match_label_id_map[m_id]["id"], + ) + for m_id, start, end in matches + if start != end + ) + return sorted(list(deduplicated_matches)) + + def set_annotations(self, doc, matches): + """Modify the document in place""" + # set doc.spans if spans_key is set + if self.key: + spans = [] + if self.key in doc.spans and not self.overwrite: + spans = doc.spans[self.key] + spans.extend( + self.spans_filter(spans, matches) if self.spans_filter else matches + ) + doc.spans[self.key] = spans + # set doc.ents if annotate_ents is set + if self.annotate_ents: + spans = [] + if not self.overwrite: + spans = list(doc.ents) + spans = self.ents_filter(spans, matches) + try: + doc.ents = sorted(spans) + except ValueError: + raise ValueError(Errors.E854) + + @property + def labels(self) -> Tuple[str, ...]: + """All labels present in the match patterns. + + RETURNS (set): The string labels. + + DOCS: https://spacy.io/api/spanruler#labels + """ + return tuple(sorted(set([cast(str, p["label"]) for p in self._patterns]))) + + @property + def ids(self) -> Tuple[str, ...]: + """All IDs present in the match patterns. + + RETURNS (set): The string IDs. + + DOCS: https://spacy.io/api/spanruler#ids + """ + return tuple( + sorted(set([cast(str, p.get("id")) for p in self._patterns]) - set([None])) + ) + + def initialize( + self, + get_examples: Callable[[], Iterable[Example]], + *, + nlp: Optional[Language] = None, + patterns: Optional[Sequence[PatternType]] = None, + ): + """Initialize the pipe for training. + + get_examples (Callable[[], Iterable[Example]]): Function that + returns a representative sample of gold-standard Example objects. + nlp (Language): The current nlp object the component is part of. + patterns (Optional[Iterable[PatternType]]): The list of patterns. + + DOCS: https://spacy.io/api/spanruler#initialize + """ + self.clear() + if patterns: + self.add_patterns(patterns) # type: ignore[arg-type] + + @property + def patterns(self) -> List[PatternType]: + """Get all patterns that were added to the span ruler. + + RETURNS (list): The original patterns, one dictionary per pattern. + + DOCS: https://spacy.io/api/spanruler#patterns + """ + return self._patterns + + def add_patterns(self, patterns: List[PatternType]) -> None: + """Add patterns to the span ruler. A pattern can either be a token + pattern (list of dicts) or a phrase pattern (string). For example: + {'label': 'ORG', 'pattern': 'Apple'} + {'label': 'ORG', 'pattern': 'Apple', 'id': 'apple'} + {'label': 'GPE', 'pattern': [{'lower': 'san'}, {'lower': 'francisco'}]} + + patterns (list): The patterns to add. + + DOCS: https://spacy.io/api/spanruler#add_patterns + """ + + # disable the nlp components after this one in case they haven't been + # initialized / deserialized yet + try: + current_index = -1 + for i, (name, pipe) in enumerate(self.nlp.pipeline): + if self == pipe: + current_index = i + break + subsequent_pipes = [pipe for pipe in self.nlp.pipe_names[current_index:]] + except ValueError: + subsequent_pipes = [] + with self.nlp.select_pipes(disable=subsequent_pipes): + phrase_pattern_labels = [] + phrase_pattern_texts = [] + for entry in patterns: + p_label = cast(str, entry["label"]) + p_id = cast(str, entry.get("id", "")) + label = repr((p_label, p_id)) + self._match_label_id_map[self.nlp.vocab.strings.as_int(label)] = { + "label": p_label, + "id": p_id, + } + if isinstance(entry["pattern"], str): + phrase_pattern_labels.append(label) + phrase_pattern_texts.append(entry["pattern"]) + elif isinstance(entry["pattern"], list): + self.matcher.add(label, [entry["pattern"]]) + else: + raise ValueError(Errors.E097.format(pattern=entry["pattern"])) + self._patterns.append(entry) + for label, pattern in zip( + phrase_pattern_labels, + self.nlp.pipe(phrase_pattern_texts), + ): + self.phrase_matcher.add(label, [pattern]) + + def clear(self) -> None: + """Reset all patterns. + + RETURNS: None + DOCS: https://spacy.io/api/spanruler#clear + """ + self._patterns: List[PatternType] = [] + self.matcher: Matcher = Matcher(self.nlp.vocab, validate=self.validate) + self.phrase_matcher: PhraseMatcher = PhraseMatcher( + self.nlp.vocab, + attr=self.phrase_matcher_attr, + validate=self.validate, + ) + + def remove(self, label: str) -> None: + """Remove a pattern by its label. + + label (str): Label of the pattern to be removed. + RETURNS: None + DOCS: https://spacy.io/api/spanruler#remove + """ + if label not in self: + raise ValueError( + Errors.E1024.format(attr_type="label", label=label, component=self.name) + ) + self._patterns = [p for p in self._patterns if p["label"] != label] + for m_label in self._match_label_id_map: + if self._match_label_id_map[m_label]["label"] == label: + m_label_str = self.nlp.vocab.strings.as_string(m_label) + if m_label_str in self.phrase_matcher: + self.phrase_matcher.remove(m_label_str) + if m_label_str in self.matcher: + self.matcher.remove(m_label_str) + + def remove_by_id(self, pattern_id: str) -> None: + """Remove a pattern by its pattern ID. + + pattern_id (str): ID of the pattern to be removed. + RETURNS: None + DOCS: https://spacy.io/api/spanruler#remove_by_id + """ + orig_len = len(self) + self._patterns = [p for p in self._patterns if p.get("id") != pattern_id] + if orig_len == len(self): + raise ValueError( + Errors.E1024.format( + attr_type="ID", label=pattern_id, component=self.name + ) + ) + for m_label in self._match_label_id_map: + if self._match_label_id_map[m_label]["id"] == pattern_id: + m_label_str = self.nlp.vocab.strings.as_string(m_label) + if m_label_str in self.phrase_matcher: + self.phrase_matcher.remove(m_label_str) + if m_label_str in self.matcher: + self.matcher.remove(m_label_str) + + def _require_patterns(self) -> None: + """Raise a warning if this component has no patterns defined.""" + if len(self) == 0: + warnings.warn(Warnings.W036.format(name=self.name)) + + def from_bytes( + self, bytes_data: bytes, *, exclude: Iterable[str] = SimpleFrozenList() + ) -> "SpanRuler": + """Load the span ruler from a bytestring. + + bytes_data (bytes): The bytestring to load. + RETURNS (SpanRuler): The loaded span ruler. + + DOCS: https://spacy.io/api/spanruler#from_bytes + """ + self.clear() + deserializers = { + "patterns": lambda b: self.add_patterns(srsly.json_loads(b)), + } + util.from_bytes(bytes_data, deserializers, exclude) + return self + + def to_bytes(self, *, exclude: Iterable[str] = SimpleFrozenList()) -> bytes: + """Serialize the span ruler to a bytestring. + + RETURNS (bytes): The serialized patterns. + + DOCS: https://spacy.io/api/spanruler#to_bytes + """ + serializers = { + "patterns": lambda: srsly.json_dumps(self.patterns), + } + return util.to_bytes(serializers, exclude) + + def from_disk( + self, path: Union[str, Path], *, exclude: Iterable[str] = SimpleFrozenList() + ) -> "SpanRuler": + """Load the span ruler from a directory. + + path (Union[str, Path]): A path to a directory. + RETURNS (SpanRuler): The loaded span ruler. + + DOCS: https://spacy.io/api/spanruler#from_disk + """ + self.clear() + path = ensure_path(path) + deserializers = { + "patterns": lambda p: self.add_patterns(srsly.read_jsonl(p)), + } + util.from_disk(path, deserializers, {}) + return self + + def to_disk( + self, path: Union[str, Path], *, exclude: Iterable[str] = SimpleFrozenList() + ) -> None: + """Save the span ruler patterns to a directory. + + path (Union[str, Path]): A path to a directory. + + DOCS: https://spacy.io/api/spanruler#to_disk + """ + path = ensure_path(path) + serializers = { + "patterns": lambda p: srsly.write_jsonl(p, self.patterns), + } + util.to_disk(path, serializers, {}) diff --git a/spacy/pipeline/spancat.py b/spacy/pipeline/spancat.py index 0a6138fbc..1b7a9eecb 100644 --- a/spacy/pipeline/spancat.py +++ b/spacy/pipeline/spancat.py @@ -75,7 +75,7 @@ def build_ngram_suggester(sizes: List[int]) -> Suggester: if spans: assert spans[-1].ndim == 2, spans[-1].shape lengths.append(length) - lengths_array = cast(Ints1d, ops.asarray(lengths, dtype="i")) + lengths_array = ops.asarray1i(lengths) if len(spans) > 0: output = Ragged(ops.xp.vstack(spans), lengths_array) else: diff --git a/spacy/pipeline/textcat.py b/spacy/pipeline/textcat.py index bc3f127fc..c45f819fc 100644 --- a/spacy/pipeline/textcat.py +++ b/spacy/pipeline/textcat.py @@ -192,7 +192,7 @@ class TextCategorizer(TrainablePipe): if not any(len(doc) for doc in docs): # Handle cases where there are no tokens in any docs. tensors = [doc.tensor for doc in docs] - xp = get_array_module(tensors) + xp = self.model.ops.xp scores = xp.zeros((len(list(docs)), len(self.labels))) return scores scores = self.model.predict(docs) diff --git a/spacy/pipeline/transition_parser.pxd b/spacy/pipeline/transition_parser.pxd index bd5bad334..1521fde60 100644 --- a/spacy/pipeline/transition_parser.pxd +++ b/spacy/pipeline/transition_parser.pxd @@ -1,4 +1,5 @@ from cymem.cymem cimport Pool +from thinc.backends.cblas cimport CBlas from ..vocab cimport Vocab from .trainable_pipe cimport TrainablePipe @@ -12,7 +13,7 @@ cdef class Parser(TrainablePipe): cdef readonly TransitionSystem moves cdef public object _multitasks - cdef void _parseC(self, StateC** states, + cdef void _parseC(self, CBlas cblas, StateC** states, WeightsC weights, SizesC sizes) nogil cdef void c_transition_batch(self, StateC** states, const float* scores, diff --git a/spacy/pipeline/transition_parser.pyx b/spacy/pipeline/transition_parser.pyx index 2571af102..1327db2ce 100644 --- a/spacy/pipeline/transition_parser.pyx +++ b/spacy/pipeline/transition_parser.pyx @@ -9,7 +9,7 @@ from libc.stdlib cimport calloc, free import random import srsly -from thinc.api import set_dropout_rate, CupyOps +from thinc.api import get_ops, set_dropout_rate, CupyOps, NumpyOps from thinc.extra.search cimport Beam import numpy.random import numpy @@ -30,6 +30,9 @@ from ..errors import Errors, Warnings from .. import util +NUMPY_OPS = NumpyOps() + + cdef class Parser(TrainablePipe): """ Base class of the DependencyParser and EntityRecognizer. @@ -259,6 +262,12 @@ cdef class Parser(TrainablePipe): def greedy_parse(self, docs, drop=0.): cdef vector[StateC*] states cdef StateClass state + ops = self.model.ops + cdef CBlas cblas + if isinstance(ops, CupyOps): + cblas = NUMPY_OPS.cblas() + else: + cblas = ops.cblas() self._ensure_labels_are_added(docs) set_dropout_rate(self.model, drop) batch = self.moves.init_batch(docs) @@ -269,8 +278,7 @@ cdef class Parser(TrainablePipe): states.push_back(state.c) sizes = get_c_sizes(model, states.size()) with nogil: - self._parseC(&states[0], - weights, sizes) + self._parseC(cblas, &states[0], weights, sizes) model.clear_memory() del model return batch @@ -297,14 +305,13 @@ cdef class Parser(TrainablePipe): del model return list(batch) - cdef void _parseC(self, StateC** states, + cdef void _parseC(self, CBlas cblas, StateC** states, WeightsC weights, SizesC sizes) nogil: cdef int i, j cdef vector[StateC*] unfinished cdef ActivationsC activations = alloc_activations(sizes) while sizes.states >= 1: - predict_states(&activations, - states, &weights, sizes) + predict_states(cblas, &activations, states, &weights, sizes) # Validate actions, argmax, take action. self.c_transition_batch(states, activations.scores, sizes.classes, sizes.states) diff --git a/spacy/schemas.py b/spacy/schemas.py index 1dfd8ee85..658e45268 100644 --- a/spacy/schemas.py +++ b/spacy/schemas.py @@ -3,12 +3,13 @@ from typing import Iterable, TypeVar, TYPE_CHECKING from .compat import Literal from enum import Enum from pydantic import BaseModel, Field, ValidationError, validator, create_model -from pydantic import StrictStr, StrictInt, StrictFloat, StrictBool +from pydantic import StrictStr, StrictInt, StrictFloat, StrictBool, ConstrainedStr from pydantic.main import ModelMetaclass from thinc.api import Optimizer, ConfigValidationError, Model from thinc.config import Promise from collections import defaultdict import inspect +import re from .attrs import NAMES from .lookups import Lookups @@ -104,7 +105,7 @@ def get_arg_model( sig_args[param.name] = (annotation, default) is_strict = strict and not has_variable sig_args["__config__"] = ArgSchemaConfig if is_strict else ArgSchemaConfigExtra # type: ignore[assignment] - return create_model(name, **sig_args) # type: ignore[arg-type, return-value] + return create_model(name, **sig_args) # type: ignore[call-overload, arg-type, return-value] def validate_init_settings( @@ -198,13 +199,18 @@ class TokenPatternNumber(BaseModel): return v -class TokenPatternOperator(str, Enum): +class TokenPatternOperatorSimple(str, Enum): plus: StrictStr = StrictStr("+") - start: StrictStr = StrictStr("*") + star: StrictStr = StrictStr("*") question: StrictStr = StrictStr("?") exclamation: StrictStr = StrictStr("!") +class TokenPatternOperatorMinMax(ConstrainedStr): + regex = re.compile("^({\d+}|{\d+,\d*}|{\d*,\d+})$") + + +TokenPatternOperator = Union[TokenPatternOperatorSimple, TokenPatternOperatorMinMax] StringValue = Union[TokenPatternString, StrictStr] NumberValue = Union[TokenPatternNumber, StrictInt, StrictFloat] UnderscoreValue = Union[ @@ -485,3 +491,29 @@ class RecommendationSchema(BaseModel): word_vectors: Optional[str] = None transformer: Optional[RecommendationTrf] = None has_letters: bool = True + + +class DocJSONSchema(BaseModel): + """ + JSON/dict format for JSON representation of Doc objects. + """ + + cats: Optional[Dict[StrictStr, StrictFloat]] = Field( + None, title="Categories with corresponding probabilities" + ) + ents: Optional[List[Dict[StrictStr, Union[StrictInt, StrictStr]]]] = Field( + None, title="Information on entities" + ) + sents: Optional[List[Dict[StrictStr, StrictInt]]] = Field( + None, title="Indices of sentences' start and end indices" + ) + text: StrictStr = Field(..., title="Document text") + spans: Dict[StrictStr, List[Dict[StrictStr, Union[StrictStr, StrictInt]]]] = Field( + None, title="Span information - end/start indices, label, KB ID" + ) + tokens: List[Dict[StrictStr, Union[StrictStr, StrictInt]]] = Field( + ..., title="Token information - ID, start, annotations" + ) + _: Optional[Dict[StrictStr, Any]] = Field( + None, title="Any custom data stored in the document's _ attribute" + ) diff --git a/spacy/strings.pxd b/spacy/strings.pxd index 370180135..5f03a9a28 100644 --- a/spacy/strings.pxd +++ b/spacy/strings.pxd @@ -26,4 +26,4 @@ cdef class StringStore: cdef public PreshMap _map cdef const Utf8Str* intern_unicode(self, str py_string) - cdef const Utf8Str* _intern_utf8(self, char* utf8_string, int length) + cdef const Utf8Str* _intern_utf8(self, char* utf8_string, int length, hash_t* precalculated_hash) diff --git a/spacy/strings.pyx b/spacy/strings.pyx index 39fc441e9..c5f218342 100644 --- a/spacy/strings.pyx +++ b/spacy/strings.pyx @@ -14,6 +14,13 @@ from .symbols import NAMES as SYMBOLS_BY_INT from .errors import Errors from . import util +# Not particularly elegant, but this is faster than `isinstance(key, numbers.Integral)` +cdef inline bint _try_coerce_to_hash(object key, hash_t* out_hash): + try: + out_hash[0] = key + return True + except: + return False def get_string_id(key): """Get a string ID, handling the reserved symbols correctly. If the key is @@ -22,15 +29,27 @@ def get_string_id(key): This function optimises for convenience over performance, so shouldn't be used in tight loops. """ - if not isinstance(key, str): - return key - elif key in SYMBOLS_BY_STR: - return SYMBOLS_BY_STR[key] - elif not key: - return 0 + cdef hash_t str_hash + if isinstance(key, str): + if len(key) == 0: + return 0 + + symbol = SYMBOLS_BY_STR.get(key, None) + if symbol is not None: + return symbol + else: + chars = key.encode("utf8") + return hash_utf8(chars, len(chars)) + elif _try_coerce_to_hash(key, &str_hash): + # Coerce the integral key to the expected primitive hash type. + # This ensures that custom/overloaded "primitive" data types + # such as those implemented by numpy are not inadvertently used + # downsteam (as these are internally implemented as custom PyObjects + # whose comparison operators can incur a significant overhead). + return str_hash else: - chars = key.encode("utf8") - return hash_utf8(chars, len(chars)) + # TODO: Raise an error instead + return key cpdef hash_t hash_string(str string) except 0: @@ -110,28 +129,36 @@ cdef class StringStore: string_or_id (bytes, str or uint64): The value to encode. Returns (str / uint64): The value to be retrieved. """ - if isinstance(string_or_id, str) and len(string_or_id) == 0: - return 0 - elif string_or_id == 0: - return "" - elif string_or_id in SYMBOLS_BY_STR: - return SYMBOLS_BY_STR[string_or_id] - cdef hash_t key + cdef hash_t str_hash + cdef Utf8Str* utf8str = NULL + if isinstance(string_or_id, str): - key = hash_string(string_or_id) - return key - elif isinstance(string_or_id, bytes): - key = hash_utf8(string_or_id, len(string_or_id)) - return key - elif string_or_id < len(SYMBOLS_BY_INT): - return SYMBOLS_BY_INT[string_or_id] - else: - key = string_or_id - utf8str = self._map.get(key) - if utf8str is NULL: - raise KeyError(Errors.E018.format(hash_value=string_or_id)) + if len(string_or_id) == 0: + return 0 + + # Return early if the string is found in the symbols LUT. + symbol = SYMBOLS_BY_STR.get(string_or_id, None) + if symbol is not None: + return symbol else: - return decode_Utf8Str(utf8str) + return hash_string(string_or_id) + elif isinstance(string_or_id, bytes): + return hash_utf8(string_or_id, len(string_or_id)) + elif _try_coerce_to_hash(string_or_id, &str_hash): + if str_hash == 0: + return "" + elif str_hash < len(SYMBOLS_BY_INT): + return SYMBOLS_BY_INT[str_hash] + else: + utf8str = self._map.get(str_hash) + else: + # TODO: Raise an error instead + utf8str = self._map.get(string_or_id) + + if utf8str is NULL: + raise KeyError(Errors.E018.format(hash_value=string_or_id)) + else: + return decode_Utf8Str(utf8str) def as_int(self, key): """If key is an int, return it; otherwise, get the int value.""" @@ -153,19 +180,22 @@ cdef class StringStore: string (str): The string to add. RETURNS (uint64): The string's hash value. """ + cdef hash_t str_hash if isinstance(string, str): if string in SYMBOLS_BY_STR: return SYMBOLS_BY_STR[string] - key = hash_string(string) - self.intern_unicode(string) + + string = string.encode("utf8") + str_hash = hash_utf8(string, len(string)) + self._intern_utf8(string, len(string), &str_hash) elif isinstance(string, bytes): if string in SYMBOLS_BY_STR: return SYMBOLS_BY_STR[string] - key = hash_utf8(string, len(string)) - self._intern_utf8(string, len(string)) + str_hash = hash_utf8(string, len(string)) + self._intern_utf8(string, len(string), &str_hash) else: raise TypeError(Errors.E017.format(value_type=type(string))) - return key + return str_hash def __len__(self): """The number of strings in the store. @@ -174,30 +204,29 @@ cdef class StringStore: """ return self.keys.size() - def __contains__(self, string not None): - """Check whether a string is in the store. + def __contains__(self, string_or_id not None): + """Check whether a string or ID is in the store. - string (str): The string to check. + string_or_id (str or int): The string to check. RETURNS (bool): Whether the store contains the string. """ - cdef hash_t key - if isinstance(string, int) or isinstance(string, long): - if string == 0: + cdef hash_t str_hash + if isinstance(string_or_id, str): + if len(string_or_id) == 0: return True - key = string - elif len(string) == 0: - return True - elif string in SYMBOLS_BY_STR: - return True - elif isinstance(string, str): - key = hash_string(string) + elif string_or_id in SYMBOLS_BY_STR: + return True + str_hash = hash_string(string_or_id) + elif _try_coerce_to_hash(string_or_id, &str_hash): + pass else: - string = string.encode("utf8") - key = hash_utf8(string, len(string)) - if key < len(SYMBOLS_BY_INT): + # TODO: Raise an error instead + return self._map.get(string_or_id) is not NULL + + if str_hash < len(SYMBOLS_BY_INT): return True else: - return self._map.get(key) is not NULL + return self._map.get(str_hash) is not NULL def __iter__(self): """Iterate over the strings in the store, in order. @@ -272,13 +301,13 @@ cdef class StringStore: cdef const Utf8Str* intern_unicode(self, str py_string): # 0 means missing, but we don't bother offsetting the index. cdef bytes byte_string = py_string.encode("utf8") - return self._intern_utf8(byte_string, len(byte_string)) + return self._intern_utf8(byte_string, len(byte_string), NULL) @cython.final - cdef const Utf8Str* _intern_utf8(self, char* utf8_string, int length): + cdef const Utf8Str* _intern_utf8(self, char* utf8_string, int length, hash_t* precalculated_hash): # TODO: This function's API/behaviour is an unholy mess... # 0 means missing, but we don't bother offsetting the index. - cdef hash_t key = hash_utf8(utf8_string, length) + cdef hash_t key = precalculated_hash[0] if precalculated_hash is not NULL else hash_utf8(utf8_string, length) cdef Utf8Str* value = self._map.get(key) if value is not NULL: return value diff --git a/spacy/tests/conftest.py b/spacy/tests/conftest.py index db17f1a8f..eb643ec2f 100644 --- a/spacy/tests/conftest.py +++ b/spacy/tests/conftest.py @@ -1,5 +1,11 @@ import pytest from spacy.util import get_lang_class +from hypothesis import settings + +# Functionally disable deadline settings for tests +# to prevent spurious test failures in CI builds. +settings.register_profile("no_deadlines", deadline=2 * 60 * 1000) # in ms +settings.load_profile("no_deadlines") def pytest_addoption(parser): diff --git a/spacy/tests/doc/test_doc_api.py b/spacy/tests/doc/test_doc_api.py index 19b554572..dd4942989 100644 --- a/spacy/tests/doc/test_doc_api.py +++ b/spacy/tests/doc/test_doc_api.py @@ -11,7 +11,7 @@ from spacy.lang.en import English from spacy.lang.xx import MultiLanguage from spacy.language import Language from spacy.lexeme import Lexeme -from spacy.tokens import Doc, Span, Token +from spacy.tokens import Doc, Span, SpanGroup, Token from spacy.vocab import Vocab from .test_underscore import clean_underscore # noqa: F401 @@ -964,3 +964,13 @@ def test_doc_spans_copy(en_tokenizer): assert weakref.ref(doc1) == doc1.spans.doc_ref doc2 = doc1.copy() assert weakref.ref(doc2) == doc2.spans.doc_ref + + +def test_doc_spans_setdefault(en_tokenizer): + doc = en_tokenizer("Some text about Colombia and the Czech Republic") + doc.spans.setdefault("key1") + assert len(doc.spans["key1"]) == 0 + doc.spans.setdefault("key2", default=[doc[0:1]]) + assert len(doc.spans["key2"]) == 1 + doc.spans.setdefault("key3", default=SpanGroup(doc, spans=[doc[0:1], doc[1:2]])) + assert len(doc.spans["key3"]) == 2 diff --git a/spacy/tests/doc/test_json_doc_conversion.py b/spacy/tests/doc/test_json_doc_conversion.py new file mode 100644 index 000000000..85e4def29 --- /dev/null +++ b/spacy/tests/doc/test_json_doc_conversion.py @@ -0,0 +1,191 @@ +import pytest +import spacy +from spacy import schemas +from spacy.tokens import Doc, Span + + +@pytest.fixture() +def doc(en_vocab): + words = ["c", "d", "e"] + pos = ["VERB", "NOUN", "NOUN"] + tags = ["VBP", "NN", "NN"] + heads = [0, 0, 1] + deps = ["ROOT", "dobj", "dobj"] + ents = ["O", "B-ORG", "O"] + morphs = ["Feat1=A", "Feat1=B", "Feat1=A|Feat2=D"] + + return Doc( + en_vocab, + words=words, + pos=pos, + tags=tags, + heads=heads, + deps=deps, + ents=ents, + morphs=morphs, + ) + + +@pytest.fixture() +def doc_without_deps(en_vocab): + words = ["c", "d", "e"] + pos = ["VERB", "NOUN", "NOUN"] + tags = ["VBP", "NN", "NN"] + ents = ["O", "B-ORG", "O"] + morphs = ["Feat1=A", "Feat1=B", "Feat1=A|Feat2=D"] + + return Doc( + en_vocab, + words=words, + pos=pos, + tags=tags, + ents=ents, + morphs=morphs, + sent_starts=[True, False, True], + ) + + +def test_doc_to_json(doc): + json_doc = doc.to_json() + assert json_doc["text"] == "c d e " + assert len(json_doc["tokens"]) == 3 + assert json_doc["tokens"][0]["pos"] == "VERB" + assert json_doc["tokens"][0]["tag"] == "VBP" + assert json_doc["tokens"][0]["dep"] == "ROOT" + assert len(json_doc["ents"]) == 1 + assert json_doc["ents"][0]["start"] == 2 # character offset! + assert json_doc["ents"][0]["end"] == 3 # character offset! + assert json_doc["ents"][0]["label"] == "ORG" + assert not schemas.validate(schemas.DocJSONSchema, json_doc) + + +def test_doc_to_json_underscore(doc): + Doc.set_extension("json_test1", default=False) + Doc.set_extension("json_test2", default=False) + doc._.json_test1 = "hello world" + doc._.json_test2 = [1, 2, 3] + json_doc = doc.to_json(underscore=["json_test1", "json_test2"]) + assert "_" in json_doc + assert json_doc["_"]["json_test1"] == "hello world" + assert json_doc["_"]["json_test2"] == [1, 2, 3] + assert not schemas.validate(schemas.DocJSONSchema, json_doc) + + +def test_doc_to_json_underscore_error_attr(doc): + """Test that Doc.to_json() raises an error if a custom attribute doesn't + exist in the ._ space.""" + with pytest.raises(ValueError): + doc.to_json(underscore=["json_test3"]) + + +def test_doc_to_json_underscore_error_serialize(doc): + """Test that Doc.to_json() raises an error if a custom attribute value + isn't JSON-serializable.""" + Doc.set_extension("json_test4", method=lambda doc: doc.text) + with pytest.raises(ValueError): + doc.to_json(underscore=["json_test4"]) + + +def test_doc_to_json_span(doc): + """Test that Doc.to_json() includes spans""" + doc.spans["test"] = [Span(doc, 0, 2, "test"), Span(doc, 0, 1, "test")] + json_doc = doc.to_json() + assert "spans" in json_doc + assert len(json_doc["spans"]) == 1 + assert len(json_doc["spans"]["test"]) == 2 + assert json_doc["spans"]["test"][0]["start"] == 0 + assert not schemas.validate(schemas.DocJSONSchema, json_doc) + + +def test_json_to_doc(doc): + new_doc = Doc(doc.vocab).from_json(doc.to_json(), validate=True) + new_tokens = [token for token in new_doc] + assert new_doc.text == doc.text == "c d e " + assert len(new_tokens) == len([token for token in doc]) == 3 + assert new_tokens[0].pos == doc[0].pos + assert new_tokens[0].tag == doc[0].tag + assert new_tokens[0].dep == doc[0].dep + assert new_tokens[0].head.idx == doc[0].head.idx + assert new_tokens[0].lemma == doc[0].lemma + assert len(new_doc.ents) == 1 + assert new_doc.ents[0].start == 1 + assert new_doc.ents[0].end == 2 + assert new_doc.ents[0].label_ == "ORG" + + +def test_json_to_doc_underscore(doc): + if not Doc.has_extension("json_test1"): + Doc.set_extension("json_test1", default=False) + if not Doc.has_extension("json_test2"): + Doc.set_extension("json_test2", default=False) + + doc._.json_test1 = "hello world" + doc._.json_test2 = [1, 2, 3] + json_doc = doc.to_json(underscore=["json_test1", "json_test2"]) + new_doc = Doc(doc.vocab).from_json(json_doc, validate=True) + assert all([new_doc.has_extension(f"json_test{i}") for i in range(1, 3)]) + assert new_doc._.json_test1 == "hello world" + assert new_doc._.json_test2 == [1, 2, 3] + + +def test_json_to_doc_spans(doc): + """Test that Doc.from_json() includes correct.spans.""" + doc.spans["test"] = [ + Span(doc, 0, 2, label="test"), + Span(doc, 0, 1, label="test", kb_id=7), + ] + json_doc = doc.to_json() + new_doc = Doc(doc.vocab).from_json(json_doc, validate=True) + assert len(new_doc.spans) == 1 + assert len(new_doc.spans["test"]) == 2 + for i in range(2): + assert new_doc.spans["test"][i].start == doc.spans["test"][i].start + assert new_doc.spans["test"][i].end == doc.spans["test"][i].end + assert new_doc.spans["test"][i].label == doc.spans["test"][i].label + assert new_doc.spans["test"][i].kb_id == doc.spans["test"][i].kb_id + + +def test_json_to_doc_sents(doc, doc_without_deps): + """Test that Doc.from_json() includes correct.sents.""" + for test_doc in (doc, doc_without_deps): + json_doc = test_doc.to_json() + new_doc = Doc(doc.vocab).from_json(json_doc, validate=True) + assert [sent.text for sent in test_doc.sents] == [ + sent.text for sent in new_doc.sents + ] + assert [token.is_sent_start for token in test_doc] == [ + token.is_sent_start for token in new_doc + ] + + +def test_json_to_doc_cats(doc): + """Test that Doc.from_json() includes correct .cats.""" + cats = {"A": 0.3, "B": 0.7} + doc.cats = cats + json_doc = doc.to_json() + new_doc = Doc(doc.vocab).from_json(json_doc, validate=True) + assert new_doc.cats == cats + + +def test_json_to_doc_spaces(): + """Test that Doc.from_json() preserves spaces correctly.""" + doc = spacy.blank("en")("This is just brilliant.") + json_doc = doc.to_json() + new_doc = Doc(doc.vocab).from_json(json_doc, validate=True) + assert doc.text == new_doc.text + + +def test_json_to_doc_attribute_consistency(doc): + """Test that Doc.from_json() raises an exception if tokens don't all have the same set of properties.""" + doc_json = doc.to_json() + doc_json["tokens"][1].pop("morph") + with pytest.raises(ValueError): + Doc(doc.vocab).from_json(doc_json) + + +def test_json_to_doc_validation_error(doc): + """Test that Doc.from_json() raises an exception when validating invalid input.""" + doc_json = doc.to_json() + doc_json.pop("tokens") + with pytest.raises(ValueError): + Doc(doc.vocab).from_json(doc_json, validate=True) diff --git a/spacy/tests/doc/test_pickle_doc.py b/spacy/tests/doc/test_pickle_doc.py index 738a751a0..28cb66714 100644 --- a/spacy/tests/doc/test_pickle_doc.py +++ b/spacy/tests/doc/test_pickle_doc.py @@ -5,11 +5,9 @@ from spacy.compat import pickle def test_pickle_single_doc(): nlp = Language() doc = nlp("pickle roundtrip") - doc._context = 3 data = pickle.dumps(doc, 1) doc2 = pickle.loads(data) assert doc2.text == "pickle roundtrip" - assert doc2._context == 3 def test_list_of_docs_pickles_efficiently(): diff --git a/spacy/tests/doc/test_span.py b/spacy/tests/doc/test_span.py index c0496cabf..3676b35af 100644 --- a/spacy/tests/doc/test_span.py +++ b/spacy/tests/doc/test_span.py @@ -428,10 +428,19 @@ def test_span_string_label_kb_id(doc): assert span.kb_id == doc.vocab.strings["Q342"] +def test_span_string_label_id(doc): + span = Span(doc, 0, 1, label="hello", span_id="Q342") + assert span.label_ == "hello" + assert span.label == doc.vocab.strings["hello"] + assert span.id_ == "Q342" + assert span.id == doc.vocab.strings["Q342"] + + def test_span_attrs_writable(doc): span = Span(doc, 0, 1) span.label_ = "label" span.kb_id_ = "kb_id" + span.id_ = "id" def test_span_ents_property(doc): @@ -619,6 +628,9 @@ def test_span_comparison(doc): assert Span(doc, 0, 4, "LABEL", kb_id="KB_ID") <= Span(doc, 1, 3) assert Span(doc, 1, 3) > Span(doc, 0, 4, "LABEL", kb_id="KB_ID") assert Span(doc, 1, 3) >= Span(doc, 0, 4, "LABEL", kb_id="KB_ID") + + # Different id + assert Span(doc, 1, 3, span_id="AAA") < Span(doc, 1, 3, span_id="BBB") # fmt: on diff --git a/spacy/tests/doc/test_to_json.py b/spacy/tests/doc/test_to_json.py deleted file mode 100644 index 202281654..000000000 --- a/spacy/tests/doc/test_to_json.py +++ /dev/null @@ -1,72 +0,0 @@ -import pytest -from spacy.tokens import Doc, Span - - -@pytest.fixture() -def doc(en_vocab): - words = ["c", "d", "e"] - pos = ["VERB", "NOUN", "NOUN"] - tags = ["VBP", "NN", "NN"] - heads = [0, 0, 0] - deps = ["ROOT", "dobj", "dobj"] - ents = ["O", "B-ORG", "O"] - morphs = ["Feat1=A", "Feat1=B", "Feat1=A|Feat2=D"] - return Doc( - en_vocab, - words=words, - pos=pos, - tags=tags, - heads=heads, - deps=deps, - ents=ents, - morphs=morphs, - ) - - -def test_doc_to_json(doc): - json_doc = doc.to_json() - assert json_doc["text"] == "c d e " - assert len(json_doc["tokens"]) == 3 - assert json_doc["tokens"][0]["pos"] == "VERB" - assert json_doc["tokens"][0]["tag"] == "VBP" - assert json_doc["tokens"][0]["dep"] == "ROOT" - assert len(json_doc["ents"]) == 1 - assert json_doc["ents"][0]["start"] == 2 # character offset! - assert json_doc["ents"][0]["end"] == 3 # character offset! - assert json_doc["ents"][0]["label"] == "ORG" - - -def test_doc_to_json_underscore(doc): - Doc.set_extension("json_test1", default=False) - Doc.set_extension("json_test2", default=False) - doc._.json_test1 = "hello world" - doc._.json_test2 = [1, 2, 3] - json_doc = doc.to_json(underscore=["json_test1", "json_test2"]) - assert "_" in json_doc - assert json_doc["_"]["json_test1"] == "hello world" - assert json_doc["_"]["json_test2"] == [1, 2, 3] - - -def test_doc_to_json_underscore_error_attr(doc): - """Test that Doc.to_json() raises an error if a custom attribute doesn't - exist in the ._ space.""" - with pytest.raises(ValueError): - doc.to_json(underscore=["json_test3"]) - - -def test_doc_to_json_underscore_error_serialize(doc): - """Test that Doc.to_json() raises an error if a custom attribute value - isn't JSON-serializable.""" - Doc.set_extension("json_test4", method=lambda doc: doc.text) - with pytest.raises(ValueError): - doc.to_json(underscore=["json_test4"]) - - -def test_doc_to_json_span(doc): - """Test that Doc.to_json() includes spans""" - doc.spans["test"] = [Span(doc, 0, 2, "test"), Span(doc, 0, 1, "test")] - json_doc = doc.to_json() - assert "spans" in json_doc - assert len(json_doc["spans"]) == 1 - assert len(json_doc["spans"]["test"]) == 2 - assert json_doc["spans"]["test"][0]["start"] == 0 diff --git a/spacy/tests/lang/bg/test_tokenizer.py b/spacy/tests/lang/bg/test_tokenizer.py new file mode 100644 index 000000000..2e2c45001 --- /dev/null +++ b/spacy/tests/lang/bg/test_tokenizer.py @@ -0,0 +1,8 @@ +import pytest + + +def test_bg_tokenizer_handles_final_diacritics(bg_tokenizer): + text = "Ня̀маше яйца̀. Ня̀маше яйца̀." + tokens = bg_tokenizer(text) + assert tokens[1].text == "яйца̀" + assert tokens[2].text == "." diff --git a/spacy/tests/lang/en/test_tokenizer.py b/spacy/tests/lang/en/test_tokenizer.py index e6d1d7d85..0133d00b0 100644 --- a/spacy/tests/lang/en/test_tokenizer.py +++ b/spacy/tests/lang/en/test_tokenizer.py @@ -167,3 +167,12 @@ def test_issue3521(en_tokenizer, word): tok = en_tokenizer(word)[1] # 'not' and 'would' should be stopwords, also in their abbreviated forms assert tok.is_stop + + +@pytest.mark.issue(10699) +@pytest.mark.parametrize("text", ["theses", "thisre"]) +def test_issue10699(en_tokenizer, text): + """Test that 'theses' and 'thisre' are excluded from the contractions + generated by the English tokenizer exceptions.""" + tokens = en_tokenizer(text) + assert len(tokens) == 1 diff --git a/spacy/tests/lang/ru/test_tokenizer.py b/spacy/tests/lang/ru/test_tokenizer.py index 1cfdc50ee..083b55a09 100644 --- a/spacy/tests/lang/ru/test_tokenizer.py +++ b/spacy/tests/lang/ru/test_tokenizer.py @@ -1,3 +1,4 @@ +from string import punctuation import pytest @@ -122,3 +123,36 @@ def test_ru_tokenizer_splits_bracket_period(ru_tokenizer): text = "(Раз, два, три, проверка)." tokens = ru_tokenizer(text) assert tokens[len(tokens) - 1].text == "." + + +@pytest.mark.parametrize( + "text", + [ + "рекоменду́я подда́ть жару́. Самого́ Баргамота", + "РЕКОМЕНДУ́Я ПОДДА́ТЬ ЖАРУ́. САМОГО́ БАРГАМОТА", + "рекоменду̍я подда̍ть жару̍.Самого̍ Баргамота", + "рекоменду̍я подда̍ть жару̍.'Самого̍ Баргамота", + "рекоменду̍я подда̍ть жару̍,самого̍ Баргамота", + "рекоменду̍я подда̍ть жару̍:самого̍ Баргамота", + "рекоменду̍я подда̍ть жару̍. самого̍ Баргамота", + "рекоменду̍я подда̍ть жару̍, самого̍ Баргамота", + "рекоменду̍я подда̍ть жару̍: самого̍ Баргамота", + "рекоменду̍я подда̍ть жару̍-самого̍ Баргамота", + ], +) +def test_ru_tokenizer_handles_final_diacritics(ru_tokenizer, text): + tokens = ru_tokenizer(text) + assert tokens[2].text in ("жару́", "ЖАРУ́", "жару̍") + assert tokens[3].text in punctuation + + +@pytest.mark.parametrize( + "text", + [ + "РЕКОМЕНДУ́Я ПОДДА́ТЬ ЖАРУ́.САМОГО́ БАРГАМОТА", + "рекоменду̍я подда̍ть жару́.самого́ Баргамота", + ], +) +def test_ru_tokenizer_handles_final_diacritic_and_period(ru_tokenizer, text): + tokens = ru_tokenizer(text) + assert tokens[2].text.lower() == "жару́.самого́" diff --git a/spacy/tests/lang/uk/test_tokenizer.py b/spacy/tests/lang/uk/test_tokenizer.py index 3d6e87301..6596f490a 100644 --- a/spacy/tests/lang/uk/test_tokenizer.py +++ b/spacy/tests/lang/uk/test_tokenizer.py @@ -140,3 +140,10 @@ def test_uk_tokenizer_splits_bracket_period(uk_tokenizer): text = "(Раз, два, три, проверка)." tokens = uk_tokenizer(text) assert tokens[len(tokens) - 1].text == "." + + +def test_uk_tokenizer_handles_final_diacritics(uk_tokenizer): + text = "Хлібі́в не було́. Хлібі́в не було́." + tokens = uk_tokenizer(text) + assert tokens[2].text == "було́" + assert tokens[3].text == "." diff --git a/spacy/tests/matcher/test_dependency_matcher.py b/spacy/tests/matcher/test_dependency_matcher.py index 1728c82af..b4e19d69d 100644 --- a/spacy/tests/matcher/test_dependency_matcher.py +++ b/spacy/tests/matcher/test_dependency_matcher.py @@ -316,6 +316,20 @@ def test_dependency_matcher_precedence_ops(en_vocab, op, num_matches): ("the", "brown", "$--", 0), ("brown", "the", "$--", 1), ("brown", "brown", "$--", 0), + ("quick", "fox", "<++", 1), + ("quick", "over", "<++", 0), + ("over", "jumped", "<++", 0), + ("the", "fox", "<++", 2), + ("brown", "fox", "<--", 0), + ("fox", "jumped", "<--", 0), + ("fox", "over", "<--", 1), + ("jumped", "over", ">++", 1), + ("fox", "lazy", ">++", 0), + ("over", "the", ">++", 0), + ("brown", "fox", ">--", 0), + ("fox", "brown", ">--", 1), + ("jumped", "fox", ">--", 1), + ("fox", "the", ">--", 2), ], ) def test_dependency_matcher_ops(en_vocab, doc, left, right, op, num_matches): diff --git a/spacy/tests/matcher/test_matcher_api.py b/spacy/tests/matcher/test_matcher_api.py index a27baf130..7c16da9f8 100644 --- a/spacy/tests/matcher/test_matcher_api.py +++ b/spacy/tests/matcher/test_matcher_api.py @@ -476,6 +476,17 @@ def test_matcher_extension_set_membership(en_vocab): assert len(matches) == 0 +@pytest.mark.xfail(reason="IN predicate must handle sequence values in extensions") +def test_matcher_extension_in_set_predicate(en_vocab): + matcher = Matcher(en_vocab) + Token.set_extension("ext", default=[]) + pattern = [{"_": {"ext": {"IN": ["A", "C"]}}}] + matcher.add("M", [pattern]) + doc = Doc(en_vocab, words=["a", "b", "c"]) + doc[0]._.ext = ["A", "B"] + assert len(matcher(doc)) == 1 + + def test_matcher_basic_check(en_vocab): matcher = Matcher(en_vocab) # Potential mistake: pass in pattern instead of list of patterns @@ -669,3 +680,38 @@ def test_matcher_ent_iob_key(en_vocab): assert matches[0] == "Maria" assert matches[1] == "Maria Esperanza" assert matches[2] == "Esperanza" + + +def test_matcher_min_max_operator(en_vocab): + # Exactly n matches {n} + doc = Doc( + en_vocab, + words=["foo", "bar", "foo", "foo", "bar", "foo", "foo", "foo", "bar", "bar"], + ) + matcher = Matcher(en_vocab) + pattern = [{"ORTH": "foo", "OP": "{3}"}] + matcher.add("TEST", [pattern]) + + matches1 = [doc[start:end].text for _, start, end in matcher(doc)] + assert len(matches1) == 1 + + # At least n matches {n,} + matcher = Matcher(en_vocab) + pattern = [{"ORTH": "foo", "OP": "{2,}"}] + matcher.add("TEST", [pattern]) + matches2 = [doc[start:end].text for _, start, end in matcher(doc)] + assert len(matches2) == 4 + + # At most m matches {,m} + matcher = Matcher(en_vocab) + pattern = [{"ORTH": "foo", "OP": "{,2}"}] + matcher.add("TEST", [pattern]) + matches3 = [doc[start:end].text for _, start, end in matcher(doc)] + assert len(matches3) == 9 + + # At least n matches and most m matches {n,m} + matcher = Matcher(en_vocab) + pattern = [{"ORTH": "foo", "OP": "{2,3}"}] + matcher.add("TEST", [pattern]) + matches4 = [doc[start:end].text for _, start, end in matcher(doc)] + assert len(matches4) == 4 diff --git a/spacy/tests/matcher/test_matcher_logic.py b/spacy/tests/matcher/test_matcher_logic.py index 3649b07ed..3b65fee23 100644 --- a/spacy/tests/matcher/test_matcher_logic.py +++ b/spacy/tests/matcher/test_matcher_logic.py @@ -699,6 +699,10 @@ def test_matcher_with_alignments_greedy_longest(en_vocab): ("aaaa", "a a a a a?", [0, 1, 2, 3]), ("aaab", "a+ a b", [0, 0, 1, 2]), ("aaab", "a+ a+ b", [0, 0, 1, 2]), + ("aaab", "a{2,} b", [0, 0, 0, 1]), + ("aaab", "a{,3} b", [0, 0, 0, 1]), + ("aaab", "a{2} b", [0, 0, 1]), + ("aaab", "a{2,3} b", [0, 0, 0, 1]), ] for string, pattern_str, result in cases: matcher = Matcher(en_vocab) @@ -711,6 +715,8 @@ def test_matcher_with_alignments_greedy_longest(en_vocab): pattern.append({"ORTH": part[0], "OP": "*"}) elif part.endswith("?"): pattern.append({"ORTH": part[0], "OP": "?"}) + elif part.endswith("}"): + pattern.append({"ORTH": part[0], "OP": part[1:]}) else: pattern.append({"ORTH": part}) matcher.add("PATTERN", [pattern], greedy="LONGEST") @@ -722,7 +728,7 @@ def test_matcher_with_alignments_greedy_longest(en_vocab): assert expected == result, (string, pattern_str, s, e, n_matches) -def test_matcher_with_alignments_nongreedy(en_vocab): +def test_matcher_with_alignments_non_greedy(en_vocab): cases = [ (0, "aaab", "a* b", [[0, 1], [0, 0, 1], [0, 0, 0, 1], [1]]), (1, "baab", "b a* b", [[0, 1, 1, 2]]), @@ -752,6 +758,10 @@ def test_matcher_with_alignments_nongreedy(en_vocab): (15, "aaaa", "a a a a a?", [[0, 1, 2, 3]]), (16, "aaab", "a+ a b", [[0, 1, 2], [0, 0, 1, 2]]), (17, "aaab", "a+ a+ b", [[0, 1, 2], [0, 0, 1, 2]]), + (18, "aaab", "a{2,} b", [[0, 0, 1], [0, 0, 0, 1]]), + (19, "aaab", "a{3} b", [[0, 0, 0, 1]]), + (20, "aaab", "a{2} b", [[0, 0, 1]]), + (21, "aaab", "a{2,3} b", [[0, 0, 1], [0, 0, 0, 1]]), ] for case_id, string, pattern_str, results in cases: matcher = Matcher(en_vocab) @@ -764,6 +774,8 @@ def test_matcher_with_alignments_nongreedy(en_vocab): pattern.append({"ORTH": part[0], "OP": "*"}) elif part.endswith("?"): pattern.append({"ORTH": part[0], "OP": "?"}) + elif part.endswith("}"): + pattern.append({"ORTH": part[0], "OP": part[1:]}) else: pattern.append({"ORTH": part}) diff --git a/spacy/tests/matcher/test_pattern_validation.py b/spacy/tests/matcher/test_pattern_validation.py index 8c265785c..e7eced02c 100644 --- a/spacy/tests/matcher/test_pattern_validation.py +++ b/spacy/tests/matcher/test_pattern_validation.py @@ -14,6 +14,14 @@ TEST_PATTERNS = [ ('[{"TEXT": "foo"}, {"LOWER": "bar"}]', 1, 1), ([{"ENT_IOB": "foo"}], 1, 1), ([1, 2, 3], 3, 1), + ([{"TEXT": "foo", "OP": "{,}"}], 1, 1), + ([{"TEXT": "foo", "OP": "{,4}4"}], 1, 1), + ([{"TEXT": "foo", "OP": "{a,3}"}], 1, 1), + ([{"TEXT": "foo", "OP": "{a}"}], 1, 1), + ([{"TEXT": "foo", "OP": "{,a}"}], 1, 1), + ([{"TEXT": "foo", "OP": "{1,2,3}"}], 1, 1), + ([{"TEXT": "foo", "OP": "{1, 3}"}], 1, 1), + ([{"TEXT": "foo", "OP": "{-2}"}], 1, 1), # Bad patterns flagged outside of Matcher ([{"_": {"foo": "bar", "baz": {"IN": "foo"}}}], 2, 0), # prev: (1, 0) # Bad patterns not flagged with minimal checks @@ -38,6 +46,7 @@ TEST_PATTERNS = [ ([{"SENT_START": True}], 0, 0), ([{"ENT_ID": "STRING"}], 0, 0), ([{"ENT_KB_ID": "STRING"}], 0, 0), + ([{"TEXT": "ha", "OP": "{3}"}], 0, 0), ] diff --git a/spacy/tests/matcher/test_phrase_matcher.py b/spacy/tests/matcher/test_phrase_matcher.py index f893d81f8..3b24f3ba8 100644 --- a/spacy/tests/matcher/test_phrase_matcher.py +++ b/spacy/tests/matcher/test_phrase_matcher.py @@ -122,6 +122,36 @@ def test_issue6839(en_vocab): assert matches +@pytest.mark.issue(10643) +def test_issue10643(en_vocab): + """Ensure overlapping terms can be removed from PhraseMatcher""" + + # fmt: off + words = ["Only", "save", "out", "the", "binary", "data", "for", "the", "individual", "components", "."] + # fmt: on + doc = Doc(en_vocab, words=words) + terms = { + "0": Doc(en_vocab, words=["binary"]), + "1": Doc(en_vocab, words=["binary", "data"]), + } + matcher = PhraseMatcher(en_vocab) + for match_id, term in terms.items(): + matcher.add(match_id, [term]) + + matches = matcher(doc) + assert matches == [(en_vocab.strings["0"], 4, 5), (en_vocab.strings["1"], 4, 6)] + + matcher.remove("0") + assert len(matcher) == 1 + new_matches = matcher(doc) + assert new_matches == [(en_vocab.strings["1"], 4, 6)] + + matcher.remove("1") + assert len(matcher) == 0 + no_matches = matcher(doc) + assert not no_matches + + def test_matcher_phrase_matcher(en_vocab): doc = Doc(en_vocab, words=["I", "like", "Google", "Now", "best"]) # intermediate phrase diff --git a/spacy/tests/parser/test_ner.py b/spacy/tests/parser/test_ner.py index b3b29d1f9..00889efdc 100644 --- a/spacy/tests/parser/test_ner.py +++ b/spacy/tests/parser/test_ner.py @@ -10,7 +10,7 @@ from spacy.lang.it import Italian from spacy.language import Language from spacy.lookups import Lookups from spacy.pipeline._parser_internals.ner import BiluoPushDown -from spacy.training import Example, iob_to_biluo +from spacy.training import Example, iob_to_biluo, split_bilu_label from spacy.tokens import Doc, Span from spacy.vocab import Vocab import logging @@ -110,6 +110,9 @@ def test_issue2385(): # maintain support for iob2 format tags3 = ("B-PERSON", "I-PERSON", "B-PERSON") assert iob_to_biluo(tags3) == ["B-PERSON", "L-PERSON", "U-PERSON"] + # ensure it works with hyphens in the name + tags4 = ("B-MULTI-PERSON", "I-MULTI-PERSON", "B-MULTI-PERSON") + assert iob_to_biluo(tags4) == ["B-MULTI-PERSON", "L-MULTI-PERSON", "U-MULTI-PERSON"] @pytest.mark.issue(2800) @@ -154,6 +157,24 @@ def test_issue3209(): assert ner2.move_names == move_names +def test_labels_from_BILUO(): + """Test that labels are inferred correctly when there's a - in label.""" + nlp = English() + ner = nlp.add_pipe("ner") + ner.add_label("LARGE-ANIMAL") + nlp.initialize() + move_names = [ + "O", + "B-LARGE-ANIMAL", + "I-LARGE-ANIMAL", + "L-LARGE-ANIMAL", + "U-LARGE-ANIMAL", + ] + labels = {"LARGE-ANIMAL"} + assert ner.move_names == move_names + assert set(ner.labels) == labels + + @pytest.mark.issue(4267) def test_issue4267(): """Test that running an entity_ruler after ner gives consistent results""" @@ -298,7 +319,7 @@ def test_oracle_moves_missing_B(en_vocab): elif tag == "O": moves.add_action(move_types.index("O"), "") else: - action, label = tag.split("-") + action, label = split_bilu_label(tag) moves.add_action(move_types.index("B"), label) moves.add_action(move_types.index("I"), label) moves.add_action(move_types.index("L"), label) @@ -324,7 +345,7 @@ def test_oracle_moves_whitespace(en_vocab): elif tag == "O": moves.add_action(move_types.index("O"), "") else: - action, label = tag.split("-") + action, label = split_bilu_label(tag) moves.add_action(move_types.index(action), label) moves.get_oracle_sequence(example) diff --git a/spacy/tests/parser/test_nonproj.py b/spacy/tests/parser/test_nonproj.py index 60d000c44..051d0ef0c 100644 --- a/spacy/tests/parser/test_nonproj.py +++ b/spacy/tests/parser/test_nonproj.py @@ -49,7 +49,9 @@ def test_parser_contains_cycle(tree, cyclic_tree, partial_tree, multirooted_tree assert contains_cycle(multirooted_tree) is None -def test_parser_is_nonproj_arc(nonproj_tree, partial_tree, multirooted_tree): +def test_parser_is_nonproj_arc( + cyclic_tree, nonproj_tree, partial_tree, multirooted_tree +): assert is_nonproj_arc(0, nonproj_tree) is False assert is_nonproj_arc(1, nonproj_tree) is False assert is_nonproj_arc(2, nonproj_tree) is False @@ -62,15 +64,23 @@ def test_parser_is_nonproj_arc(nonproj_tree, partial_tree, multirooted_tree): assert is_nonproj_arc(7, partial_tree) is False assert is_nonproj_arc(17, multirooted_tree) is False assert is_nonproj_arc(16, multirooted_tree) is True + with pytest.raises( + ValueError, match=r"Found cycle in dependency graph: \[1, 2, 2, 4, 5, 3, 2\]" + ): + is_nonproj_arc(6, cyclic_tree) def test_parser_is_nonproj_tree( - proj_tree, nonproj_tree, partial_tree, multirooted_tree + proj_tree, cyclic_tree, nonproj_tree, partial_tree, multirooted_tree ): assert is_nonproj_tree(proj_tree) is False assert is_nonproj_tree(nonproj_tree) is True assert is_nonproj_tree(partial_tree) is False assert is_nonproj_tree(multirooted_tree) is True + with pytest.raises( + ValueError, match=r"Found cycle in dependency graph: \[1, 2, 2, 4, 5, 3, 2\]" + ): + is_nonproj_tree(cyclic_tree) def test_parser_pseudoprojectivity(en_vocab): @@ -84,8 +94,10 @@ def test_parser_pseudoprojectivity(en_vocab): tree = [1, 2, 2] nonproj_tree = [1, 2, 2, 4, 5, 2, 7, 4, 2] nonproj_tree2 = [9, 1, 3, 1, 5, 6, 9, 8, 6, 1, 6, 12, 13, 10, 1] + cyclic_tree = [1, 2, 2, 4, 5, 3, 2] labels = ["det", "nsubj", "root", "det", "dobj", "aux", "nsubj", "acl", "punct"] labels2 = ["advmod", "root", "det", "nsubj", "advmod", "det", "dobj", "det", "nmod", "aux", "nmod", "advmod", "det", "amod", "punct"] + cyclic_labels = ["det", "nsubj", "root", "det", "dobj", "aux", "punct"] # fmt: on assert nonproj.decompose("X||Y") == ("X", "Y") assert nonproj.decompose("X") == ("X", "") @@ -97,6 +109,8 @@ def test_parser_pseudoprojectivity(en_vocab): assert nonproj.get_smallest_nonproj_arc_slow(nonproj_tree2) == 10 # fmt: off proj_heads, deco_labels = nonproj.projectivize(nonproj_tree, labels) + with pytest.raises(ValueError, match=r'Found cycle in dependency graph: \[1, 2, 2, 4, 5, 3, 2\]'): + nonproj.projectivize(cyclic_tree, cyclic_labels) assert proj_heads == [1, 2, 2, 4, 5, 2, 7, 5, 2] assert deco_labels == ["det", "nsubj", "root", "det", "dobj", "aux", "nsubj", "acl||dobj", "punct"] diff --git a/spacy/tests/parser/test_parse.py b/spacy/tests/parser/test_parse.py index 7bbb30d8e..aaf31ed56 100644 --- a/spacy/tests/parser/test_parse.py +++ b/spacy/tests/parser/test_parse.py @@ -12,6 +12,7 @@ from spacy.vocab import Vocab from ...pipeline import DependencyParser from ...pipeline.dep_parser import DEFAULT_PARSER_MODEL from ..util import apply_transition_sequence, make_tempdir +from ...pipeline.tok2vec import DEFAULT_TOK2VEC_MODEL TRAIN_DATA = [ ( @@ -395,6 +396,34 @@ def test_overfitting_IO(pipe_name): assert_equal(batch_deps_1, no_batch_deps) +# fmt: off +@pytest.mark.slow +@pytest.mark.parametrize("pipe_name", ["parser", "beam_parser"]) +@pytest.mark.parametrize( + "parser_config", + [ + # TransitionBasedParser V1 + ({"@architectures": "spacy.TransitionBasedParser.v1", "tok2vec": DEFAULT_TOK2VEC_MODEL, "state_type": "parser", "extra_state_tokens": False, "hidden_width": 64, "maxout_pieces": 2, "use_upper": True}), + # TransitionBasedParser V2 + ({"@architectures": "spacy.TransitionBasedParser.v2", "tok2vec": DEFAULT_TOK2VEC_MODEL, "state_type": "parser", "extra_state_tokens": False, "hidden_width": 64, "maxout_pieces": 2, "use_upper": True}), + ], +) +# fmt: on +def test_parser_configs(pipe_name, parser_config): + pipe_config = {"model": parser_config} + nlp = English() + parser = nlp.add_pipe(pipe_name, config=pipe_config) + train_examples = [] + for text, annotations in TRAIN_DATA: + train_examples.append(Example.from_dict(nlp.make_doc(text), annotations)) + for dep in annotations.get("deps", []): + parser.add_label(dep) + optimizer = nlp.initialize() + for i in range(5): + losses = {} + nlp.update(train_examples, sgd=optimizer, losses=losses) + + def test_beam_parser_scores(): # Test that we can get confidence values out of the beam_parser pipe beam_width = 16 diff --git a/spacy/tests/pipeline/test_entity_linker.py b/spacy/tests/pipeline/test_entity_linker.py index 83d5bf0e2..14995d7b8 100644 --- a/spacy/tests/pipeline/test_entity_linker.py +++ b/spacy/tests/pipeline/test_entity_linker.py @@ -1,4 +1,4 @@ -from typing import Callable, Iterable +from typing import Callable, Iterable, Dict, Any import pytest from numpy.testing import assert_equal @@ -14,7 +14,7 @@ from spacy.pipeline.legacy import EntityLinker_v1 from spacy.pipeline.tok2vec import DEFAULT_TOK2VEC_MODEL from spacy.scorer import Scorer from spacy.tests.util import make_tempdir -from spacy.tokens import Span +from spacy.tokens import Span, Doc from spacy.training import Example from spacy.util import ensure_path from spacy.vocab import Vocab @@ -207,7 +207,7 @@ def test_no_entities(): nlp.add_pipe("sentencizer", first=True) # this will run the pipeline on the examples and shouldn't crash - results = nlp.evaluate(train_examples) + nlp.evaluate(train_examples) def test_partial_links(): @@ -1063,7 +1063,7 @@ def test_no_gold_ents(patterns): "entity_linker", config={"use_gold_ents": False}, last=True ) entity_linker.set_kb(create_kb) - assert entity_linker.use_gold_ents == False + assert entity_linker.use_gold_ents is False optimizer = nlp.initialize(get_examples=lambda: train_examples) for i in range(2): @@ -1074,4 +1074,101 @@ def test_no_gold_ents(patterns): nlp.add_pipe("sentencizer", first=True) # this will run the pipeline on the examples and shouldn't crash - results = nlp.evaluate(train_examples) + nlp.evaluate(train_examples) + + +@pytest.mark.issue(9575) +def test_tokenization_mismatch(): + nlp = English() + # include a matching entity so that update isn't skipped + doc1 = Doc( + nlp.vocab, + words=["Kirby", "123456"], + spaces=[True, False], + ents=["B-CHARACTER", "B-CARDINAL"], + ) + doc2 = Doc( + nlp.vocab, + words=["Kirby", "123", "456"], + spaces=[True, False, False], + ents=["B-CHARACTER", "B-CARDINAL", "B-CARDINAL"], + ) + + eg = Example(doc1, doc2) + train_examples = [eg] + vector_length = 3 + + def create_kb(vocab): + # create placeholder KB + mykb = KnowledgeBase(vocab, entity_vector_length=vector_length) + mykb.add_entity(entity="Q613241", freq=12, entity_vector=[6, -4, 3]) + mykb.add_alias("Kirby", ["Q613241"], [0.9]) + return mykb + + entity_linker = nlp.add_pipe("entity_linker", last=True) + entity_linker.set_kb(create_kb) + + optimizer = nlp.initialize(get_examples=lambda: train_examples) + for i in range(2): + losses = {} + nlp.update(train_examples, sgd=optimizer, losses=losses) + + nlp.add_pipe("sentencizer", first=True) + nlp.evaluate(train_examples) + + +# fmt: off +@pytest.mark.parametrize( + "meet_threshold,config", + [ + (False, {"@architectures": "spacy.EntityLinker.v2", "tok2vec": DEFAULT_TOK2VEC_MODEL}), + (True, {"@architectures": "spacy.EntityLinker.v2", "tok2vec": DEFAULT_TOK2VEC_MODEL}), + ], +) +# fmt: on +def test_threshold(meet_threshold: bool, config: Dict[str, Any]): + """Tests abstention threshold. + meet_threshold (bool): Whether to configure NEL setup so that confidence threshold is met. + config (Dict[str, Any]): NEL architecture config. + """ + nlp = English() + nlp.add_pipe("sentencizer") + text = "Mahler's Symphony No. 8 was beautiful." + entities = [(0, 6, "PERSON")] + links = {(0, 6): {"Q7304": 1.0}} + sent_starts = [1, -1, 0, 0, 0, 0, 0, 0, 0] + entity_id = "Q7304" + doc = nlp(text) + train_examples = [ + Example.from_dict( + doc, {"entities": entities, "links": links, "sent_starts": sent_starts} + ) + ] + + def create_kb(vocab): + # create artificial KB + mykb = KnowledgeBase(vocab, entity_vector_length=3) + mykb.add_entity(entity=entity_id, freq=12, entity_vector=[6, -4, 3]) + mykb.add_alias( + alias="Mahler", + entities=[entity_id], + probabilities=[1 if meet_threshold else 0.01], + ) + return mykb + + # Create the Entity Linker component and add it to the pipeline + entity_linker = nlp.add_pipe( + "entity_linker", + last=True, + config={"threshold": 0.99, "model": config}, + ) + entity_linker.set_kb(create_kb) # type: ignore + nlp.initialize(get_examples=lambda: train_examples) + + # Add a custom rule-based component to mimick NER + ruler = nlp.add_pipe("entity_ruler", before="entity_linker") + ruler.add_patterns([{"label": "PERSON", "pattern": [{"LOWER": "mahler"}]}]) # type: ignore + doc = nlp(text) + + assert len(doc.ents) == 1 + assert doc.ents[0].kb_id_ == entity_id if meet_threshold else EntityLinker.NIL diff --git a/spacy/tests/pipeline/test_entity_ruler.py b/spacy/tests/pipeline/test_entity_ruler.py index f2031d0a9..6851e2a7c 100644 --- a/spacy/tests/pipeline/test_entity_ruler.py +++ b/spacy/tests/pipeline/test_entity_ruler.py @@ -5,12 +5,15 @@ from spacy.tokens import Doc, Span from spacy.language import Language from spacy.lang.en import English from spacy.pipeline import EntityRuler, EntityRecognizer, merge_entities +from spacy.pipeline import SpanRuler from spacy.pipeline.ner import DEFAULT_NER_MODEL from spacy.errors import MatchPatternError from spacy.tests.util import make_tempdir from thinc.api import NumpyOps, get_current_ops +ENTITY_RULERS = ["entity_ruler", "future_entity_ruler"] + @pytest.fixture def nlp(): @@ -37,12 +40,14 @@ def add_ent_component(doc): @pytest.mark.issue(3345) -def test_issue3345(): +@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS) +def test_issue3345(entity_ruler_factory): """Test case where preset entity crosses sentence boundary.""" nlp = English() doc = Doc(nlp.vocab, words=["I", "live", "in", "New", "York"]) doc[4].is_sent_start = True - ruler = EntityRuler(nlp, patterns=[{"label": "GPE", "pattern": "New York"}]) + ruler = nlp.add_pipe(entity_ruler_factory, name="entity_ruler") + ruler.add_patterns([{"label": "GPE", "pattern": "New York"}]) cfg = {"model": DEFAULT_NER_MODEL} model = registry.resolve(cfg, validate=True)["model"] ner = EntityRecognizer(doc.vocab, model) @@ -60,13 +65,18 @@ def test_issue3345(): @pytest.mark.issue(4849) -def test_issue4849(): +@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS) +def test_issue4849(entity_ruler_factory): nlp = English() patterns = [ {"label": "PERSON", "pattern": "joe biden", "id": "joe-biden"}, {"label": "PERSON", "pattern": "bernie sanders", "id": "bernie-sanders"}, ] - ruler = nlp.add_pipe("entity_ruler", config={"phrase_matcher_attr": "LOWER"}) + ruler = nlp.add_pipe( + entity_ruler_factory, + name="entity_ruler", + config={"phrase_matcher_attr": "LOWER"}, + ) ruler.add_patterns(patterns) text = """ The left is starting to take aim at Democratic front-runner Joe Biden. @@ -86,10 +96,11 @@ def test_issue4849(): @pytest.mark.issue(5918) -def test_issue5918(): +@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS) +def test_issue5918(entity_ruler_factory): # Test edge case when merging entities. nlp = English() - ruler = nlp.add_pipe("entity_ruler") + ruler = nlp.add_pipe(entity_ruler_factory, name="entity_ruler") patterns = [ {"label": "ORG", "pattern": "Digicon Inc"}, {"label": "ORG", "pattern": "Rotan Mosle Inc's"}, @@ -114,9 +125,10 @@ def test_issue5918(): @pytest.mark.issue(8168) -def test_issue8168(): +@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS) +def test_issue8168(entity_ruler_factory): nlp = English() - ruler = nlp.add_pipe("entity_ruler") + ruler = nlp.add_pipe(entity_ruler_factory, name="entity_ruler") patterns = [ {"label": "ORG", "pattern": "Apple"}, { @@ -131,14 +143,17 @@ def test_issue8168(): }, ] ruler.add_patterns(patterns) - - assert ruler._ent_ids == {8043148519967183733: ("GPE", "san-francisco")} + doc = nlp("San Francisco San Fran") + assert all(t.ent_id_ == "san-francisco" for t in doc) @pytest.mark.issue(8216) -def test_entity_ruler_fix8216(nlp, patterns): +@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS) +def test_entity_ruler_fix8216(nlp, patterns, entity_ruler_factory): """Test that patterns don't get added excessively.""" - ruler = nlp.add_pipe("entity_ruler", config={"validate": True}) + ruler = nlp.add_pipe( + entity_ruler_factory, name="entity_ruler", config={"validate": True} + ) ruler.add_patterns(patterns) pattern_count = sum(len(mm) for mm in ruler.matcher._patterns.values()) assert pattern_count > 0 @@ -147,13 +162,16 @@ def test_entity_ruler_fix8216(nlp, patterns): assert after_count == pattern_count -def test_entity_ruler_init(nlp, patterns): - ruler = EntityRuler(nlp, patterns=patterns) +@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS) +def test_entity_ruler_init(nlp, patterns, entity_ruler_factory): + ruler = nlp.add_pipe(entity_ruler_factory, name="entity_ruler") + ruler.add_patterns(patterns) assert len(ruler) == len(patterns) assert len(ruler.labels) == 4 assert "HELLO" in ruler assert "BYE" in ruler - ruler = nlp.add_pipe("entity_ruler") + nlp.remove_pipe("entity_ruler") + ruler = nlp.add_pipe(entity_ruler_factory, name="entity_ruler") ruler.add_patterns(patterns) doc = nlp("hello world bye bye") assert len(doc.ents) == 2 @@ -161,20 +179,23 @@ def test_entity_ruler_init(nlp, patterns): assert doc.ents[1].label_ == "BYE" -def test_entity_ruler_no_patterns_warns(nlp): - ruler = EntityRuler(nlp) +@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS) +def test_entity_ruler_no_patterns_warns(nlp, entity_ruler_factory): + ruler = nlp.add_pipe(entity_ruler_factory, name="entity_ruler") assert len(ruler) == 0 assert len(ruler.labels) == 0 - nlp.add_pipe("entity_ruler") + nlp.remove_pipe("entity_ruler") + nlp.add_pipe(entity_ruler_factory, name="entity_ruler") assert nlp.pipe_names == ["entity_ruler"] with pytest.warns(UserWarning): doc = nlp("hello world bye bye") assert len(doc.ents) == 0 -def test_entity_ruler_init_patterns(nlp, patterns): +@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS) +def test_entity_ruler_init_patterns(nlp, patterns, entity_ruler_factory): # initialize with patterns - ruler = nlp.add_pipe("entity_ruler") + ruler = nlp.add_pipe(entity_ruler_factory, name="entity_ruler") assert len(ruler.labels) == 0 ruler.initialize(lambda: [], patterns=patterns) assert len(ruler.labels) == 4 @@ -186,7 +207,7 @@ def test_entity_ruler_init_patterns(nlp, patterns): nlp.config["initialize"]["components"]["entity_ruler"] = { "patterns": {"@misc": "entity_ruler_patterns"} } - ruler = nlp.add_pipe("entity_ruler") + ruler = nlp.add_pipe(entity_ruler_factory, name="entity_ruler") assert len(ruler.labels) == 0 nlp.initialize() assert len(ruler.labels) == 4 @@ -195,18 +216,20 @@ def test_entity_ruler_init_patterns(nlp, patterns): assert doc.ents[1].label_ == "BYE" -def test_entity_ruler_init_clear(nlp, patterns): +@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS) +def test_entity_ruler_init_clear(nlp, patterns, entity_ruler_factory): """Test that initialization clears patterns.""" - ruler = nlp.add_pipe("entity_ruler") + ruler = nlp.add_pipe(entity_ruler_factory, name="entity_ruler") ruler.add_patterns(patterns) assert len(ruler.labels) == 4 ruler.initialize(lambda: []) assert len(ruler.labels) == 0 -def test_entity_ruler_clear(nlp, patterns): +@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS) +def test_entity_ruler_clear(nlp, patterns, entity_ruler_factory): """Test that initialization clears patterns.""" - ruler = nlp.add_pipe("entity_ruler") + ruler = nlp.add_pipe(entity_ruler_factory, name="entity_ruler") ruler.add_patterns(patterns) assert len(ruler.labels) == 4 doc = nlp("hello world") @@ -218,8 +241,9 @@ def test_entity_ruler_clear(nlp, patterns): assert len(doc.ents) == 0 -def test_entity_ruler_existing(nlp, patterns): - ruler = nlp.add_pipe("entity_ruler") +@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS) +def test_entity_ruler_existing(nlp, patterns, entity_ruler_factory): + ruler = nlp.add_pipe(entity_ruler_factory, name="entity_ruler") ruler.add_patterns(patterns) nlp.add_pipe("add_ent", before="entity_ruler") doc = nlp("OH HELLO WORLD bye bye") @@ -228,8 +252,11 @@ def test_entity_ruler_existing(nlp, patterns): assert doc.ents[1].label_ == "BYE" -def test_entity_ruler_existing_overwrite(nlp, patterns): - ruler = nlp.add_pipe("entity_ruler", config={"overwrite_ents": True}) +@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS) +def test_entity_ruler_existing_overwrite(nlp, patterns, entity_ruler_factory): + ruler = nlp.add_pipe( + entity_ruler_factory, name="entity_ruler", config={"overwrite_ents": True} + ) ruler.add_patterns(patterns) nlp.add_pipe("add_ent", before="entity_ruler") doc = nlp("OH HELLO WORLD bye bye") @@ -239,8 +266,11 @@ def test_entity_ruler_existing_overwrite(nlp, patterns): assert doc.ents[1].label_ == "BYE" -def test_entity_ruler_existing_complex(nlp, patterns): - ruler = nlp.add_pipe("entity_ruler", config={"overwrite_ents": True}) +@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS) +def test_entity_ruler_existing_complex(nlp, patterns, entity_ruler_factory): + ruler = nlp.add_pipe( + entity_ruler_factory, name="entity_ruler", config={"overwrite_ents": True} + ) ruler.add_patterns(patterns) nlp.add_pipe("add_ent", before="entity_ruler") doc = nlp("foo foo bye bye") @@ -251,8 +281,11 @@ def test_entity_ruler_existing_complex(nlp, patterns): assert len(doc.ents[1]) == 2 -def test_entity_ruler_entity_id(nlp, patterns): - ruler = nlp.add_pipe("entity_ruler", config={"overwrite_ents": True}) +@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS) +def test_entity_ruler_entity_id(nlp, patterns, entity_ruler_factory): + ruler = nlp.add_pipe( + entity_ruler_factory, name="entity_ruler", config={"overwrite_ents": True} + ) ruler.add_patterns(patterns) doc = nlp("Apple is a technology company") assert len(doc.ents) == 1 @@ -260,18 +293,21 @@ def test_entity_ruler_entity_id(nlp, patterns): assert doc.ents[0].ent_id_ == "a1" -def test_entity_ruler_cfg_ent_id_sep(nlp, patterns): +@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS) +def test_entity_ruler_cfg_ent_id_sep(nlp, patterns, entity_ruler_factory): config = {"overwrite_ents": True, "ent_id_sep": "**"} - ruler = nlp.add_pipe("entity_ruler", config=config) + ruler = nlp.add_pipe(entity_ruler_factory, name="entity_ruler", config=config) ruler.add_patterns(patterns) - assert "TECH_ORG**a1" in ruler.phrase_patterns doc = nlp("Apple is a technology company") + if isinstance(ruler, EntityRuler): + assert "TECH_ORG**a1" in ruler.phrase_patterns assert len(doc.ents) == 1 assert doc.ents[0].label_ == "TECH_ORG" assert doc.ents[0].ent_id_ == "a1" -def test_entity_ruler_serialize_bytes(nlp, patterns): +@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS) +def test_entity_ruler_serialize_bytes(nlp, patterns, entity_ruler_factory): ruler = EntityRuler(nlp, patterns=patterns) assert len(ruler) == len(patterns) assert len(ruler.labels) == 4 @@ -288,7 +324,10 @@ def test_entity_ruler_serialize_bytes(nlp, patterns): assert sorted(new_ruler.labels) == sorted(ruler.labels) -def test_entity_ruler_serialize_phrase_matcher_attr_bytes(nlp, patterns): +@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS) +def test_entity_ruler_serialize_phrase_matcher_attr_bytes( + nlp, patterns, entity_ruler_factory +): ruler = EntityRuler(nlp, phrase_matcher_attr="LOWER", patterns=patterns) assert len(ruler) == len(patterns) assert len(ruler.labels) == 4 @@ -303,8 +342,9 @@ def test_entity_ruler_serialize_phrase_matcher_attr_bytes(nlp, patterns): assert new_ruler.phrase_matcher_attr == "LOWER" -def test_entity_ruler_validate(nlp): - ruler = EntityRuler(nlp) +@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS) +def test_entity_ruler_validate(nlp, entity_ruler_factory): + ruler = nlp.add_pipe(entity_ruler_factory, name="entity_ruler") validated_ruler = EntityRuler(nlp, validate=True) valid_pattern = {"label": "HELLO", "pattern": [{"LOWER": "HELLO"}]} @@ -322,32 +362,35 @@ def test_entity_ruler_validate(nlp): validated_ruler.add_patterns([invalid_pattern]) -def test_entity_ruler_properties(nlp, patterns): +@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS) +def test_entity_ruler_properties(nlp, patterns, entity_ruler_factory): ruler = EntityRuler(nlp, patterns=patterns, overwrite_ents=True) assert sorted(ruler.labels) == sorted(["HELLO", "BYE", "COMPLEX", "TECH_ORG"]) assert sorted(ruler.ent_ids) == ["a1", "a2"] -def test_entity_ruler_overlapping_spans(nlp): - ruler = EntityRuler(nlp) +@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS) +def test_entity_ruler_overlapping_spans(nlp, entity_ruler_factory): + ruler = nlp.add_pipe(entity_ruler_factory, name="entity_ruler") patterns = [ {"label": "FOOBAR", "pattern": "foo bar"}, {"label": "BARBAZ", "pattern": "bar baz"}, ] ruler.add_patterns(patterns) - doc = ruler(nlp.make_doc("foo bar baz")) + doc = nlp("foo bar baz") assert len(doc.ents) == 1 assert doc.ents[0].label_ == "FOOBAR" @pytest.mark.parametrize("n_process", [1, 2]) -def test_entity_ruler_multiprocessing(nlp, n_process): +@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS) +def test_entity_ruler_multiprocessing(nlp, n_process, entity_ruler_factory): if isinstance(get_current_ops, NumpyOps) or n_process < 2: texts = ["I enjoy eating Pizza Hut pizza."] patterns = [{"label": "FASTFOOD", "pattern": "Pizza Hut", "id": "1234"}] - ruler = nlp.add_pipe("entity_ruler") + ruler = nlp.add_pipe(entity_ruler_factory, name="entity_ruler") ruler.add_patterns(patterns) for doc in nlp.pipe(texts, n_process=2): @@ -355,8 +398,9 @@ def test_entity_ruler_multiprocessing(nlp, n_process): assert ent.ent_id_ == "1234" -def test_entity_ruler_serialize_jsonl(nlp, patterns): - ruler = nlp.add_pipe("entity_ruler") +@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS) +def test_entity_ruler_serialize_jsonl(nlp, patterns, entity_ruler_factory): + ruler = nlp.add_pipe(entity_ruler_factory, name="entity_ruler") ruler.add_patterns(patterns) with make_tempdir() as d: ruler.to_disk(d / "test_ruler.jsonl") @@ -365,8 +409,9 @@ def test_entity_ruler_serialize_jsonl(nlp, patterns): ruler.from_disk(d / "non_existing.jsonl") # read from a bad jsonl file -def test_entity_ruler_serialize_dir(nlp, patterns): - ruler = nlp.add_pipe("entity_ruler") +@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS) +def test_entity_ruler_serialize_dir(nlp, patterns, entity_ruler_factory): + ruler = nlp.add_pipe(entity_ruler_factory, name="entity_ruler") ruler.add_patterns(patterns) with make_tempdir() as d: ruler.to_disk(d / "test_ruler") @@ -375,52 +420,65 @@ def test_entity_ruler_serialize_dir(nlp, patterns): ruler.from_disk(d / "non_existing_dir") # read from a bad directory -def test_entity_ruler_remove_basic(nlp): - ruler = EntityRuler(nlp) +@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS) +def test_entity_ruler_remove_basic(nlp, entity_ruler_factory): + ruler = nlp.add_pipe(entity_ruler_factory, name="entity_ruler") patterns = [ - {"label": "PERSON", "pattern": "Duygu", "id": "duygu"}, + {"label": "PERSON", "pattern": "Dina", "id": "dina"}, {"label": "ORG", "pattern": "ACME", "id": "acme"}, {"label": "ORG", "pattern": "ACM"}, ] ruler.add_patterns(patterns) - doc = ruler(nlp.make_doc("Duygu went to school")) + doc = nlp("Dina went to school") assert len(ruler.patterns) == 3 assert len(doc.ents) == 1 + if isinstance(ruler, EntityRuler): + assert "PERSON||dina" in ruler.phrase_matcher assert doc.ents[0].label_ == "PERSON" - assert doc.ents[0].text == "Duygu" - assert "PERSON||duygu" in ruler.phrase_matcher - ruler.remove("duygu") - doc = ruler(nlp.make_doc("Duygu went to school")) + assert doc.ents[0].text == "Dina" + if isinstance(ruler, EntityRuler): + ruler.remove("dina") + else: + ruler.remove_by_id("dina") + doc = nlp("Dina went to school") assert len(doc.ents) == 0 - assert "PERSON||duygu" not in ruler.phrase_matcher + if isinstance(ruler, EntityRuler): + assert "PERSON||dina" not in ruler.phrase_matcher assert len(ruler.patterns) == 2 -def test_entity_ruler_remove_same_id_multiple_patterns(nlp): - ruler = EntityRuler(nlp) +@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS) +def test_entity_ruler_remove_same_id_multiple_patterns(nlp, entity_ruler_factory): + ruler = nlp.add_pipe(entity_ruler_factory, name="entity_ruler") patterns = [ - {"label": "PERSON", "pattern": "Duygu", "id": "duygu"}, - {"label": "ORG", "pattern": "DuyguCorp", "id": "duygu"}, + {"label": "PERSON", "pattern": "Dina", "id": "dina"}, + {"label": "ORG", "pattern": "DinaCorp", "id": "dina"}, {"label": "ORG", "pattern": "ACME", "id": "acme"}, ] ruler.add_patterns(patterns) - doc = ruler(nlp.make_doc("Duygu founded DuyguCorp and ACME.")) + doc = nlp("Dina founded DinaCorp and ACME.") assert len(ruler.patterns) == 3 - assert "PERSON||duygu" in ruler.phrase_matcher - assert "ORG||duygu" in ruler.phrase_matcher + if isinstance(ruler, EntityRuler): + assert "PERSON||dina" in ruler.phrase_matcher + assert "ORG||dina" in ruler.phrase_matcher assert len(doc.ents) == 3 - ruler.remove("duygu") - doc = ruler(nlp.make_doc("Duygu founded DuyguCorp and ACME.")) + if isinstance(ruler, EntityRuler): + ruler.remove("dina") + else: + ruler.remove_by_id("dina") + doc = nlp("Dina founded DinaCorp and ACME.") assert len(ruler.patterns) == 1 - assert "PERSON||duygu" not in ruler.phrase_matcher - assert "ORG||duygu" not in ruler.phrase_matcher + if isinstance(ruler, EntityRuler): + assert "PERSON||dina" not in ruler.phrase_matcher + assert "ORG||dina" not in ruler.phrase_matcher assert len(doc.ents) == 1 -def test_entity_ruler_remove_nonexisting_pattern(nlp): - ruler = EntityRuler(nlp) +@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS) +def test_entity_ruler_remove_nonexisting_pattern(nlp, entity_ruler_factory): + ruler = nlp.add_pipe(entity_ruler_factory, name="entity_ruler") patterns = [ - {"label": "PERSON", "pattern": "Duygu", "id": "duygu"}, + {"label": "PERSON", "pattern": "Dina", "id": "dina"}, {"label": "ORG", "pattern": "ACME", "id": "acme"}, {"label": "ORG", "pattern": "ACM"}, ] @@ -428,82 +486,108 @@ def test_entity_ruler_remove_nonexisting_pattern(nlp): assert len(ruler.patterns) == 3 with pytest.raises(ValueError): ruler.remove("nepattern") - assert len(ruler.patterns) == 3 + if isinstance(ruler, SpanRuler): + with pytest.raises(ValueError): + ruler.remove_by_id("nepattern") -def test_entity_ruler_remove_several_patterns(nlp): - ruler = EntityRuler(nlp) +@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS) +def test_entity_ruler_remove_several_patterns(nlp, entity_ruler_factory): + ruler = nlp.add_pipe(entity_ruler_factory, name="entity_ruler") patterns = [ - {"label": "PERSON", "pattern": "Duygu", "id": "duygu"}, + {"label": "PERSON", "pattern": "Dina", "id": "dina"}, {"label": "ORG", "pattern": "ACME", "id": "acme"}, {"label": "ORG", "pattern": "ACM"}, ] ruler.add_patterns(patterns) - doc = ruler(nlp.make_doc("Duygu founded her company ACME.")) + doc = nlp("Dina founded her company ACME.") assert len(ruler.patterns) == 3 assert len(doc.ents) == 2 assert doc.ents[0].label_ == "PERSON" - assert doc.ents[0].text == "Duygu" + assert doc.ents[0].text == "Dina" assert doc.ents[1].label_ == "ORG" assert doc.ents[1].text == "ACME" - ruler.remove("duygu") - doc = ruler(nlp.make_doc("Duygu founded her company ACME")) + if isinstance(ruler, EntityRuler): + ruler.remove("dina") + else: + ruler.remove_by_id("dina") + doc = nlp("Dina founded her company ACME") assert len(ruler.patterns) == 2 assert len(doc.ents) == 1 assert doc.ents[0].label_ == "ORG" assert doc.ents[0].text == "ACME" - ruler.remove("acme") - doc = ruler(nlp.make_doc("Duygu founded her company ACME")) + if isinstance(ruler, EntityRuler): + ruler.remove("acme") + else: + ruler.remove_by_id("acme") + doc = nlp("Dina founded her company ACME") assert len(ruler.patterns) == 1 assert len(doc.ents) == 0 -def test_entity_ruler_remove_patterns_in_a_row(nlp): - ruler = EntityRuler(nlp) +@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS) +def test_entity_ruler_remove_patterns_in_a_row(nlp, entity_ruler_factory): + ruler = nlp.add_pipe(entity_ruler_factory, name="entity_ruler") patterns = [ - {"label": "PERSON", "pattern": "Duygu", "id": "duygu"}, + {"label": "PERSON", "pattern": "Dina", "id": "dina"}, {"label": "ORG", "pattern": "ACME", "id": "acme"}, {"label": "DATE", "pattern": "her birthday", "id": "bday"}, {"label": "ORG", "pattern": "ACM"}, ] ruler.add_patterns(patterns) - doc = ruler(nlp.make_doc("Duygu founded her company ACME on her birthday")) + doc = nlp("Dina founded her company ACME on her birthday") assert len(doc.ents) == 3 assert doc.ents[0].label_ == "PERSON" - assert doc.ents[0].text == "Duygu" + assert doc.ents[0].text == "Dina" assert doc.ents[1].label_ == "ORG" assert doc.ents[1].text == "ACME" assert doc.ents[2].label_ == "DATE" assert doc.ents[2].text == "her birthday" - ruler.remove("duygu") - ruler.remove("acme") - ruler.remove("bday") - doc = ruler(nlp.make_doc("Duygu went to school")) + if isinstance(ruler, EntityRuler): + ruler.remove("dina") + ruler.remove("acme") + ruler.remove("bday") + else: + ruler.remove_by_id("dina") + ruler.remove_by_id("acme") + ruler.remove_by_id("bday") + doc = nlp("Dina went to school") assert len(doc.ents) == 0 -def test_entity_ruler_remove_all_patterns(nlp): - ruler = EntityRuler(nlp) +@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS) +def test_entity_ruler_remove_all_patterns(nlp, entity_ruler_factory): + ruler = nlp.add_pipe(entity_ruler_factory, name="entity_ruler") patterns = [ - {"label": "PERSON", "pattern": "Duygu", "id": "duygu"}, + {"label": "PERSON", "pattern": "Dina", "id": "dina"}, {"label": "ORG", "pattern": "ACME", "id": "acme"}, {"label": "DATE", "pattern": "her birthday", "id": "bday"}, ] ruler.add_patterns(patterns) assert len(ruler.patterns) == 3 - ruler.remove("duygu") + if isinstance(ruler, EntityRuler): + ruler.remove("dina") + else: + ruler.remove_by_id("dina") assert len(ruler.patterns) == 2 - ruler.remove("acme") + if isinstance(ruler, EntityRuler): + ruler.remove("acme") + else: + ruler.remove_by_id("acme") assert len(ruler.patterns) == 1 - ruler.remove("bday") + if isinstance(ruler, EntityRuler): + ruler.remove("bday") + else: + ruler.remove_by_id("bday") assert len(ruler.patterns) == 0 with pytest.warns(UserWarning): - doc = ruler(nlp.make_doc("Duygu founded her company ACME on her birthday")) + doc = nlp("Dina founded her company ACME on her birthday") assert len(doc.ents) == 0 -def test_entity_ruler_remove_and_add(nlp): - ruler = EntityRuler(nlp) +@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS) +def test_entity_ruler_remove_and_add(nlp, entity_ruler_factory): + ruler = nlp.add_pipe(entity_ruler_factory, name="entity_ruler") patterns = [{"label": "DATE", "pattern": "last time"}] ruler.add_patterns(patterns) doc = ruler( @@ -524,7 +608,10 @@ def test_entity_ruler_remove_and_add(nlp): assert doc.ents[0].text == "last time" assert doc.ents[1].label_ == "DATE" assert doc.ents[1].text == "this time" - ruler.remove("ttime") + if isinstance(ruler, EntityRuler): + ruler.remove("ttime") + else: + ruler.remove_by_id("ttime") doc = ruler( nlp.make_doc("I saw him last time we met, this time he brought some flowers") ) @@ -547,7 +634,10 @@ def test_entity_ruler_remove_and_add(nlp): ) assert len(ruler.patterns) == 3 assert len(doc.ents) == 3 - ruler.remove("ttime") + if isinstance(ruler, EntityRuler): + ruler.remove("ttime") + else: + ruler.remove_by_id("ttime") doc = ruler( nlp.make_doc( "I saw him last time we met, this time he brought some flowers, another time some chocolate." diff --git a/spacy/tests/pipeline/test_pipe_factories.py b/spacy/tests/pipeline/test_pipe_factories.py index 4128e2a48..232b0512e 100644 --- a/spacy/tests/pipeline/test_pipe_factories.py +++ b/spacy/tests/pipeline/test_pipe_factories.py @@ -119,6 +119,7 @@ def test_pipe_class_component_config(): self.value1 = value1 self.value2 = value2 self.is_base = True + self.name = name def __call__(self, doc: Doc) -> Doc: return doc @@ -141,12 +142,16 @@ def test_pipe_class_component_config(): nlp.add_pipe(name) with pytest.raises(ConfigValidationError): # invalid config nlp.add_pipe(name, config={"value1": "10", "value2": "hello"}) - nlp.add_pipe(name, config={"value1": 10, "value2": "hello"}) + with pytest.warns(UserWarning): + nlp.add_pipe( + name, config={"value1": 10, "value2": "hello", "name": "wrong_name"} + ) pipe = nlp.get_pipe(name) assert isinstance(pipe.nlp, Language) assert pipe.value1 == 10 assert pipe.value2 == "hello" assert pipe.is_base is True + assert pipe.name == name nlp_en = English() with pytest.raises(ConfigValidationError): # invalid config diff --git a/spacy/tests/pipeline/test_pipe_methods.py b/spacy/tests/pipeline/test_pipe_methods.py index 4b8fb8ebc..6f00a1cd9 100644 --- a/spacy/tests/pipeline/test_pipe_methods.py +++ b/spacy/tests/pipeline/test_pipe_methods.py @@ -4,13 +4,14 @@ import numpy import pytest from thinc.api import get_current_ops +import spacy from spacy.lang.en import English from spacy.lang.en.syntax_iterators import noun_chunks from spacy.language import Language from spacy.pipeline import TrainablePipe from spacy.tokens import Doc from spacy.training import Example -from spacy.util import SimpleFrozenList, get_arg_names +from spacy.util import SimpleFrozenList, get_arg_names, make_tempdir from spacy.vocab import Vocab @@ -602,3 +603,52 @@ def test_update_with_annotates(): assert results[component] == "".join(eg.predicted.text for eg in examples) for component in components - set(components_to_annotate): assert results[component] == "" + + +def test_load_disable_enable() -> None: + """ + Tests spacy.load() with dis-/enabling components. + """ + + base_nlp = English() + for pipe in ("sentencizer", "tagger", "parser"): + base_nlp.add_pipe(pipe) + + with make_tempdir() as tmp_dir: + base_nlp.to_disk(tmp_dir) + to_disable = ["parser", "tagger"] + to_enable = ["tagger", "parser"] + + # Setting only `disable`. + nlp = spacy.load(tmp_dir, disable=to_disable) + assert all([comp_name in nlp.disabled for comp_name in to_disable]) + + # Setting only `enable`. + nlp = spacy.load(tmp_dir, enable=to_enable) + assert all( + [ + (comp_name in nlp.disabled) is (comp_name not in to_enable) + for comp_name in nlp.component_names + ] + ) + + # Testing consistent enable/disable combination. + nlp = spacy.load( + tmp_dir, + enable=to_enable, + disable=[ + comp_name + for comp_name in nlp.component_names + if comp_name not in to_enable + ], + ) + assert all( + [ + (comp_name in nlp.disabled) is (comp_name not in to_enable) + for comp_name in nlp.component_names + ] + ) + + # Inconsistent enable/disable combination. + with pytest.raises(ValueError): + spacy.load(tmp_dir, enable=to_enable, disable=["parser"]) diff --git a/spacy/tests/pipeline/test_span_ruler.py b/spacy/tests/pipeline/test_span_ruler.py new file mode 100644 index 000000000..794815359 --- /dev/null +++ b/spacy/tests/pipeline/test_span_ruler.py @@ -0,0 +1,465 @@ +import pytest + +import spacy +from spacy import registry +from spacy.errors import MatchPatternError +from spacy.tokens import Span +from spacy.training import Example +from spacy.tests.util import make_tempdir + +from thinc.api import NumpyOps, get_current_ops + + +@pytest.fixture +@registry.misc("span_ruler_patterns") +def patterns(): + return [ + {"label": "HELLO", "pattern": "hello world", "id": "hello1"}, + {"label": "BYE", "pattern": [{"LOWER": "bye"}, {"LOWER": "bye"}]}, + {"label": "HELLO", "pattern": [{"ORTH": "HELLO"}], "id": "hello2"}, + {"label": "COMPLEX", "pattern": [{"ORTH": "foo", "OP": "*"}]}, + {"label": "TECH_ORG", "pattern": "Apple"}, + {"label": "TECH_ORG", "pattern": "Microsoft"}, + ] + + +@pytest.fixture +def overlapping_patterns(): + return [ + {"label": "FOOBAR", "pattern": "foo bar"}, + {"label": "BARBAZ", "pattern": "bar baz"}, + ] + + +@pytest.fixture +def person_org_patterns(): + return [ + {"label": "PERSON", "pattern": "Dina"}, + {"label": "ORG", "pattern": "ACME"}, + {"label": "ORG", "pattern": "ACM"}, + ] + + +@pytest.fixture +def person_org_date_patterns(person_org_patterns): + return person_org_patterns + [{"label": "DATE", "pattern": "June 14th"}] + + +def test_span_ruler_add_empty(patterns): + """Test that patterns don't get added excessively.""" + nlp = spacy.blank("xx") + ruler = nlp.add_pipe("span_ruler", config={"validate": True}) + ruler.add_patterns(patterns) + pattern_count = sum(len(mm) for mm in ruler.matcher._patterns.values()) + assert pattern_count > 0 + ruler.add_patterns([]) + after_count = sum(len(mm) for mm in ruler.matcher._patterns.values()) + assert after_count == pattern_count + + +def test_span_ruler_init(patterns): + nlp = spacy.blank("xx") + ruler = nlp.add_pipe("span_ruler") + ruler.add_patterns(patterns) + assert len(ruler) == len(patterns) + assert len(ruler.labels) == 4 + assert "HELLO" in ruler + assert "BYE" in ruler + doc = nlp("hello world bye bye") + assert len(doc.spans["ruler"]) == 2 + assert doc.spans["ruler"][0].label_ == "HELLO" + assert doc.spans["ruler"][0].id_ == "hello1" + assert doc.spans["ruler"][1].label_ == "BYE" + assert doc.spans["ruler"][1].id_ == "" + + +def test_span_ruler_no_patterns_warns(): + nlp = spacy.blank("xx") + ruler = nlp.add_pipe("span_ruler") + assert len(ruler) == 0 + assert len(ruler.labels) == 0 + assert nlp.pipe_names == ["span_ruler"] + with pytest.warns(UserWarning): + doc = nlp("hello world bye bye") + assert len(doc.spans["ruler"]) == 0 + + +def test_span_ruler_init_patterns(patterns): + # initialize with patterns + nlp = spacy.blank("xx") + ruler = nlp.add_pipe("span_ruler") + assert len(ruler.labels) == 0 + ruler.initialize(lambda: [], patterns=patterns) + assert len(ruler.labels) == 4 + doc = nlp("hello world bye bye") + assert doc.spans["ruler"][0].label_ == "HELLO" + assert doc.spans["ruler"][1].label_ == "BYE" + nlp.remove_pipe("span_ruler") + # initialize with patterns from misc registry + nlp.config["initialize"]["components"]["span_ruler"] = { + "patterns": {"@misc": "span_ruler_patterns"} + } + ruler = nlp.add_pipe("span_ruler") + assert len(ruler.labels) == 0 + nlp.initialize() + assert len(ruler.labels) == 4 + doc = nlp("hello world bye bye") + assert doc.spans["ruler"][0].label_ == "HELLO" + assert doc.spans["ruler"][1].label_ == "BYE" + + +def test_span_ruler_init_clear(patterns): + """Test that initialization clears patterns.""" + nlp = spacy.blank("xx") + ruler = nlp.add_pipe("span_ruler") + ruler.add_patterns(patterns) + assert len(ruler.labels) == 4 + ruler.initialize(lambda: []) + assert len(ruler.labels) == 0 + + +def test_span_ruler_clear(patterns): + nlp = spacy.blank("xx") + ruler = nlp.add_pipe("span_ruler") + ruler.add_patterns(patterns) + assert len(ruler.labels) == 4 + doc = nlp("hello world") + assert len(doc.spans["ruler"]) == 1 + ruler.clear() + assert len(ruler.labels) == 0 + with pytest.warns(UserWarning): + doc = nlp("hello world") + assert len(doc.spans["ruler"]) == 0 + + +def test_span_ruler_existing(patterns): + nlp = spacy.blank("xx") + ruler = nlp.add_pipe("span_ruler", config={"overwrite": False}) + ruler.add_patterns(patterns) + doc = nlp.make_doc("OH HELLO WORLD bye bye") + doc.spans["ruler"] = [doc[0:2]] + doc = nlp(doc) + assert len(doc.spans["ruler"]) == 3 + assert doc.spans["ruler"][0] == doc[0:2] + assert doc.spans["ruler"][1].label_ == "HELLO" + assert doc.spans["ruler"][1].id_ == "hello2" + assert doc.spans["ruler"][2].label_ == "BYE" + assert doc.spans["ruler"][2].id_ == "" + + +def test_span_ruler_existing_overwrite(patterns): + nlp = spacy.blank("xx") + ruler = nlp.add_pipe("span_ruler", config={"overwrite": True}) + ruler.add_patterns(patterns) + doc = nlp.make_doc("OH HELLO WORLD bye bye") + doc.spans["ruler"] = [doc[0:2]] + doc = nlp(doc) + assert len(doc.spans["ruler"]) == 2 + assert doc.spans["ruler"][0].label_ == "HELLO" + assert doc.spans["ruler"][0].text == "HELLO" + assert doc.spans["ruler"][1].label_ == "BYE" + + +def test_span_ruler_serialize_bytes(patterns): + nlp = spacy.blank("xx") + ruler = nlp.add_pipe("span_ruler") + ruler.add_patterns(patterns) + assert len(ruler) == len(patterns) + assert len(ruler.labels) == 4 + ruler_bytes = ruler.to_bytes() + new_nlp = spacy.blank("xx") + new_ruler = new_nlp.add_pipe("span_ruler") + assert len(new_ruler) == 0 + assert len(new_ruler.labels) == 0 + new_ruler = new_ruler.from_bytes(ruler_bytes) + assert len(new_ruler) == len(patterns) + assert len(new_ruler.labels) == 4 + assert len(new_ruler.patterns) == len(ruler.patterns) + for pattern in ruler.patterns: + assert pattern in new_ruler.patterns + assert sorted(new_ruler.labels) == sorted(ruler.labels) + + +def test_span_ruler_validate(): + nlp = spacy.blank("xx") + ruler = nlp.add_pipe("span_ruler") + validated_ruler = nlp.add_pipe( + "span_ruler", name="validated_span_ruler", config={"validate": True} + ) + + valid_pattern = {"label": "HELLO", "pattern": [{"LOWER": "HELLO"}]} + invalid_pattern = {"label": "HELLO", "pattern": [{"ASDF": "HELLO"}]} + + # invalid pattern raises error without validate + with pytest.raises(ValueError): + ruler.add_patterns([invalid_pattern]) + + # valid pattern is added without errors with validate + validated_ruler.add_patterns([valid_pattern]) + + # invalid pattern raises error with validate + with pytest.raises(MatchPatternError): + validated_ruler.add_patterns([invalid_pattern]) + + +def test_span_ruler_properties(patterns): + nlp = spacy.blank("xx") + ruler = nlp.add_pipe("span_ruler", config={"overwrite": True}) + ruler.add_patterns(patterns) + assert sorted(ruler.labels) == sorted(set([p["label"] for p in patterns])) + + +def test_span_ruler_overlapping_spans(overlapping_patterns): + nlp = spacy.blank("xx") + ruler = nlp.add_pipe("span_ruler") + ruler.add_patterns(overlapping_patterns) + doc = ruler(nlp.make_doc("foo bar baz")) + assert len(doc.spans["ruler"]) == 2 + assert doc.spans["ruler"][0].label_ == "FOOBAR" + assert doc.spans["ruler"][1].label_ == "BARBAZ" + + +def test_span_ruler_scorer(overlapping_patterns): + nlp = spacy.blank("xx") + ruler = nlp.add_pipe("span_ruler") + ruler.add_patterns(overlapping_patterns) + text = "foo bar baz" + pred_doc = ruler(nlp.make_doc(text)) + assert len(pred_doc.spans["ruler"]) == 2 + assert pred_doc.spans["ruler"][0].label_ == "FOOBAR" + assert pred_doc.spans["ruler"][1].label_ == "BARBAZ" + + ref_doc = nlp.make_doc(text) + ref_doc.spans["ruler"] = [Span(ref_doc, 0, 2, label="FOOBAR")] + scores = nlp.evaluate([Example(pred_doc, ref_doc)]) + assert scores["spans_ruler_p"] == 0.5 + assert scores["spans_ruler_r"] == 1.0 + + +@pytest.mark.parametrize("n_process", [1, 2]) +def test_span_ruler_multiprocessing(n_process): + if isinstance(get_current_ops, NumpyOps) or n_process < 2: + texts = ["I enjoy eating Pizza Hut pizza."] + + patterns = [{"label": "FASTFOOD", "pattern": "Pizza Hut"}] + + nlp = spacy.blank("xx") + ruler = nlp.add_pipe("span_ruler") + ruler.add_patterns(patterns) + + for doc in nlp.pipe(texts, n_process=2): + for ent in doc.spans["ruler"]: + assert ent.label_ == "FASTFOOD" + + +def test_span_ruler_serialize_dir(patterns): + nlp = spacy.blank("xx") + ruler = nlp.add_pipe("span_ruler") + ruler.add_patterns(patterns) + with make_tempdir() as d: + ruler.to_disk(d / "test_ruler") + ruler.from_disk(d / "test_ruler") # read from an existing directory + with pytest.raises(ValueError): + ruler.from_disk(d / "non_existing_dir") # read from a bad directory + + +def test_span_ruler_remove_basic(person_org_patterns): + nlp = spacy.blank("xx") + ruler = nlp.add_pipe("span_ruler") + ruler.add_patterns(person_org_patterns) + doc = ruler(nlp.make_doc("Dina went to school")) + assert len(ruler.patterns) == 3 + assert len(doc.spans["ruler"]) == 1 + assert doc.spans["ruler"][0].label_ == "PERSON" + assert doc.spans["ruler"][0].text == "Dina" + ruler.remove("PERSON") + doc = ruler(nlp.make_doc("Dina went to school")) + assert len(doc.spans["ruler"]) == 0 + assert len(ruler.patterns) == 2 + + +def test_span_ruler_remove_nonexisting_pattern(person_org_patterns): + nlp = spacy.blank("xx") + ruler = nlp.add_pipe("span_ruler") + ruler.add_patterns(person_org_patterns) + assert len(ruler.patterns) == 3 + with pytest.raises(ValueError): + ruler.remove("NE") + with pytest.raises(ValueError): + ruler.remove_by_id("NE") + + +def test_span_ruler_remove_several_patterns(person_org_patterns): + nlp = spacy.blank("xx") + ruler = nlp.add_pipe("span_ruler") + ruler.add_patterns(person_org_patterns) + doc = ruler(nlp.make_doc("Dina founded the company ACME.")) + assert len(ruler.patterns) == 3 + assert len(doc.spans["ruler"]) == 2 + assert doc.spans["ruler"][0].label_ == "PERSON" + assert doc.spans["ruler"][0].text == "Dina" + assert doc.spans["ruler"][1].label_ == "ORG" + assert doc.spans["ruler"][1].text == "ACME" + ruler.remove("PERSON") + doc = ruler(nlp.make_doc("Dina founded the company ACME")) + assert len(ruler.patterns) == 2 + assert len(doc.spans["ruler"]) == 1 + assert doc.spans["ruler"][0].label_ == "ORG" + assert doc.spans["ruler"][0].text == "ACME" + ruler.remove("ORG") + with pytest.warns(UserWarning): + doc = ruler(nlp.make_doc("Dina founded the company ACME")) + assert len(ruler.patterns) == 0 + assert len(doc.spans["ruler"]) == 0 + + +def test_span_ruler_remove_patterns_in_a_row(person_org_date_patterns): + nlp = spacy.blank("xx") + ruler = nlp.add_pipe("span_ruler") + ruler.add_patterns(person_org_date_patterns) + doc = ruler(nlp.make_doc("Dina founded the company ACME on June 14th")) + assert len(doc.spans["ruler"]) == 3 + assert doc.spans["ruler"][0].label_ == "PERSON" + assert doc.spans["ruler"][0].text == "Dina" + assert doc.spans["ruler"][1].label_ == "ORG" + assert doc.spans["ruler"][1].text == "ACME" + assert doc.spans["ruler"][2].label_ == "DATE" + assert doc.spans["ruler"][2].text == "June 14th" + ruler.remove("ORG") + ruler.remove("DATE") + doc = ruler(nlp.make_doc("Dina went to school")) + assert len(doc.spans["ruler"]) == 1 + + +def test_span_ruler_remove_all_patterns(person_org_date_patterns): + nlp = spacy.blank("xx") + ruler = nlp.add_pipe("span_ruler") + ruler.add_patterns(person_org_date_patterns) + assert len(ruler.patterns) == 4 + ruler.remove("PERSON") + assert len(ruler.patterns) == 3 + ruler.remove("ORG") + assert len(ruler.patterns) == 1 + ruler.remove("DATE") + assert len(ruler.patterns) == 0 + with pytest.warns(UserWarning): + doc = ruler(nlp.make_doc("Dina founded the company ACME on June 14th")) + assert len(doc.spans["ruler"]) == 0 + + +def test_span_ruler_remove_and_add(): + nlp = spacy.blank("xx") + ruler = nlp.add_pipe("span_ruler") + patterns1 = [{"label": "DATE1", "pattern": "last time"}] + ruler.add_patterns(patterns1) + doc = ruler( + nlp.make_doc("I saw him last time we met, this time he brought some flowers") + ) + assert len(ruler.patterns) == 1 + assert len(doc.spans["ruler"]) == 1 + assert doc.spans["ruler"][0].label_ == "DATE1" + assert doc.spans["ruler"][0].text == "last time" + patterns2 = [{"label": "DATE2", "pattern": "this time"}] + ruler.add_patterns(patterns2) + doc = ruler( + nlp.make_doc("I saw him last time we met, this time he brought some flowers") + ) + assert len(ruler.patterns) == 2 + assert len(doc.spans["ruler"]) == 2 + assert doc.spans["ruler"][0].label_ == "DATE1" + assert doc.spans["ruler"][0].text == "last time" + assert doc.spans["ruler"][1].label_ == "DATE2" + assert doc.spans["ruler"][1].text == "this time" + ruler.remove("DATE1") + doc = ruler( + nlp.make_doc("I saw him last time we met, this time he brought some flowers") + ) + assert len(ruler.patterns) == 1 + assert len(doc.spans["ruler"]) == 1 + assert doc.spans["ruler"][0].label_ == "DATE2" + assert doc.spans["ruler"][0].text == "this time" + ruler.add_patterns(patterns1) + doc = ruler( + nlp.make_doc("I saw him last time we met, this time he brought some flowers") + ) + assert len(ruler.patterns) == 2 + assert len(doc.spans["ruler"]) == 2 + patterns3 = [{"label": "DATE3", "pattern": "another time"}] + ruler.add_patterns(patterns3) + doc = ruler( + nlp.make_doc( + "I saw him last time we met, this time he brought some flowers, another time some chocolate." + ) + ) + assert len(ruler.patterns) == 3 + assert len(doc.spans["ruler"]) == 3 + ruler.remove("DATE3") + doc = ruler( + nlp.make_doc( + "I saw him last time we met, this time he brought some flowers, another time some chocolate." + ) + ) + assert len(ruler.patterns) == 2 + assert len(doc.spans["ruler"]) == 2 + + +def test_span_ruler_spans_filter(overlapping_patterns): + nlp = spacy.blank("xx") + ruler = nlp.add_pipe( + "span_ruler", + config={"spans_filter": {"@misc": "spacy.first_longest_spans_filter.v1"}}, + ) + ruler.add_patterns(overlapping_patterns) + doc = ruler(nlp.make_doc("foo bar baz")) + assert len(doc.spans["ruler"]) == 1 + assert doc.spans["ruler"][0].label_ == "FOOBAR" + + +def test_span_ruler_ents_default_filter(overlapping_patterns): + nlp = spacy.blank("xx") + ruler = nlp.add_pipe("span_ruler", config={"annotate_ents": True}) + ruler.add_patterns(overlapping_patterns) + doc = ruler(nlp.make_doc("foo bar baz")) + assert len(doc.ents) == 1 + assert doc.ents[0].label_ == "FOOBAR" + + +def test_span_ruler_ents_overwrite_filter(overlapping_patterns): + nlp = spacy.blank("xx") + ruler = nlp.add_pipe( + "span_ruler", + config={ + "annotate_ents": True, + "overwrite": False, + "ents_filter": {"@misc": "spacy.prioritize_new_ents_filter.v1"}, + }, + ) + ruler.add_patterns(overlapping_patterns) + # overlapping ents are clobbered, non-overlapping ents are preserved + doc = nlp.make_doc("foo bar baz a b c") + doc.ents = [Span(doc, 1, 3, label="BARBAZ"), Span(doc, 3, 6, label="ABC")] + doc = ruler(doc) + assert len(doc.ents) == 2 + assert doc.ents[0].label_ == "FOOBAR" + assert doc.ents[1].label_ == "ABC" + + +def test_span_ruler_ents_bad_filter(overlapping_patterns): + @registry.misc("test_pass_through_filter") + def make_pass_through_filter(): + def pass_through_filter(spans1, spans2): + return spans1 + spans2 + + return pass_through_filter + + nlp = spacy.blank("xx") + ruler = nlp.add_pipe( + "span_ruler", + config={ + "annotate_ents": True, + "ents_filter": {"@misc": "test_pass_through_filter"}, + }, + ) + ruler.add_patterns(overlapping_patterns) + with pytest.raises(ValueError): + ruler(nlp.make_doc("foo bar baz")) diff --git a/spacy/tests/pipeline/test_textcat.py b/spacy/tests/pipeline/test_textcat.py index 798dd165e..0bb036a33 100644 --- a/spacy/tests/pipeline/test_textcat.py +++ b/spacy/tests/pipeline/test_textcat.py @@ -382,6 +382,7 @@ def test_implicit_label(name, get_examples): # fmt: off +@pytest.mark.slow @pytest.mark.parametrize( "name,textcat_config", [ @@ -390,7 +391,10 @@ def test_implicit_label(name, get_examples): ("textcat", {"@architectures": "spacy.TextCatBOW.v1", "exclusive_classes": True, "no_output_layer": True, "ngram_size": 3}), ("textcat_multilabel", {"@architectures": "spacy.TextCatBOW.v1", "exclusive_classes": False, "no_output_layer": False, "ngram_size": 3}), ("textcat_multilabel", {"@architectures": "spacy.TextCatBOW.v1", "exclusive_classes": False, "no_output_layer": True, "ngram_size": 3}), - # ENSEMBLE + # ENSEMBLE V1 + ("textcat", {"@architectures": "spacy.TextCatEnsemble.v1", "exclusive_classes": False, "pretrained_vectors": None, "width": 64, "embed_size": 2000, "conv_depth": 2, "window_size": 1, "ngram_size": 1, "dropout": None}), + ("textcat_multilabel", {"@architectures": "spacy.TextCatEnsemble.v1", "exclusive_classes": False, "pretrained_vectors": None, "width": 64, "embed_size": 2000, "conv_depth": 2, "window_size": 1, "ngram_size": 1, "dropout": None}), + # ENSEMBLE V2 ("textcat", {"@architectures": "spacy.TextCatEnsemble.v2", "tok2vec": DEFAULT_TOK2VEC_MODEL, "linear_model": {"@architectures": "spacy.TextCatBOW.v1", "exclusive_classes": True, "no_output_layer": False, "ngram_size": 3}}), ("textcat", {"@architectures": "spacy.TextCatEnsemble.v2", "tok2vec": DEFAULT_TOK2VEC_MODEL, "linear_model": {"@architectures": "spacy.TextCatBOW.v1", "exclusive_classes": True, "no_output_layer": True, "ngram_size": 3}}), ("textcat_multilabel", {"@architectures": "spacy.TextCatEnsemble.v2", "tok2vec": DEFAULT_TOK2VEC_MODEL, "linear_model": {"@architectures": "spacy.TextCatBOW.v1", "exclusive_classes": False, "no_output_layer": False, "ngram_size": 3}}), @@ -643,15 +647,28 @@ def test_overfitting_IO_multi(): # fmt: off +@pytest.mark.slow @pytest.mark.parametrize( "name,train_data,textcat_config", [ + # BOW V1 + ("textcat_multilabel", TRAIN_DATA_MULTI_LABEL, {"@architectures": "spacy.TextCatBOW.v1", "exclusive_classes": False, "ngram_size": 1, "no_output_layer": False}), + ("textcat", TRAIN_DATA_SINGLE_LABEL, {"@architectures": "spacy.TextCatBOW.v1", "exclusive_classes": True, "ngram_size": 4, "no_output_layer": False}), + # ENSEMBLE V1 + ("textcat_multilabel", TRAIN_DATA_MULTI_LABEL, {"@architectures": "spacy.TextCatEnsemble.v1", "exclusive_classes": False, "pretrained_vectors": None, "width": 64, "embed_size": 2000, "conv_depth": 2, "window_size": 1, "ngram_size": 1, "dropout": None}), + ("textcat", TRAIN_DATA_SINGLE_LABEL, {"@architectures": "spacy.TextCatEnsemble.v1", "exclusive_classes": False, "pretrained_vectors": None, "width": 64, "embed_size": 2000, "conv_depth": 2, "window_size": 1, "ngram_size": 1, "dropout": None}), + # CNN V1 + ("textcat", TRAIN_DATA_SINGLE_LABEL, {"@architectures": "spacy.TextCatCNN.v1", "tok2vec": DEFAULT_TOK2VEC_MODEL, "exclusive_classes": True}), + ("textcat_multilabel", TRAIN_DATA_MULTI_LABEL, {"@architectures": "spacy.TextCatCNN.v1", "tok2vec": DEFAULT_TOK2VEC_MODEL, "exclusive_classes": False}), + # BOW V2 ("textcat_multilabel", TRAIN_DATA_MULTI_LABEL, {"@architectures": "spacy.TextCatBOW.v2", "exclusive_classes": False, "ngram_size": 1, "no_output_layer": False}), ("textcat", TRAIN_DATA_SINGLE_LABEL, {"@architectures": "spacy.TextCatBOW.v2", "exclusive_classes": True, "ngram_size": 4, "no_output_layer": False}), ("textcat_multilabel", TRAIN_DATA_MULTI_LABEL, {"@architectures": "spacy.TextCatBOW.v2", "exclusive_classes": False, "ngram_size": 3, "no_output_layer": True}), ("textcat", TRAIN_DATA_SINGLE_LABEL, {"@architectures": "spacy.TextCatBOW.v2", "exclusive_classes": True, "ngram_size": 2, "no_output_layer": True}), + # ENSEMBLE V2 ("textcat_multilabel", TRAIN_DATA_MULTI_LABEL, {"@architectures": "spacy.TextCatEnsemble.v2", "tok2vec": DEFAULT_TOK2VEC_MODEL, "linear_model": {"@architectures": "spacy.TextCatBOW.v2", "exclusive_classes": False, "ngram_size": 1, "no_output_layer": False}}), ("textcat", TRAIN_DATA_SINGLE_LABEL, {"@architectures": "spacy.TextCatEnsemble.v2", "tok2vec": DEFAULT_TOK2VEC_MODEL, "linear_model": {"@architectures": "spacy.TextCatBOW.v2", "exclusive_classes": True, "ngram_size": 5, "no_output_layer": False}}), + # CNN V2 ("textcat", TRAIN_DATA_SINGLE_LABEL, {"@architectures": "spacy.TextCatCNN.v2", "tok2vec": DEFAULT_TOK2VEC_MODEL, "exclusive_classes": True}), ("textcat_multilabel", TRAIN_DATA_MULTI_LABEL, {"@architectures": "spacy.TextCatCNN.v2", "tok2vec": DEFAULT_TOK2VEC_MODEL, "exclusive_classes": False}), ], diff --git a/spacy/tests/pipeline/test_tok2vec.py b/spacy/tests/pipeline/test_tok2vec.py index 37104c78a..64faf133d 100644 --- a/spacy/tests/pipeline/test_tok2vec.py +++ b/spacy/tests/pipeline/test_tok2vec.py @@ -1,13 +1,13 @@ import pytest from spacy.ml.models.tok2vec import build_Tok2Vec_model -from spacy.ml.models.tok2vec import MultiHashEmbed, CharacterEmbed -from spacy.ml.models.tok2vec import MishWindowEncoder, MaxoutWindowEncoder +from spacy.ml.models.tok2vec import MultiHashEmbed, MaxoutWindowEncoder from spacy.pipeline.tok2vec import Tok2Vec, Tok2VecListener from spacy.vocab import Vocab from spacy.tokens import Doc from spacy.training import Example from spacy import util from spacy.lang.en import English +from spacy.util import registry from thinc.api import Config, get_current_ops from numpy.testing import assert_array_equal @@ -55,24 +55,41 @@ def test_tok2vec_batch_sizes(batch_size, width, embed_size): assert doc_vec.shape == (len(doc), width) +@pytest.mark.slow +@pytest.mark.parametrize("width", [8]) @pytest.mark.parametrize( - "width,embed_arch,embed_config,encode_arch,encode_config", + "embed_arch,embed_config", # fmt: off [ - (8, MultiHashEmbed, {"rows": [100, 100], "attrs": ["SHAPE", "LOWER"], "include_static_vectors": False}, MaxoutWindowEncoder, {"window_size": 1, "maxout_pieces": 3, "depth": 2}), - (8, MultiHashEmbed, {"rows": [100, 20], "attrs": ["ORTH", "PREFIX"], "include_static_vectors": False}, MishWindowEncoder, {"window_size": 1, "depth": 6}), - (8, CharacterEmbed, {"rows": 100, "nM": 64, "nC": 8, "include_static_vectors": False}, MaxoutWindowEncoder, {"window_size": 1, "maxout_pieces": 3, "depth": 3}), - (8, CharacterEmbed, {"rows": 100, "nM": 16, "nC": 2, "include_static_vectors": False}, MishWindowEncoder, {"window_size": 1, "depth": 3}), + ("spacy.MultiHashEmbed.v1", {"rows": [100, 100], "attrs": ["SHAPE", "LOWER"], "include_static_vectors": False}), + ("spacy.MultiHashEmbed.v1", {"rows": [100, 20], "attrs": ["ORTH", "PREFIX"], "include_static_vectors": False}), + ("spacy.CharacterEmbed.v1", {"rows": 100, "nM": 64, "nC": 8, "include_static_vectors": False}), + ("spacy.CharacterEmbed.v1", {"rows": 100, "nM": 16, "nC": 2, "include_static_vectors": False}), ], # fmt: on ) -def test_tok2vec_configs(width, embed_arch, embed_config, encode_arch, encode_config): +@pytest.mark.parametrize( + "tok2vec_arch,encode_arch,encode_config", + # fmt: off + [ + ("spacy.Tok2Vec.v1", "spacy.MaxoutWindowEncoder.v1", {"window_size": 1, "maxout_pieces": 3, "depth": 2}), + ("spacy.Tok2Vec.v2", "spacy.MaxoutWindowEncoder.v2", {"window_size": 1, "maxout_pieces": 3, "depth": 2}), + ("spacy.Tok2Vec.v1", "spacy.MishWindowEncoder.v1", {"window_size": 1, "depth": 6}), + ("spacy.Tok2Vec.v2", "spacy.MishWindowEncoder.v2", {"window_size": 1, "depth": 6}), + ], + # fmt: on +) +def test_tok2vec_configs( + width, tok2vec_arch, embed_arch, embed_config, encode_arch, encode_config +): + embed = registry.get("architectures", embed_arch) + encode = registry.get("architectures", encode_arch) + tok2vec_model = registry.get("architectures", tok2vec_arch) + embed_config["width"] = width encode_config["width"] = width docs = get_batch(3) - tok2vec = build_Tok2Vec_model( - embed_arch(**embed_config), encode_arch(**encode_config) - ) + tok2vec = tok2vec_model(embed(**embed_config), encode(**encode_config)) tok2vec.initialize(docs) vectors, backprop = tok2vec.begin_update(docs) assert len(vectors) == len(docs) diff --git a/spacy/tests/serialize/test_serialize_span_groups.py b/spacy/tests/serialize/test_serialize_span_groups.py new file mode 100644 index 000000000..85313fcdc --- /dev/null +++ b/spacy/tests/serialize/test_serialize_span_groups.py @@ -0,0 +1,161 @@ +import pytest + +from spacy.tokens import Span, SpanGroup +from spacy.tokens._dict_proxies import SpanGroups + + +@pytest.mark.issue(10685) +def test_issue10685(en_tokenizer): + """Test `SpanGroups` de/serialization""" + # Start with a Doc with no SpanGroups + doc = en_tokenizer("Will it blend?") + + # Test empty `SpanGroups` de/serialization: + assert len(doc.spans) == 0 + doc.spans.from_bytes(doc.spans.to_bytes()) + assert len(doc.spans) == 0 + + # Test non-empty `SpanGroups` de/serialization: + doc.spans["test"] = SpanGroup(doc, name="test", spans=[doc[0:1]]) + doc.spans["test2"] = SpanGroup(doc, name="test", spans=[doc[1:2]]) + + def assert_spangroups(): + assert len(doc.spans) == 2 + assert doc.spans["test"].name == "test" + assert doc.spans["test2"].name == "test" + assert list(doc.spans["test"]) == [doc[0:1]] + assert list(doc.spans["test2"]) == [doc[1:2]] + + # Sanity check the currently-expected behavior + assert_spangroups() + + # Now test serialization/deserialization: + doc.spans.from_bytes(doc.spans.to_bytes()) + + assert_spangroups() + + +def test_span_groups_serialization_mismatches(en_tokenizer): + """Test the serialization of multiple mismatching `SpanGroups` keys and `SpanGroup.name`s""" + doc = en_tokenizer("How now, brown cow?") + # Some variety: + # 1 SpanGroup where its name matches its key + # 2 SpanGroups that have the same name--which is not a key + # 2 SpanGroups that have the same name--which is a key + # 1 SpanGroup that is a value for 2 different keys (where its name is a key) + # 1 SpanGroup that is a value for 2 different keys (where its name is not a key) + groups = doc.spans + groups["key1"] = SpanGroup(doc, name="key1", spans=[doc[0:1], doc[1:2]]) + groups["key2"] = SpanGroup(doc, name="too", spans=[doc[3:4], doc[4:5]]) + groups["key3"] = SpanGroup(doc, name="too", spans=[doc[1:2], doc[0:1]]) + groups["key4"] = SpanGroup(doc, name="key4", spans=[doc[0:1]]) + groups["key5"] = SpanGroup(doc, name="key4", spans=[doc[0:1]]) + sg6 = SpanGroup(doc, name="key6", spans=[doc[0:1]]) + groups["key6"] = sg6 + groups["key7"] = sg6 + sg8 = SpanGroup(doc, name="also", spans=[doc[1:2]]) + groups["key8"] = sg8 + groups["key9"] = sg8 + + regroups = SpanGroups(doc).from_bytes(groups.to_bytes()) + + # Assert regroups == groups + assert regroups.keys() == groups.keys() + for key, regroup in regroups.items(): + # Assert regroup == groups[key] + assert regroup.name == groups[key].name + assert list(regroup) == list(groups[key]) + + +@pytest.mark.parametrize( + "spans_bytes,doc_text,expected_spangroups,expected_warning", + # The bytestrings below were generated from an earlier version of spaCy + # that serialized `SpanGroups` as a list of SpanGroup bytes (via SpanGroups.to_bytes). + # Comments preceding the bytestrings indicate from what Doc they were created. + [ + # Empty SpanGroups: + (b"\x90", "", {}, False), + # doc = nlp("Will it blend?") + # doc.spans['test'] = SpanGroup(doc, name='test', spans=[doc[0:1]]) + ( + b"\x91\xc4C\x83\xa4name\xa4test\xa5attrs\x80\xa5spans\x91\xc4(\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x04", + "Will it blend?", + {"test": {"name": "test", "spans": [(0, 1)]}}, + False, + ), + # doc = nlp("Will it blend?") + # doc.spans['test'] = SpanGroup(doc, name='test', spans=[doc[0:1]]) + # doc.spans['test2'] = SpanGroup(doc, name='test', spans=[doc[1:2]]) + ( + b"\x92\xc4C\x83\xa4name\xa4test\xa5attrs\x80\xa5spans\x91\xc4(\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x04\xc4C\x83\xa4name\xa4test\xa5attrs\x80\xa5spans\x91\xc4(\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x02\x00\x00\x00\x05\x00\x00\x00\x07", + "Will it blend?", + # We expect only 1 SpanGroup to be in doc.spans in this example + # because there are 2 `SpanGroup`s that have the same .name. See #10685. + {"test": {"name": "test", "spans": [(1, 2)]}}, + True, + ), + # doc = nlp('How now, brown cow?') + # doc.spans['key1'] = SpanGroup(doc, name='key1', spans=[doc[0:1], doc[1:2]]) + # doc.spans['key2'] = SpanGroup(doc, name='too', spans=[doc[3:4], doc[4:5]]) + # doc.spans['key3'] = SpanGroup(doc, name='too', spans=[doc[1:2], doc[0:1]]) + # doc.spans['key4'] = SpanGroup(doc, name='key4', spans=[doc[0:1]]) + # doc.spans['key5'] = SpanGroup(doc, name='key4', spans=[doc[0:1]]) + ( + b"\x95\xc4m\x83\xa4name\xa4key1\xa5attrs\x80\xa5spans\x92\xc4(\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x03\xc4(\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x02\x00\x00\x00\x04\x00\x00\x00\x07\xc4l\x83\xa4name\xa3too\xa5attrs\x80\xa5spans\x92\xc4(\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x03\x00\x00\x00\x04\x00\x00\x00\t\x00\x00\x00\x0e\xc4(\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x04\x00\x00\x00\x05\x00\x00\x00\x0f\x00\x00\x00\x12\xc4l\x83\xa4name\xa3too\xa5attrs\x80\xa5spans\x92\xc4(\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x02\x00\x00\x00\x04\x00\x00\x00\x07\xc4(\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x03\xc4C\x83\xa4name\xa4key4\xa5attrs\x80\xa5spans\x91\xc4(\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x03\xc4C\x83\xa4name\xa4key4\xa5attrs\x80\xa5spans\x91\xc4(\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x03", + "How now, brown cow?", + { + "key1": {"name": "key1", "spans": [(0, 1), (1, 2)]}, + "too": {"name": "too", "spans": [(1, 2), (0, 1)]}, + "key4": {"name": "key4", "spans": [(0, 1)]}, + }, + True, + ), + ], +) +def test_deserialize_span_groups_compat( + en_tokenizer, spans_bytes, doc_text, expected_spangroups, expected_warning +): + """Test backwards-compatibility of `SpanGroups` deserialization. + This uses serializations (bytes) from a prior version of spaCy (before 3.3.1). + + spans_bytes (bytes): Serialized `SpanGroups` object. + doc_text (str): Doc text. + expected_spangroups (dict): + Dict mapping every expected (after deserialization) `SpanGroups` key + to a SpanGroup's "args", where a SpanGroup's args are given as a dict: + {"name": span_group.name, + "spans": [(span0.start, span0.end), ...]} + expected_warning (bool): Whether a warning is to be expected from .from_bytes() + --i.e. if more than 1 SpanGroup has the same .name within the `SpanGroups`. + """ + doc = en_tokenizer(doc_text) + + if expected_warning: + with pytest.warns(UserWarning): + doc.spans.from_bytes(spans_bytes) + else: + # TODO: explicitly check for lack of a warning + doc.spans.from_bytes(spans_bytes) + + assert doc.spans.keys() == expected_spangroups.keys() + for name, spangroup_args in expected_spangroups.items(): + assert doc.spans[name].name == spangroup_args["name"] + spans = [Span(doc, start, end) for start, end in spangroup_args["spans"]] + assert list(doc.spans[name]) == spans + + +def test_span_groups_serialization(en_tokenizer): + doc = en_tokenizer("0 1 2 3 4 5 6") + span_groups = SpanGroups(doc) + spans = [doc[0:2], doc[1:3]] + sg1 = SpanGroup(doc, spans=spans) + span_groups["key1"] = sg1 + span_groups["key2"] = sg1 + span_groups["key3"] = [] + reloaded_span_groups = SpanGroups(doc).from_bytes(span_groups.to_bytes()) + assert span_groups.keys() == reloaded_span_groups.keys() + for key, value in span_groups.items(): + assert all( + span == reloaded_span + for span, reloaded_span in zip(span_groups[key], reloaded_span_groups[key]) + ) diff --git a/spacy/tests/test_cli.py b/spacy/tests/test_cli.py index 0fa6f5670..838e00369 100644 --- a/spacy/tests/test_cli.py +++ b/spacy/tests/test_cli.py @@ -1,4 +1,7 @@ import os +import math +from random import sample +from typing import Counter import pytest import srsly @@ -14,6 +17,10 @@ from spacy.cli._util import substitute_project_variables from spacy.cli._util import validate_project_commands from spacy.cli.debug_data import _compile_gold, _get_labels_from_model from spacy.cli.debug_data import _get_labels_from_spancat +from spacy.cli.debug_data import _get_distribution, _get_kl_divergence +from spacy.cli.debug_data import _get_span_characteristics +from spacy.cli.debug_data import _print_span_characteristics +from spacy.cli.debug_data import _get_spans_length_freq_dist from spacy.cli.download import get_compatibility, get_version from spacy.cli.init_config import RECOMMENDATIONS, init_config, fill_config from spacy.cli.package import get_third_party_dependencies @@ -24,6 +31,7 @@ from spacy.lang.nl import Dutch from spacy.language import Language from spacy.schemas import ProjectConfigSchema, RecommendationSchema, validate from spacy.tokens import Doc +from spacy.tokens.span import Span from spacy.training import Example, docs_to_json, offsets_to_biluo_tags from spacy.training.converters import conll_ner_to_docs, conllu_to_docs from spacy.training.converters import iob_to_docs @@ -341,6 +349,7 @@ def test_project_config_validation_full(): "assets": [ { "dest": "x", + "extra": True, "url": "https://example.com", "checksum": "63373dd656daa1fd3043ce166a59474c", }, @@ -352,6 +361,12 @@ def test_project_config_validation_full(): "path": "y", }, }, + { + "dest": "z", + "extra": False, + "url": "https://example.com", + "checksum": "63373dd656daa1fd3043ce166a59474c", + }, ], "commands": [ { @@ -733,3 +748,110 @@ def test_debug_data_compile_gold(): eg = Example(pred, ref) data = _compile_gold([eg], ["ner"], nlp, True) assert data["boundary_cross_ents"] == 1 + + +def test_debug_data_compile_gold_for_spans(): + nlp = English() + spans_key = "sc" + + pred = Doc(nlp.vocab, words=["Welcome", "to", "the", "Bank", "of", "China", "."]) + pred.spans[spans_key] = [Span(pred, 3, 6, "ORG"), Span(pred, 5, 6, "GPE")] + ref = Doc(nlp.vocab, words=["Welcome", "to", "the", "Bank", "of", "China", "."]) + ref.spans[spans_key] = [Span(ref, 3, 6, "ORG"), Span(ref, 5, 6, "GPE")] + eg = Example(pred, ref) + + data = _compile_gold([eg], ["spancat"], nlp, True) + + assert data["spancat"][spans_key] == Counter({"ORG": 1, "GPE": 1}) + assert data["spans_length"][spans_key] == {"ORG": [3], "GPE": [1]} + assert data["spans_per_type"][spans_key] == { + "ORG": [Span(ref, 3, 6, "ORG")], + "GPE": [Span(ref, 5, 6, "GPE")], + } + assert data["sb_per_type"][spans_key] == { + "ORG": {"start": [ref[2:3]], "end": [ref[6:7]]}, + "GPE": {"start": [ref[4:5]], "end": [ref[6:7]]}, + } + + +def test_frequency_distribution_is_correct(): + nlp = English() + docs = [ + Doc(nlp.vocab, words=["Bank", "of", "China"]), + Doc(nlp.vocab, words=["China"]), + ] + + expected = Counter({"china": 0.5, "bank": 0.25, "of": 0.25}) + freq_distribution = _get_distribution(docs, normalize=True) + assert freq_distribution == expected + + +def test_kl_divergence_computation_is_correct(): + p = Counter({"a": 0.5, "b": 0.25}) + q = Counter({"a": 0.25, "b": 0.50, "c": 0.15, "d": 0.10}) + result = _get_kl_divergence(p, q) + expected = 0.1733 + assert math.isclose(result, expected, rel_tol=1e-3) + + +def test_get_span_characteristics_return_value(): + nlp = English() + spans_key = "sc" + + pred = Doc(nlp.vocab, words=["Welcome", "to", "the", "Bank", "of", "China", "."]) + pred.spans[spans_key] = [Span(pred, 3, 6, "ORG"), Span(pred, 5, 6, "GPE")] + ref = Doc(nlp.vocab, words=["Welcome", "to", "the", "Bank", "of", "China", "."]) + ref.spans[spans_key] = [Span(ref, 3, 6, "ORG"), Span(ref, 5, 6, "GPE")] + eg = Example(pred, ref) + + examples = [eg] + data = _compile_gold(examples, ["spancat"], nlp, True) + span_characteristics = _get_span_characteristics( + examples=examples, compiled_gold=data, spans_key=spans_key + ) + + assert {"sd", "bd", "lengths"}.issubset(span_characteristics.keys()) + assert span_characteristics["min_length"] == 1 + assert span_characteristics["max_length"] == 3 + + +def test_ensure_print_span_characteristics_wont_fail(): + """Test if interface between two methods aren't destroyed if refactored""" + nlp = English() + spans_key = "sc" + + pred = Doc(nlp.vocab, words=["Welcome", "to", "the", "Bank", "of", "China", "."]) + pred.spans[spans_key] = [Span(pred, 3, 6, "ORG"), Span(pred, 5, 6, "GPE")] + ref = Doc(nlp.vocab, words=["Welcome", "to", "the", "Bank", "of", "China", "."]) + ref.spans[spans_key] = [Span(ref, 3, 6, "ORG"), Span(ref, 5, 6, "GPE")] + eg = Example(pred, ref) + + examples = [eg] + data = _compile_gold(examples, ["spancat"], nlp, True) + span_characteristics = _get_span_characteristics( + examples=examples, compiled_gold=data, spans_key=spans_key + ) + _print_span_characteristics(span_characteristics) + + +@pytest.mark.parametrize("threshold", [70, 80, 85, 90, 95]) +def test_span_length_freq_dist_threshold_must_be_correct(threshold): + sample_span_lengths = { + "span_type_1": [1, 4, 4, 5], + "span_type_2": [5, 3, 3, 2], + "span_type_3": [3, 1, 3, 3], + } + span_freqs = _get_spans_length_freq_dist(sample_span_lengths, threshold) + assert sum(span_freqs.values()) >= threshold + + +def test_span_length_freq_dist_output_must_be_correct(): + sample_span_lengths = { + "span_type_1": [1, 4, 4, 5], + "span_type_2": [5, 3, 3, 2], + "span_type_3": [3, 1, 3, 3], + } + threshold = 90 + span_freqs = _get_spans_length_freq_dist(sample_span_lengths, threshold) + assert sum(span_freqs.values()) >= threshold + assert list(span_freqs.keys()) == [3, 1, 4, 5, 2] diff --git a/spacy/tests/tokenizer/test_explain.py b/spacy/tests/tokenizer/test_explain.py index 0a10ae67d..5b4eeca16 100644 --- a/spacy/tests/tokenizer/test_explain.py +++ b/spacy/tests/tokenizer/test_explain.py @@ -1,7 +1,13 @@ -import pytest import re -from spacy.util import get_lang_class +import string + +import hypothesis +import hypothesis.strategies +import pytest + +import spacy from spacy.tokenizer import Tokenizer +from spacy.util import get_lang_class # Only include languages with no external dependencies # "is" seems to confuse importlib, so we're also excluding it for now @@ -77,3 +83,46 @@ def test_tokenizer_explain_special_matcher(en_vocab): tokens = [t.text for t in tokenizer("a/a.")] explain_tokens = [t[1] for t in tokenizer.explain("a/a.")] assert tokens == explain_tokens + + +@hypothesis.strategies.composite +def sentence_strategy(draw: hypothesis.strategies.DrawFn, max_n_words: int = 4) -> str: + """ + Composite strategy for fuzzily generating sentence with varying interpunctation. + + draw (hypothesis.strategies.DrawFn): Protocol for drawing function allowing to fuzzily pick from hypothesis' + strategies. + max_n_words (int): Max. number of words in generated sentence. + RETURNS (str): Fuzzily generated sentence. + """ + + punctuation_and_space_regex = "|".join( + [*[re.escape(p) for p in string.punctuation], r"\s"] + ) + sentence = [ + [ + draw(hypothesis.strategies.text(min_size=1)), + draw(hypothesis.strategies.from_regex(punctuation_and_space_regex)), + ] + for _ in range( + draw(hypothesis.strategies.integers(min_value=2, max_value=max_n_words)) + ) + ] + + return " ".join([token for token_pair in sentence for token in token_pair]) + + +@pytest.mark.xfail +@pytest.mark.parametrize("lang", LANGUAGES) +@hypothesis.given(sentence=sentence_strategy()) +def test_tokenizer_explain_fuzzy(lang: str, sentence: str) -> None: + """ + Tests whether output of tokenizer.explain() matches tokenizer output. Input generated by hypothesis. + lang (str): Language to test. + text (str): Fuzzily generated sentence to tokenize. + """ + + tokenizer: Tokenizer = spacy.blank(lang).tokenizer + tokens = [t.text for t in tokenizer(sentence) if not t.is_space] + debug_tokens = [t[1] for t in tokenizer.explain(sentence)] + assert tokens == debug_tokens, f"{tokens}, {debug_tokens}, {sentence}" diff --git a/spacy/tests/training/test_rehearse.py b/spacy/tests/training/test_rehearse.py index 84c507702..5ac7fc217 100644 --- a/spacy/tests/training/test_rehearse.py +++ b/spacy/tests/training/test_rehearse.py @@ -181,7 +181,7 @@ def _optimize(nlp, component: str, data: List, rehearse: bool): elif component == "tagger": _add_tagger_label(pipe, data) elif component == "parser": - _add_tagger_label(pipe, data) + _add_parser_label(pipe, data) elif component == "textcat_multilabel": _add_textcat_label(pipe, data) else: diff --git a/spacy/tests/training/test_training.py b/spacy/tests/training/test_training.py index 8e08a25fb..4384a796d 100644 --- a/spacy/tests/training/test_training.py +++ b/spacy/tests/training/test_training.py @@ -671,13 +671,38 @@ def test_gold_ner_missing_tags(en_tokenizer): def test_projectivize(en_tokenizer): doc = en_tokenizer("He pretty quickly walks away") - heads = [3, 2, 3, 0, 2] + heads = [3, 2, 3, 3, 2] deps = ["dep"] * len(heads) example = Example.from_dict(doc, {"heads": heads, "deps": deps}) proj_heads, proj_labels = example.get_aligned_parse(projectivize=True) nonproj_heads, nonproj_labels = example.get_aligned_parse(projectivize=False) - assert proj_heads == [3, 2, 3, 0, 3] - assert nonproj_heads == [3, 2, 3, 0, 2] + assert proj_heads == [3, 2, 3, 3, 3] + assert nonproj_heads == [3, 2, 3, 3, 2] + + # Test single token documents + doc = en_tokenizer("Conrail") + heads = [0] + deps = ["dep"] + example = Example.from_dict(doc, {"heads": heads, "deps": deps}) + proj_heads, proj_labels = example.get_aligned_parse(projectivize=True) + assert proj_heads == heads + assert proj_labels == deps + + # Test documents with no alignments + doc_a = Doc( + doc.vocab, words=["Double-Jointed"], spaces=[False], deps=["ROOT"], heads=[0] + ) + doc_b = Doc( + doc.vocab, + words=["Double", "-", "Jointed"], + spaces=[True, True, True], + deps=["amod", "punct", "ROOT"], + heads=[2, 2, 2], + ) + example = Example(doc_a, doc_b) + proj_heads, proj_deps = example.get_aligned_parse(projectivize=True) + assert proj_heads == [None] + assert proj_deps == [None] def test_iob_to_biluo(): diff --git a/spacy/tests/util.py b/spacy/tests/util.py index 365ea4349..d5f3c39ff 100644 --- a/spacy/tests/util.py +++ b/spacy/tests/util.py @@ -5,6 +5,7 @@ import srsly from spacy.tokens import Doc from spacy.vocab import Vocab from spacy.util import make_tempdir # noqa: F401 +from spacy.training import split_bilu_label from thinc.api import get_current_ops @@ -40,7 +41,7 @@ def apply_transition_sequence(parser, doc, sequence): desired state.""" for action_name in sequence: if "-" in action_name: - move, label = action_name.split("-") + move, label = split_bilu_label(action_name) parser.add_label(label) with parser.step_through(doc) as stepwise: for transition in sequence: diff --git a/spacy/tests/vocab_vectors/test_similarity.py b/spacy/tests/vocab_vectors/test_similarity.py index 47cd1f060..1efcdd81e 100644 --- a/spacy/tests/vocab_vectors/test_similarity.py +++ b/spacy/tests/vocab_vectors/test_similarity.py @@ -1,6 +1,7 @@ import pytest import numpy from spacy.tokens import Doc +from spacy.vocab import Vocab from ..util import get_cosine, add_vecs_to_vocab @@ -71,19 +72,17 @@ def test_vectors_similarity_DD(vocab, vectors): def test_vectors_similarity_TD(vocab, vectors): [(word1, vec1), (word2, vec2)] = vectors doc = Doc(vocab, words=[word1, word2]) - with pytest.warns(UserWarning): - assert isinstance(doc.similarity(doc[0]), float) - assert isinstance(doc[0].similarity(doc), float) - assert doc.similarity(doc[0]) == doc[0].similarity(doc) + assert isinstance(doc.similarity(doc[0]), float) + assert isinstance(doc[0].similarity(doc), float) + assert doc.similarity(doc[0]) == doc[0].similarity(doc) def test_vectors_similarity_TS(vocab, vectors): [(word1, vec1), (word2, vec2)] = vectors doc = Doc(vocab, words=[word1, word2]) - with pytest.warns(UserWarning): - assert isinstance(doc[:2].similarity(doc[0]), float) - assert isinstance(doc[0].similarity(doc[-2]), float) - assert doc[:2].similarity(doc[0]) == doc[0].similarity(doc[:2]) + assert isinstance(doc[:2].similarity(doc[0]), float) + assert isinstance(doc[0].similarity(doc[:2]), float) + assert doc[:2].similarity(doc[0]) == doc[0].similarity(doc[:2]) def test_vectors_similarity_DS(vocab, vectors): @@ -91,3 +90,21 @@ def test_vectors_similarity_DS(vocab, vectors): doc = Doc(vocab, words=[word1, word2]) assert isinstance(doc.similarity(doc[:2]), float) assert doc.similarity(doc[:2]) == doc[:2].similarity(doc) + + +def test_vectors_similarity_no_vectors(): + vocab = Vocab() + doc1 = Doc(vocab, words=["a", "b"]) + doc2 = Doc(vocab, words=["c", "d", "e"]) + with pytest.warns(UserWarning): + doc1.similarity(doc2) + with pytest.warns(UserWarning): + doc1.similarity(doc2[1]) + with pytest.warns(UserWarning): + doc1.similarity(doc2[:2]) + with pytest.warns(UserWarning): + doc2.similarity(doc1) + with pytest.warns(UserWarning): + doc2[1].similarity(doc1) + with pytest.warns(UserWarning): + doc2[:2].similarity(doc1) diff --git a/spacy/tests/vocab_vectors/test_vectors.py b/spacy/tests/vocab_vectors/test_vectors.py index e3ad206f4..dd2cfc596 100644 --- a/spacy/tests/vocab_vectors/test_vectors.py +++ b/spacy/tests/vocab_vectors/test_vectors.py @@ -318,17 +318,15 @@ def test_vectors_lexeme_doc_similarity(vocab, text): @pytest.mark.parametrize("text", [["apple", "orange", "juice"]]) def test_vectors_span_span_similarity(vocab, text): doc = Doc(vocab, words=text) - with pytest.warns(UserWarning): - assert doc[0:2].similarity(doc[1:3]) == doc[1:3].similarity(doc[0:2]) - assert -1.0 < doc[0:2].similarity(doc[1:3]) < 1.0 + assert doc[0:2].similarity(doc[1:3]) == doc[1:3].similarity(doc[0:2]) + assert -1.0 < doc[0:2].similarity(doc[1:3]) < 1.0 @pytest.mark.parametrize("text", [["apple", "orange", "juice"]]) def test_vectors_span_doc_similarity(vocab, text): doc = Doc(vocab, words=text) - with pytest.warns(UserWarning): - assert doc[0:2].similarity(doc) == doc.similarity(doc[0:2]) - assert -1.0 < doc[0:2].similarity(doc) < 1.0 + assert doc[0:2].similarity(doc) == doc.similarity(doc[0:2]) + assert -1.0 < doc[0:2].similarity(doc) < 1.0 @pytest.mark.parametrize( diff --git a/spacy/tokens/_dict_proxies.py b/spacy/tokens/_dict_proxies.py index 8643243fa..9630da261 100644 --- a/spacy/tokens/_dict_proxies.py +++ b/spacy/tokens/_dict_proxies.py @@ -1,10 +1,11 @@ -from typing import Iterable, Tuple, Union, Optional, TYPE_CHECKING +from typing import Dict, Iterable, List, Tuple, Union, Optional, TYPE_CHECKING +import warnings import weakref from collections import UserDict import srsly from .span_group import SpanGroup -from ..errors import Errors +from ..errors import Errors, Warnings if TYPE_CHECKING: @@ -16,7 +17,7 @@ if TYPE_CHECKING: # Why inherit from UserDict instead of dict here? # Well, the 'dict' class doesn't necessarily delegate everything nicely, # for performance reasons. The UserDict is slower but better behaved. -# See https://treyhunner.com/2019/04/why-you-shouldnt-inherit-from-list-and-dict-in-python/0ww +# See https://treyhunner.com/2019/04/why-you-shouldnt-inherit-from-list-and-dict-in-python/ class SpanGroups(UserDict): """A dict-like proxy held by the Doc, to control access to span groups.""" @@ -43,21 +44,62 @@ class SpanGroups(UserDict): doc = self._ensure_doc() return SpanGroups(doc).from_bytes(self.to_bytes()) + def setdefault(self, key, default=None): + if not isinstance(default, SpanGroup): + if default is None: + spans = [] + else: + spans = default + default = self._make_span_group(key, spans) + return super().setdefault(key, default=default) + def to_bytes(self) -> bytes: - # We don't need to serialize this as a dict, because the groups - # know their names. + # We serialize this as a dict in order to track the key(s) a SpanGroup + # is a value of (in a backward- and forward-compatible way), since + # a SpanGroup can have a key that doesn't match its `.name` (See #10685) if len(self) == 0: return self._EMPTY_BYTES - msg = [value.to_bytes() for value in self.values()] + msg: Dict[bytes, List[str]] = {} + for key, value in self.items(): + msg.setdefault(value.to_bytes(), []).append(key) return srsly.msgpack_dumps(msg) def from_bytes(self, bytes_data: bytes) -> "SpanGroups": - msg = [] if bytes_data == self._EMPTY_BYTES else srsly.msgpack_loads(bytes_data) + # backwards-compatibility: bytes_data may be one of: + # b'', a serialized empty list, a serialized list of SpanGroup bytes + # or a serialized dict of SpanGroup bytes -> keys + msg = ( + [] + if not bytes_data or bytes_data == self._EMPTY_BYTES + else srsly.msgpack_loads(bytes_data) + ) self.clear() doc = self._ensure_doc() - for value_bytes in msg: - group = SpanGroup(doc).from_bytes(value_bytes) - self[group.name] = group + if isinstance(msg, list): + # This is either the 1st version of `SpanGroups` serialization + # or there were no SpanGroups serialized + for value_bytes in msg: + group = SpanGroup(doc).from_bytes(value_bytes) + if group.name in self: + # Display a warning if `msg` contains `SpanGroup`s + # that have the same .name (attribute). + # Because, for `SpanGroups` serialized as lists, + # only 1 SpanGroup per .name is loaded. (See #10685) + warnings.warn( + Warnings.W120.format( + group_name=group.name, group_values=self[group.name] + ) + ) + self[group.name] = group + else: + for value_bytes, keys in msg.items(): + group = SpanGroup(doc).from_bytes(value_bytes) + # Deserialize `SpanGroup`s as copies because it's possible for two + # different `SpanGroup`s (pre-serialization) to have the same bytes + # (since they can have the same `.name`). + self[keys[0]] = group + for key in keys[1:]: + self[key] = group.copy() return self def _ensure_doc(self) -> "Doc": diff --git a/spacy/tokens/doc.pyi b/spacy/tokens/doc.pyi index 7e9340d58..a40fa74aa 100644 --- a/spacy/tokens/doc.pyi +++ b/spacy/tokens/doc.pyi @@ -170,6 +170,9 @@ class Doc: def extend_tensor(self, tensor: Floats2d) -> None: ... def retokenize(self) -> Retokenizer: ... def to_json(self, underscore: Optional[List[str]] = ...) -> Dict[str, Any]: ... + def from_json( + self, doc_json: Dict[str, Any] = ..., validate: bool = False + ) -> Doc: ... def to_utf8_array(self, nr_char: int = ...) -> Ints2d: ... @staticmethod def _get_array_attrs() -> Tuple[Any]: ... diff --git a/spacy/tokens/doc.pyx b/spacy/tokens/doc.pyx index c36e3a02f..d9a104ac8 100644 --- a/spacy/tokens/doc.pyx +++ b/spacy/tokens/doc.pyx @@ -1,4 +1,6 @@ # cython: infer_types=True, bounds_check=False, profile=True +from typing import Set + cimport cython cimport numpy as np from libc.string cimport memcpy @@ -31,10 +33,11 @@ from ..errors import Errors, Warnings from ..morphology import Morphology from .. import util from .. import parts_of_speech +from .. import schemas from .underscore import Underscore, get_ext_args from ._retokenize import Retokenizer from ._serialize import ALL_ATTRS as DOCBIN_ALL_ATTRS - +from ..util import get_words_and_spaces DEF PADDING = 5 @@ -414,6 +417,7 @@ cdef class Doc: """ # empty docs are always annotated + input_attr = attr if self.length == 0: return True cdef int i @@ -423,6 +427,10 @@ cdef class Doc: elif attr == "IS_SENT_END" or attr == self.vocab.strings["IS_SENT_END"]: attr = SENT_START attr = intify_attr(attr) + if attr is None: + raise ValueError( + Errors.E1037.format(attr=input_attr) + ) # adjust attributes if attr == HEAD: # HEAD does not have an unset state, so rely on DEP @@ -511,7 +519,7 @@ cdef class Doc: def doc(self): return self - def char_span(self, int start_idx, int end_idx, label=0, kb_id=0, vector=None, alignment_mode="strict"): + def char_span(self, int start_idx, int end_idx, label=0, kb_id=0, vector=None, alignment_mode="strict", span_id=0): """Create a `Span` object from the slice `doc.text[start_idx : end_idx]`. Returns None if no valid `Span` can be created. @@ -570,7 +578,7 @@ cdef class Doc: start += 1 # Currently we have the token index, we want the range-end index end += 1 - cdef Span span = Span(self, start, end, label=label, kb_id=kb_id, vector=vector) + cdef Span span = Span(self, start, end, label=label, kb_id=kb_id, span_id=span_id, vector=vector) return span def similarity(self, other): @@ -599,7 +607,8 @@ cdef class Doc: if self.vocab.vectors.n_keys == 0: warnings.warn(Warnings.W007.format(obj="Doc")) if self.vector_norm == 0 or other.vector_norm == 0: - warnings.warn(Warnings.W008.format(obj="Doc")) + if not self.has_vector or not other.has_vector: + warnings.warn(Warnings.W008.format(obj="Doc")) return 0.0 vector = self.vector xp = get_array_module(vector) @@ -619,7 +628,7 @@ cdef class Doc: if "has_vector" in self.user_hooks: return self.user_hooks["has_vector"](self) elif self.vocab.vectors.size: - return True + return any(token.has_vector for token in self) elif self.tensor.size: return True else: @@ -708,6 +717,7 @@ cdef class Doc: cdef int start = -1 cdef attr_t label = 0 cdef attr_t kb_id = 0 + cdef attr_t ent_id = 0 output = [] for i in range(self.length): token = &self.c[i] @@ -718,18 +728,20 @@ cdef class Doc: elif token.ent_iob == 2 or token.ent_iob == 0 or \ (token.ent_iob == 3 and token.ent_type == 0): if start != -1: - output.append(Span(self, start, i, label=label, kb_id=kb_id)) + output.append(Span(self, start, i, label=label, kb_id=kb_id, span_id=ent_id)) start = -1 label = 0 kb_id = 0 + ent_id = 0 elif token.ent_iob == 3: if start != -1: - output.append(Span(self, start, i, label=label, kb_id=kb_id)) + output.append(Span(self, start, i, label=label, kb_id=kb_id, span_id=ent_id)) start = i label = token.ent_type kb_id = token.ent_kb_id + ent_id = token.ent_id if start != -1: - output.append(Span(self, start, self.length, label=label, kb_id=kb_id)) + output.append(Span(self, start, self.length, label=label, kb_id=kb_id, span_id=ent_id)) # remove empty-label spans output = [o for o in output if o.label_ != ""] return tuple(output) @@ -738,14 +750,14 @@ cdef class Doc: # TODO: # 1. Test basic data-driven ORTH gazetteer # 2. Test more nuanced date and currency regex - cdef attr_t entity_type, kb_id + cdef attr_t entity_type, kb_id, ent_id cdef int ent_start, ent_end ent_spans = [] for ent_info in ents: - entity_type_, kb_id, ent_start, ent_end = get_entity_info(ent_info) + entity_type_, kb_id, ent_start, ent_end, ent_id = get_entity_info(ent_info) if isinstance(entity_type_, str): self.vocab.strings.add(entity_type_) - span = Span(self, ent_start, ent_end, label=entity_type_, kb_id=kb_id) + span = Span(self, ent_start, ent_end, label=entity_type_, kb_id=kb_id, span_id=ent_id) ent_spans.append(span) self.set_ents(ent_spans, default=SetEntsDefault.outside) @@ -796,6 +808,9 @@ cdef class Doc: self.c[i].ent_iob = 1 self.c[i].ent_type = span.label self.c[i].ent_kb_id = span.kb_id + # for backwards compatibility in v3, only set ent_id from + # span.id if it's set, otherwise don't override + self.c[i].ent_id = span.id if span.id else self.c[i].ent_id for span in blocked: for i in range(span.start, span.end): self.c[i].ent_iob = 3 @@ -1175,6 +1190,7 @@ cdef class Doc: span.end_char + char_offset, span.label, span.kb_id, + span.id, span.text, # included as a check )) char_offset += len(doc.text) @@ -1210,8 +1226,9 @@ cdef class Doc: span_tuple[1], label=span_tuple[2], kb_id=span_tuple[3], + span_id=span_tuple[4], ) - text = span_tuple[4] + text = span_tuple[5] if span is not None and span.text == text: concat_doc.spans[key].append(span) else: @@ -1462,6 +1479,138 @@ cdef class Doc: remove_label_if_necessary(attributes[i]) retokenizer.merge(span, attributes[i]) + def from_json(self, doc_json, *, validate=False): + """Convert a JSON document generated by Doc.to_json() to a Doc. + + doc_json (Dict): JSON representation of doc object to load. + validate (bool): Whether to validate `doc_json` against the expected schema. + Defaults to False. + RETURNS (Doc): A doc instance corresponding to the specified JSON representation. + """ + + if validate: + schema_validation_message = schemas.validate(schemas.DocJSONSchema, doc_json) + if schema_validation_message: + raise ValueError(Errors.E1038.format(message=schema_validation_message)) + + ### Token-level properties ### + + words = [] + token_attrs_ids = (POS, HEAD, DEP, LEMMA, TAG, MORPH) + # Map annotation type IDs to their string equivalents. + token_attrs = {t: self.vocab.strings[t].lower() for t in token_attrs_ids} + token_annotations = {} + + # Gather token-level properties. + for token_json in doc_json["tokens"]: + words.append(doc_json["text"][token_json["start"]:token_json["end"]]) + for attr, attr_json in token_attrs.items(): + if attr_json in token_json: + if token_json["id"] == 0 and attr not in token_annotations: + token_annotations[attr] = [] + elif attr not in token_annotations: + raise ValueError(Errors.E1040.format(partial_attrs=attr)) + token_annotations[attr].append(token_json[attr_json]) + + # Initialize doc instance. + start = 0 + cdef const LexemeC* lex + cdef bint has_space + reconstructed_words, spaces = get_words_and_spaces(words, doc_json["text"]) + assert words == reconstructed_words + + for word, has_space in zip(words, spaces): + lex = self.vocab.get(self.mem, word) + self.push_back(lex, has_space) + + # Set remaining token-level attributes via Doc.from_array(). + if HEAD in token_annotations: + token_annotations[HEAD] = [ + head - i for i, head in enumerate(token_annotations[HEAD]) + ] + + if DEP in token_annotations and HEAD not in token_annotations: + token_annotations[HEAD] = [0] * len(token_annotations[DEP]) + if HEAD in token_annotations and DEP not in token_annotations: + raise ValueError(Errors.E1017) + if POS in token_annotations: + for pp in set(token_annotations[POS]): + if pp not in parts_of_speech.IDS: + raise ValueError(Errors.E1021.format(pp=pp)) + + # Collect token attributes, assert all tokens have exactly the same set of attributes. + attrs = [] + partial_attrs: Set[str] = set() + for attr in token_attrs.keys(): + if attr in token_annotations: + if len(token_annotations[attr]) != len(words): + partial_attrs.add(token_attrs[attr]) + attrs.append(attr) + if len(partial_attrs): + raise ValueError(Errors.E1040.format(partial_attrs=partial_attrs)) + + # If there are any other annotations, set them. + if attrs: + array = self.to_array(attrs) + if array.ndim == 1: + array = numpy.reshape(array, (array.size, 1)) + j = 0 + + for j, (attr, annot) in enumerate(token_annotations.items()): + if attr is HEAD: + for i in range(len(words)): + array[i, j] = annot[i] + elif attr is MORPH: + for i in range(len(words)): + array[i, j] = self.vocab.morphology.add(annot[i]) + else: + for i in range(len(words)): + array[i, j] = self.vocab.strings.add(annot[i]) + self.from_array(attrs, array) + + ### Span/document properties ### + + # Complement other document-level properties (cats, spans, ents). + self.cats = doc_json.get("cats", {}) + + # Set sentence boundaries, if dependency parser not available but sentences are specified in JSON. + if not self.has_annotation("DEP"): + for sent in doc_json.get("sents", {}): + char_span = self.char_span(sent["start"], sent["end"]) + if char_span is None: + raise ValueError(Errors.E1039.format(obj="sentence", start=sent["start"], end=sent["end"])) + char_span[0].is_sent_start = True + for token in char_span[1:]: + token.is_sent_start = False + + + for span_group in doc_json.get("spans", {}): + spans = [] + for span in doc_json["spans"][span_group]: + char_span = self.char_span(span["start"], span["end"], span["label"], span["kb_id"]) + if char_span is None: + raise ValueError(Errors.E1039.format(obj="span", start=span["start"], end=span["end"])) + spans.append(char_span) + self.spans[span_group] = spans + + if "ents" in doc_json: + ents = [] + for ent in doc_json["ents"]: + char_span = self.char_span(ent["start"], ent["end"], ent["label"]) + if char_span is None: + raise ValueError(Errors.E1039.format(obj="entity"), start=ent["start"], end=ent["end"]) + ents.append(char_span) + self.ents = ents + + # Add custom attributes. Note that only Doc extensions are currently considered, Token and Span extensions are + # not yet supported. + for attr in doc_json.get("_", {}): + if not Doc.has_extension(attr): + Doc.set_extension(attr) + self._.set(attr, doc_json["_"][attr]) + + return self + def to_json(self, underscore=None): """Convert a Doc to JSON. @@ -1472,12 +1621,10 @@ cdef class Doc: """ data = {"text": self.text} if self.has_annotation("ENT_IOB"): - data["ents"] = [{"start": ent.start_char, "end": ent.end_char, - "label": ent.label_} for ent in self.ents] + data["ents"] = [{"start": ent.start_char, "end": ent.end_char, "label": ent.label_} for ent in self.ents] if self.has_annotation("SENT_START"): sents = list(self.sents) - data["sents"] = [{"start": sent.start_char, "end": sent.end_char} - for sent in sents] + data["sents"] = [{"start": sent.start_char, "end": sent.end_char} for sent in sents] if self.cats: data["cats"] = self.cats data["tokens"] = [] @@ -1503,7 +1650,9 @@ cdef class Doc: for span_group in self.spans: data["spans"][span_group] = [] for span in self.spans[span_group]: - span_data = {"start": span.start_char, "end": span.end_char, "label": span.label_, "kb_id": span.kb_id_} + span_data = { + "start": span.start_char, "end": span.end_char, "label": span.label_, "kb_id": span.kb_id_ + } data["spans"][span_group].append(span_data) if underscore: @@ -1732,18 +1881,17 @@ cdef int [:,:] _get_lca_matrix(Doc doc, int start, int end): def pickle_doc(doc): bytes_data = doc.to_bytes(exclude=["vocab", "user_data", "user_hooks"]) hooks_and_data = (doc.user_data, doc.user_hooks, doc.user_span_hooks, - doc.user_token_hooks, doc._context) + doc.user_token_hooks) return (unpickle_doc, (doc.vocab, srsly.pickle_dumps(hooks_and_data), bytes_data)) def unpickle_doc(vocab, hooks_and_data, bytes_data): - user_data, doc_hooks, span_hooks, token_hooks, _context = srsly.pickle_loads(hooks_and_data) + user_data, doc_hooks, span_hooks, token_hooks = srsly.pickle_loads(hooks_and_data) doc = Doc(vocab, user_data=user_data).from_bytes(bytes_data, exclude=["user_data"]) doc.user_hooks.update(doc_hooks) doc.user_span_hooks.update(span_hooks) doc.user_token_hooks.update(token_hooks) - doc._context = _context return doc @@ -1767,16 +1915,18 @@ def fix_attributes(doc, attributes): def get_entity_info(ent_info): + ent_kb_id = 0 + ent_id = 0 if isinstance(ent_info, Span): ent_type = ent_info.label ent_kb_id = ent_info.kb_id start = ent_info.start end = ent_info.end + ent_id = ent_info.id elif len(ent_info) == 3: ent_type, start, end = ent_info - ent_kb_id = 0 elif len(ent_info) == 4: ent_type, ent_kb_id, start, end = ent_info else: ent_id, ent_kb_id, ent_type, start, end = ent_info - return ent_type, ent_kb_id, start, end + return ent_type, ent_kb_id, start, end, ent_id diff --git a/spacy/tokens/span.pyi b/spacy/tokens/span.pyi index 697051e81..617e3d19d 100644 --- a/spacy/tokens/span.pyi +++ b/spacy/tokens/span.pyi @@ -48,7 +48,8 @@ class Span: label: Union[str, int] = ..., vector: Optional[Floats1d] = ..., vector_norm: Optional[float] = ..., - kb_id: Optional[int] = ..., + kb_id: Union[str, int] = ..., + span_id: Union[str, int] = ..., ) -> None: ... def __richcmp__(self, other: Span, op: int) -> bool: ... def __hash__(self) -> int: ... @@ -119,6 +120,10 @@ class Span: ent_id: int ent_id_: str @property + def id(self) -> int: ... + @property + def id_(self) -> str: ... + @property def orth_(self) -> str: ... @property def lemma_(self) -> str: ... diff --git a/spacy/tokens/span.pyx b/spacy/tokens/span.pyx index 305d7caf4..c3495f497 100644 --- a/spacy/tokens/span.pyx +++ b/spacy/tokens/span.pyx @@ -80,17 +80,20 @@ cdef class Span: return Underscore.span_extensions.pop(name) def __cinit__(self, Doc doc, int start, int end, label=0, vector=None, - vector_norm=None, kb_id=0): + vector_norm=None, kb_id=0, span_id=0): """Create a `Span` object from the slice `doc[start : end]`. doc (Doc): The parent document. start (int): The index of the first token of the span. end (int): The index of the first token after the span. - label (int or str): A label to attach to the Span, e.g. for named entities. + label (Union[int, str]): A label to attach to the Span, e.g. for named + entities. vector (ndarray[ndim=1, dtype='float32']): A meaning representation of the span. vector_norm (float): The L2 norm of the span's vector representation. - kb_id (uint64): An identifier from a Knowledge Base to capture the meaning of a named entity. + kb_id (Union[int, str]): An identifier from a Knowledge Base to capture + the meaning of a named entity. + span_id (Union[int, str]): An identifier to associate with the span. DOCS: https://spacy.io/api/span#init """ @@ -101,6 +104,8 @@ cdef class Span: label = doc.vocab.strings.add(label) if isinstance(kb_id, str): kb_id = doc.vocab.strings.add(kb_id) + if isinstance(span_id, str): + span_id = doc.vocab.strings.add(span_id) if label not in doc.vocab.strings: raise ValueError(Errors.E084.format(label=label)) @@ -112,6 +117,7 @@ cdef class Span: self.c = SpanC( label=label, kb_id=kb_id, + id=span_id, start=start, end=end, start_char=start_char, @@ -126,8 +132,8 @@ cdef class Span: return False else: return True - self_tuple = (self.c.start_char, self.c.end_char, self.c.label, self.c.kb_id, self.doc) - other_tuple = (other.c.start_char, other.c.end_char, other.c.label, other.c.kb_id, other.doc) + self_tuple = (self.c.start_char, self.c.end_char, self.c.label, self.c.kb_id, self.id, self.doc) + other_tuple = (other.c.start_char, other.c.end_char, other.c.label, other.c.kb_id, other.id, other.doc) # < if op == 0: return self_tuple < other_tuple @@ -148,7 +154,7 @@ cdef class Span: return self_tuple >= other_tuple def __hash__(self): - return hash((self.doc, self.c.start_char, self.c.end_char, self.c.label, self.c.kb_id)) + return hash((self.doc, self.c.start_char, self.c.end_char, self.c.label, self.c.kb_id, self.c.id)) def __len__(self): """Get the number of tokens in the span. @@ -348,7 +354,8 @@ cdef class Span: if self.vocab.vectors.n_keys == 0: warnings.warn(Warnings.W007.format(obj="Span")) if self.vector_norm == 0.0 or other.vector_norm == 0.0: - warnings.warn(Warnings.W008.format(obj="Span")) + if not self.has_vector or not other.has_vector: + warnings.warn(Warnings.W008.format(obj="Span")) return 0.0 vector = self.vector xp = get_array_module(vector) @@ -632,7 +639,7 @@ cdef class Span: else: return self.doc[root] - def char_span(self, int start_idx, int end_idx, label=0, kb_id=0, vector=None): + def char_span(self, int start_idx, int end_idx, label=0, kb_id=0, vector=None, id=0): """Create a `Span` object from the slice `span.text[start : end]`. start (int): The index of the first character of the span. @@ -774,6 +781,13 @@ cdef class Span: def __set__(self, attr_t kb_id): self.c.kb_id = kb_id + property id: + def __get__(self): + return self.c.id + + def __set__(self, attr_t id): + self.c.id = id + property ent_id: """RETURNS (uint64): The entity ID.""" def __get__(self): @@ -812,13 +826,21 @@ cdef class Span: self.label = self.doc.vocab.strings.add(label_) property kb_id_: - """RETURNS (str): The named entity's KB ID.""" + """RETURNS (str): The span's KB ID.""" def __get__(self): return self.doc.vocab.strings[self.kb_id] def __set__(self, str kb_id_): self.kb_id = self.doc.vocab.strings.add(kb_id_) + property id_: + """RETURNS (str): The span's ID.""" + def __get__(self): + return self.doc.vocab.strings[self.id] + + def __set__(self, str id_): + self.id = self.doc.vocab.strings.add(id_) + cdef int _count_words_to_root(const TokenC* token, int sent_length) except -1: # Don't allow spaces to be the root, if there are diff --git a/spacy/tokens/span_group.pyi b/spacy/tokens/span_group.pyi index 26efc3ba0..245eb4dbe 100644 --- a/spacy/tokens/span_group.pyi +++ b/spacy/tokens/span_group.pyi @@ -24,3 +24,4 @@ class SpanGroup: def __getitem__(self, i: int) -> Span: ... def to_bytes(self) -> bytes: ... def from_bytes(self, bytes_data: bytes) -> SpanGroup: ... + def copy(self) -> SpanGroup: ... diff --git a/spacy/tokens/token.pyx b/spacy/tokens/token.pyx index d14930348..7fff6b162 100644 --- a/spacy/tokens/token.pyx +++ b/spacy/tokens/token.pyx @@ -206,7 +206,8 @@ cdef class Token: if self.vocab.vectors.n_keys == 0: warnings.warn(Warnings.W007.format(obj="Token")) if self.vector_norm == 0 or other.vector_norm == 0: - warnings.warn(Warnings.W008.format(obj="Token")) + if not self.has_vector or not other.has_vector: + warnings.warn(Warnings.W008.format(obj="Token")) return 0.0 vector = self.vector xp = get_array_module(vector) diff --git a/spacy/training/__init__.py b/spacy/training/__init__.py index a4feb01f4..71d1fa775 100644 --- a/spacy/training/__init__.py +++ b/spacy/training/__init__.py @@ -5,6 +5,7 @@ from .augment import dont_augment, orth_variants_augmenter # noqa: F401 from .iob_utils import iob_to_biluo, biluo_to_iob # noqa: F401 from .iob_utils import offsets_to_biluo_tags, biluo_tags_to_offsets # noqa: F401 from .iob_utils import biluo_tags_to_spans, tags_to_entities # noqa: F401 +from .iob_utils import split_bilu_label, remove_bilu_prefix # noqa: F401 from .gold_io import docs_to_json, read_json_file # noqa: F401 from .batchers import minibatch_by_padded_size, minibatch_by_words # noqa: F401 from .loggers import console_logger # noqa: F401 diff --git a/spacy/training/alignment_array.pyx b/spacy/training/alignment_array.pyx index b58f08786..01e9d9bf8 100644 --- a/spacy/training/alignment_array.pyx +++ b/spacy/training/alignment_array.pyx @@ -1,33 +1,39 @@ from typing import List from ..errors import Errors import numpy +from libc.stdint cimport int32_t cdef class AlignmentArray: """AlignmentArray is similar to Thinc's Ragged with two simplfications: indexing returns numpy arrays and this type can only be used for CPU arrays. - However, these changes make AlginmentArray more efficient for indexing in a + However, these changes make AlignmentArray more efficient for indexing in a tight loop.""" __slots__ = [] def __init__(self, alignment: List[List[int]]): - self._lengths = None - self._starts_ends = numpy.zeros(len(alignment) + 1, dtype="i") - cdef int data_len = 0 cdef int outer_len cdef int idx + + self._starts_ends = numpy.zeros(len(alignment) + 1, dtype='int32') + cdef int32_t* starts_ends_ptr = self._starts_ends.data + for idx, outer in enumerate(alignment): outer_len = len(outer) - self._starts_ends[idx + 1] = self._starts_ends[idx] + outer_len + starts_ends_ptr[idx + 1] = starts_ends_ptr[idx] + outer_len data_len += outer_len - self._data = numpy.empty(data_len, dtype="i") + self._lengths = None + self._data = numpy.empty(data_len, dtype="int32") + idx = 0 + cdef int32_t* data_ptr = self._data.data + for outer in alignment: for inner in outer: - self._data[idx] = inner + data_ptr[idx] = inner idx += 1 def __getitem__(self, idx): diff --git a/spacy/training/augment.py b/spacy/training/augment.py index 59a39c7ee..55d780ba4 100644 --- a/spacy/training/augment.py +++ b/spacy/training/augment.py @@ -3,10 +3,10 @@ from typing import Optional import random import itertools from functools import partial -from pydantic import BaseModel, StrictStr from ..util import registry from .example import Example +from .iob_utils import split_bilu_label if TYPE_CHECKING: from ..language import Language # noqa: F401 @@ -278,10 +278,8 @@ def make_whitespace_variant( ent_prev = doc_dict["entities"][position - 1] ent_next = doc_dict["entities"][position] if "-" in ent_prev and "-" in ent_next: - ent_iob_prev = ent_prev.split("-")[0] - ent_type_prev = ent_prev.split("-", 1)[1] - ent_iob_next = ent_next.split("-")[0] - ent_type_next = ent_next.split("-", 1)[1] + ent_iob_prev, ent_type_prev = split_bilu_label(ent_prev) + ent_iob_next, ent_type_next = split_bilu_label(ent_next) if ( ent_iob_prev in ("B", "I") and ent_iob_next in ("I", "L") diff --git a/spacy/training/example.pyx b/spacy/training/example.pyx index ab92f78c6..d592e5a52 100644 --- a/spacy/training/example.pyx +++ b/spacy/training/example.pyx @@ -9,11 +9,11 @@ from ..tokens.span import Span from ..attrs import IDS from .alignment import Alignment from .iob_utils import biluo_to_iob, offsets_to_biluo_tags, doc_to_biluo_tags -from .iob_utils import biluo_tags_to_spans +from .iob_utils import biluo_tags_to_spans, remove_bilu_prefix from ..errors import Errors, Warnings from ..pipeline._parser_internals import nonproj from ..tokens.token cimport MISSING_DEP -from ..util import logger, to_ternary_int +from ..util import logger, to_ternary_int, all_equal cpdef Doc annotations_to_doc(vocab, tok_annot, doc_annot): @@ -151,54 +151,131 @@ cdef class Example: self._y_sig = y_sig return self._cached_alignment + + def _get_aligned_vectorized(self, align, gold_values): + # Fast path for Doc attributes/fields that are predominantly a single value, + # i.e., TAG, POS, MORPH. + x2y_single_toks = [] + x2y_single_toks_i = [] + + x2y_multiple_toks = [] + x2y_multiple_toks_i = [] + + # Gather indices of gold tokens aligned to the candidate tokens into two buckets. + # Bucket 1: All tokens that have a one-to-one alignment. + # Bucket 2: All tokens that have a one-to-many alignment. + for idx, token in enumerate(self.predicted): + aligned_gold_i = align[token.i] + aligned_gold_len = len(aligned_gold_i) + + if aligned_gold_len == 1: + x2y_single_toks.append(aligned_gold_i.item()) + x2y_single_toks_i.append(idx) + elif aligned_gold_len > 1: + x2y_multiple_toks.append(aligned_gold_i) + x2y_multiple_toks_i.append(idx) + + # Map elements of the first bucket directly to the output array. + output = numpy.full(len(self.predicted), None) + output[x2y_single_toks_i] = gold_values[x2y_single_toks].squeeze() + + # Collapse many-to-one alignments into one-to-one alignments if they + # share the same value. Map to None in all other cases. + for i in range(len(x2y_multiple_toks)): + aligned_gold_values = gold_values[x2y_multiple_toks[i]] + + # If all aligned tokens have the same value, use it. + if all_equal(aligned_gold_values): + x2y_multiple_toks[i] = aligned_gold_values[0].item() + else: + x2y_multiple_toks[i] = None + + output[x2y_multiple_toks_i] = x2y_multiple_toks + + return output.tolist() + + + def _get_aligned_non_vectorized(self, align, gold_values): + # Slower path for fields that return multiple values (resulting + # in ragged arrays that cannot be vectorized trivially). + output = [None] * len(self.predicted) + + for token in self.predicted: + aligned_gold_i = align[token.i] + values = gold_values[aligned_gold_i].ravel() + if len(values) == 1: + output[token.i] = values.item() + elif all_equal(values): + # If all aligned tokens have the same value, use it. + output[token.i] = values[0].item() + + return output + + def get_aligned(self, field, as_string=False): """Return an aligned array for a token attribute.""" align = self.alignment.x2y + gold_values = self.reference.to_array([field]) + + if len(gold_values.shape) == 1: + output = self._get_aligned_vectorized(align, gold_values) + else: + output = self._get_aligned_non_vectorized(align, gold_values) vocab = self.reference.vocab - gold_values = self.reference.to_array([field]) - output = [None] * len(self.predicted) - for token in self.predicted: - values = gold_values[align[token.i]] - values = values.ravel() - if len(values) == 0: - output[token.i] = None - elif len(values) == 1: - output[token.i] = values[0] - elif len(set(list(values))) == 1: - # If all aligned tokens have the same value, use it. - output[token.i] = values[0] - else: - output[token.i] = None if as_string and field not in ["ENT_IOB", "SENT_START"]: output = [vocab.strings[o] if o is not None else o for o in output] + return output def get_aligned_parse(self, projectivize=True): cand_to_gold = self.alignment.x2y gold_to_cand = self.alignment.y2x - aligned_heads = [None] * self.x.length - aligned_deps = [None] * self.x.length - has_deps = [token.has_dep() for token in self.y] - has_heads = [token.has_head() for token in self.y] heads = [token.head.i for token in self.y] deps = [token.dep_ for token in self.y] + if projectivize: proj_heads, proj_deps = nonproj.projectivize(heads, deps) + has_deps = [token.has_dep() for token in self.y] + has_heads = [token.has_head() for token in self.y] + # ensure that missing data remains missing heads = [h if has_heads[i] else heads[i] for i, h in enumerate(proj_heads)] deps = [d if has_deps[i] else deps[i] for i, d in enumerate(proj_deps)] - for cand_i in range(self.x.length): - if cand_to_gold.lengths[cand_i] == 1: - gold_i = cand_to_gold[cand_i][0] - if gold_to_cand.lengths[heads[gold_i]] == 1: - aligned_heads[cand_i] = int(gold_to_cand[heads[gold_i]][0]) - aligned_deps[cand_i] = deps[gold_i] - return aligned_heads, aligned_deps + + # Select all candidate tokens that are aligned to a single gold token. + c2g_single_toks = numpy.where(cand_to_gold.lengths == 1)[0] + + # Fetch all aligned gold token incides. + if c2g_single_toks.shape == cand_to_gold.lengths.shape: + # This the most likely case. + gold_i = cand_to_gold[:] + else: + gold_i = numpy.vectorize(lambda x: cand_to_gold[int(x)][0], otypes='i')(c2g_single_toks) + + # Fetch indices of all gold heads for the aligned gold tokens. + heads = numpy.asarray(heads, dtype='i') + gold_head_i = heads[gold_i] + + # Select all gold tokens that are heads of the previously selected + # gold tokens (and are aligned to a single candidate token). + g2c_len_heads = gold_to_cand.lengths[gold_head_i] + g2c_len_heads = numpy.where(g2c_len_heads == 1)[0] + g2c_i = numpy.vectorize(lambda x: gold_to_cand[int(x)][0], otypes='i')(gold_head_i[g2c_len_heads]).squeeze() + + # Update head/dep alignments with the above. + aligned_heads = numpy.full((self.x.length), None) + aligned_heads[c2g_single_toks[g2c_len_heads]] = g2c_i + + deps = numpy.asarray(deps) + aligned_deps = numpy.full((self.x.length), None) + aligned_deps[c2g_single_toks] = deps[gold_i] + + return aligned_heads.tolist(), aligned_deps.tolist() def get_aligned_sent_starts(self): """Get list of SENT_START attributes aligned to the predicted tokenization. - If the reference has not sentence starts, return a list of None values. + If the reference does not have sentence starts, return a list of None values. """ if self.y.has_annotation("SENT_START"): align = self.alignment.y2x @@ -519,7 +596,7 @@ def _parse_ner_tags(biluo_or_offsets, vocab, words, spaces): else: ent_iobs.append(iob_tag.split("-")[0]) if iob_tag.startswith("I") or iob_tag.startswith("B"): - ent_types.append(iob_tag.split("-", 1)[1]) + ent_types.append(remove_bilu_prefix(iob_tag)) else: ent_types.append("") return ent_iobs, ent_types diff --git a/spacy/training/iob_utils.py b/spacy/training/iob_utils.py index 64492c2bc..61f83a1c3 100644 --- a/spacy/training/iob_utils.py +++ b/spacy/training/iob_utils.py @@ -1,4 +1,4 @@ -from typing import List, Dict, Tuple, Iterable, Union, Iterator +from typing import List, Dict, Tuple, Iterable, Union, Iterator, cast import warnings from ..errors import Errors, Warnings @@ -218,6 +218,14 @@ def tags_to_entities(tags: Iterable[str]) -> List[Tuple[str, int, int]]: return entities +def split_bilu_label(label: str) -> Tuple[str, str]: + return cast(Tuple[str, str], label.split("-", 1)) + + +def remove_bilu_prefix(label: str) -> str: + return label.split("-", 1)[1] + + # Fallbacks to make backwards-compat easier offsets_from_biluo_tags = biluo_tags_to_offsets spans_from_biluo_tags = biluo_tags_to_spans diff --git a/spacy/util.py b/spacy/util.py index 66e257dd8..d170fc15b 100644 --- a/spacy/util.py +++ b/spacy/util.py @@ -1,6 +1,6 @@ -from typing import List, Mapping, NoReturn, Union, Dict, Any, Set +from typing import List, Mapping, NoReturn, Union, Dict, Any, Set, cast from typing import Optional, Iterable, Callable, Tuple, Type -from typing import Iterator, Type, Pattern, Generator, TYPE_CHECKING +from typing import Iterator, Pattern, Generator, TYPE_CHECKING from types import ModuleType import os import importlib @@ -12,7 +12,6 @@ from thinc.api import NumpyOps, get_current_ops, Adam, Config, Optimizer from thinc.api import ConfigValidationError, Model import functools import itertools -import numpy.random import numpy import srsly import catalogue @@ -294,7 +293,7 @@ def find_matching_language(lang: str) -> Optional[str]: # Find out which language modules we have possible_languages = [] - for modinfo in pkgutil.iter_modules(spacy.lang.__path__): # type: ignore + for modinfo in pkgutil.iter_modules(spacy.lang.__path__): # type: ignore[attr-defined] code = modinfo.name if code == "xx": # Temporarily make 'xx' into a valid language code @@ -391,7 +390,8 @@ def get_module_path(module: ModuleType) -> Path: """ if not hasattr(module, "__module__"): raise ValueError(Errors.E169.format(module=repr(module))) - return Path(sys.modules[module.__module__].__file__).parent + file_path = Path(cast(os.PathLike, sys.modules[module.__module__].__file__)) + return file_path.parent def load_model( @@ -399,6 +399,7 @@ def load_model( *, vocab: Union["Vocab", bool] = True, disable: Iterable[str] = SimpleFrozenList(), + enable: Iterable[str] = SimpleFrozenList(), exclude: Iterable[str] = SimpleFrozenList(), config: Union[Dict[str, Any], Config] = SimpleFrozenDict(), ) -> "Language": @@ -408,11 +409,19 @@ def load_model( vocab (Vocab / True): Optional vocab to pass in on initialization. If True, a new Vocab object will be created. disable (Iterable[str]): Names of pipeline components to disable. + enable (Iterable[str]): Names of pipeline components to enable. All others will be disabled. + exclude (Iterable[str]): Names of pipeline components to exclude. config (Dict[str, Any] / Config): Config overrides as nested dict or dict keyed by section values in dot notation. RETURNS (Language): The loaded nlp object. """ - kwargs = {"vocab": vocab, "disable": disable, "exclude": exclude, "config": config} + kwargs = { + "vocab": vocab, + "disable": disable, + "enable": enable, + "exclude": exclude, + "config": config, + } if isinstance(name, str): # name or string path if name.startswith("blank:"): # shortcut for blank model return get_lang_class(name.replace("blank:", ""))() @@ -432,6 +441,7 @@ def load_model_from_package( *, vocab: Union["Vocab", bool] = True, disable: Iterable[str] = SimpleFrozenList(), + enable: Iterable[str] = SimpleFrozenList(), exclude: Iterable[str] = SimpleFrozenList(), config: Union[Dict[str, Any], Config] = SimpleFrozenDict(), ) -> "Language": @@ -443,6 +453,8 @@ def load_model_from_package( disable (Iterable[str]): Names of pipeline components to disable. Disabled pipes will be loaded but they won't be run unless you explicitly enable them by calling nlp.enable_pipe. + enable (Iterable[str]): Names of pipeline components to enable. All other + pipes will be disabled (and can be enabled using `nlp.enable_pipe`). exclude (Iterable[str]): Names of pipeline components to exclude. Excluded components won't be loaded. config (Dict[str, Any] / Config): Config overrides as nested dict or dict @@ -450,7 +462,7 @@ def load_model_from_package( RETURNS (Language): The loaded nlp object. """ cls = importlib.import_module(name) - return cls.load(vocab=vocab, disable=disable, exclude=exclude, config=config) # type: ignore[attr-defined] + return cls.load(vocab=vocab, disable=disable, enable=enable, exclude=exclude, config=config) # type: ignore[attr-defined] def load_model_from_path( @@ -459,6 +471,7 @@ def load_model_from_path( meta: Optional[Dict[str, Any]] = None, vocab: Union["Vocab", bool] = True, disable: Iterable[str] = SimpleFrozenList(), + enable: Iterable[str] = SimpleFrozenList(), exclude: Iterable[str] = SimpleFrozenList(), config: Union[Dict[str, Any], Config] = SimpleFrozenDict(), ) -> "Language": @@ -472,6 +485,8 @@ def load_model_from_path( disable (Iterable[str]): Names of pipeline components to disable. Disabled pipes will be loaded but they won't be run unless you explicitly enable them by calling nlp.enable_pipe. + enable (Iterable[str]): Names of pipeline components to enable. All other + pipes will be disabled (and can be enabled using `nlp.enable_pipe`). exclude (Iterable[str]): Names of pipeline components to exclude. Excluded components won't be loaded. config (Dict[str, Any] / Config): Config overrides as nested dict or dict @@ -486,7 +501,12 @@ def load_model_from_path( overrides = dict_to_dot(config) config = load_config(config_path, overrides=overrides) nlp = load_model_from_config( - config, vocab=vocab, disable=disable, exclude=exclude, meta=meta + config, + vocab=vocab, + disable=disable, + enable=enable, + exclude=exclude, + meta=meta, ) return nlp.from_disk(model_path, exclude=exclude, overrides=overrides) @@ -497,6 +517,7 @@ def load_model_from_config( meta: Dict[str, Any] = SimpleFrozenDict(), vocab: Union["Vocab", bool] = True, disable: Iterable[str] = SimpleFrozenList(), + enable: Iterable[str] = SimpleFrozenList(), exclude: Iterable[str] = SimpleFrozenList(), auto_fill: bool = False, validate: bool = True, @@ -511,6 +532,8 @@ def load_model_from_config( disable (Iterable[str]): Names of pipeline components to disable. Disabled pipes will be loaded but they won't be run unless you explicitly enable them by calling nlp.enable_pipe. + enable (Iterable[str]): Names of pipeline components to enable. All other + pipes will be disabled (and can be enabled using `nlp.enable_pipe`). exclude (Iterable[str]): Names of pipeline components to exclude. Excluded components won't be loaded. auto_fill (bool): Whether to auto-fill config with missing defaults. @@ -529,6 +552,7 @@ def load_model_from_config( config, vocab=vocab, disable=disable, + enable=enable, exclude=exclude, auto_fill=auto_fill, validate=validate, @@ -593,6 +617,7 @@ def load_model_from_init_py( *, vocab: Union["Vocab", bool] = True, disable: Iterable[str] = SimpleFrozenList(), + enable: Iterable[str] = SimpleFrozenList(), exclude: Iterable[str] = SimpleFrozenList(), config: Union[Dict[str, Any], Config] = SimpleFrozenDict(), ) -> "Language": @@ -604,6 +629,8 @@ def load_model_from_init_py( disable (Iterable[str]): Names of pipeline components to disable. Disabled pipes will be loaded but they won't be run unless you explicitly enable them by calling nlp.enable_pipe. + enable (Iterable[str]): Names of pipeline components to enable. All other + pipes will be disabled (and can be enabled using `nlp.enable_pipe`). exclude (Iterable[str]): Names of pipeline components to exclude. Excluded components won't be loaded. config (Dict[str, Any] / Config): Config overrides as nested dict or dict @@ -621,6 +648,7 @@ def load_model_from_init_py( vocab=vocab, meta=meta, disable=disable, + enable=enable, exclude=exclude, config=config, ) @@ -767,6 +795,15 @@ def get_model_lower_version(constraint: str) -> Optional[str]: return None +def is_prerelease_version(version: str) -> bool: + """Check whether a version is a prerelease version. + + version (str): The version, e.g. "3.0.0.dev1". + RETURNS (bool): Whether the version is a prerelease version. + """ + return Version(version).is_prerelease + + def get_base_version(version: str) -> str: """Generate the base version without any prerelease identifiers. @@ -878,7 +915,7 @@ def get_package_path(name: str) -> Path: # Here we're importing the module just to find it. This is worryingly # indirect, but it's otherwise very difficult to find the package. pkg = importlib.import_module(name) - return Path(pkg.__file__).parent + return Path(cast(Union[str, os.PathLike], pkg.__file__)).parent def replace_model_node(model: Model, target: Model, replacement: Model) -> None: @@ -1241,6 +1278,15 @@ def filter_spans(spans: Iterable["Span"]) -> List["Span"]: return result +def filter_chain_spans(*spans: Iterable["Span"]) -> List["Span"]: + return filter_spans(itertools.chain(*spans)) + + +@registry.misc("spacy.first_longest_spans_filter.v1") +def make_first_longest_spans_filter(): + return filter_chain_spans + + def to_bytes(getters: Dict[str, Callable[[], bytes]], exclude: Iterable[str]) -> bytes: return srsly.msgpack_dumps(to_dict(getters, exclude)) @@ -1675,7 +1721,14 @@ def packages_distributions() -> Dict[str, List[str]]: it's not available in the builtin importlib.metadata. """ pkg_to_dist = defaultdict(list) - for dist in importlib_metadata.distributions(): # type: ignore[attr-defined] + for dist in importlib_metadata.distributions(): for pkg in (dist.read_text("top_level.txt") or "").split(): pkg_to_dist[pkg].append(dist.metadata["Name"]) return dict(pkg_to_dist) + + +def all_equal(iterable): + """Return True if all the elements are equal to each other + (or if the input is an empty sequence), False otherwise.""" + g = itertools.groupby(iterable) + return next(g, True) and not next(g, False) diff --git a/spacy/vectors.pyx b/spacy/vectors.pyx index bcba9d03f..8300220c1 100644 --- a/spacy/vectors.pyx +++ b/spacy/vectors.pyx @@ -336,10 +336,10 @@ cdef class Vectors: xp = get_array_module(self.data) if key is not None: key = get_string_id(key) - return self.key2row.get(key, -1) + return self.key2row.get(int(key), -1) elif keys is not None: keys = [get_string_id(key) for key in keys] - rows = [self.key2row.get(key, -1.) for key in keys] + rows = [self.key2row.get(int(key), -1) for key in keys] return xp.asarray(rows, dtype="i") else: row2key = {row: key for key, row in self.key2row.items()} diff --git a/website/docs/api/architectures.md b/website/docs/api/architectures.md index 2bddcb28c..2537faff6 100644 --- a/website/docs/api/architectures.md +++ b/website/docs/api/architectures.md @@ -587,7 +587,7 @@ consists of either two or three subnetworks: run once for each batch. - **lower**: Construct a feature-specific vector for each `(token, feature)` pair. This is also run once for each batch. Constructing the state - representation is then simply a matter of summing the component features and + representation is then a matter of summing the component features and applying the non-linearity. - **upper** (optional): A feed-forward network that predicts scores from the state representation. If not present, the output from the lower model is used @@ -628,7 +628,7 @@ same signature, but the `use_upper` argument was `True` by default. > ``` Build a tagger model, using a provided token-to-vector component. The tagger -model simply adds a linear layer with softmax activation to predict scores given +model adds a linear layer with softmax activation to predict scores given the token vectors. | Name | Description | @@ -920,5 +920,5 @@ A function that reads an existing `KnowledgeBase` from file. A function that takes as input a [`KnowledgeBase`](/api/kb) and a [`Span`](/api/span) object denoting a named entity, and returns a list of plausible [`Candidate`](/api/kb/#candidate) objects. The default -`CandidateGenerator` simply uses the text of a mention to find its potential +`CandidateGenerator` uses the text of a mention to find its potential aliases in the `KnowledgeBase`. Note that this function is case-dependent. diff --git a/website/docs/api/attributes.md b/website/docs/api/attributes.md new file mode 100644 index 000000000..adacd3898 --- /dev/null +++ b/website/docs/api/attributes.md @@ -0,0 +1,78 @@ +--- +title: Attributes +teaser: Token attributes +source: spacy/attrs.pyx +--- + +[Token](/api/token) attributes are specified using internal IDs in many places +including: + +- [`Matcher` patterns](/api/matcher#patterns), +- [`Doc.to_array`](/api/doc#to_array) and + [`Doc.from_array`](/api/doc#from_array) +- [`Doc.has_annotation`](/api/doc#has_annotation) +- [`MultiHashEmbed`](/api/architectures#MultiHashEmbed) Tok2Vec architecture + `attrs` + +> ```python +> import spacy +> from spacy.attrs import DEP +> +> nlp = spacy.blank("en") +> doc = nlp("There are many attributes.") +> +> # DEP always has the same internal value +> assert DEP == 76 +> +> # "DEP" is automatically converted to DEP +> assert DEP == nlp.vocab.strings["DEP"] +> assert doc.has_annotation(DEP) == doc.has_annotation("DEP") +> +> # look up IDs in spacy.attrs.IDS +> from spacy.attrs import IDS +> assert IDS["DEP"] == DEP +> ``` + +All methods automatically convert between the string version of an ID (`"DEP"`) +and the internal integer symbols (`DEP`). The internal IDs can be imported from +`spacy.attrs` or retrieved from the [`StringStore`](/api/stringstore). A map +from string attribute names to internal attribute IDs is stored in +`spacy.attrs.IDS`. + +The corresponding [`Token` object attributes](/api/token#attributes) can be +accessed using the same names in lowercase, e.g. `token.orth` or `token.length`. +For attributes that represent string values, the internal integer ID is +accessed as `Token.attr`, e.g. `token.dep`, while the string value can be +retrieved by appending `_` as in `token.dep_`. + + +| Attribute | Description | +| ------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `DEP` | The token's dependency label. ~~str~~ | +| `ENT_ID` | The token's entity ID (`ent_id`). ~~str~~ | +| `ENT_IOB` | The IOB part of the token's entity tag. Uses custom integer vaues rather than the string store: unset is `0`, `I` is `1`, `O` is `2`, and `B` is `3`. ~~str~~ | +| `ENT_KB_ID` | The token's entity knowledge base ID. ~~str~~ | +| `ENT_TYPE` | The token's entity label. ~~str~~ | +| `IS_ALPHA` | Token text consists of alphabetic characters. ~~bool~~ | +| `IS_ASCII` | Token text consists of ASCII characters. ~~bool~~ | +| `IS_DIGIT` | Token text consists of digits. ~~bool~~ | +| `IS_LOWER` | Token text is in lowercase. ~~bool~~ | +| `IS_PUNCT` | Token is punctuation. ~~bool~~ | +| `IS_SPACE` | Token is whitespace. ~~bool~~ | +| `IS_STOP` | Token is a stop word. ~~bool~~ | +| `IS_TITLE` | Token text is in titlecase. ~~bool~~ | +| `IS_UPPER` | Token text is in uppercase. ~~bool~~ | +| `LEMMA` | The token's lemma. ~~str~~ | +| `LENGTH` | The length of the token text. ~~int~~ | +| `LIKE_EMAIL` | Token text resembles an email address. ~~bool~~ | +| `LIKE_NUM` | Token text resembles a number. ~~bool~~ | +| `LIKE_URL` | Token text resembles a URL. ~~bool~~ | +| `LOWER` | The lowercase form of the token text. ~~str~~ | +| `MORPH` | The token's morphological analysis. ~~MorphAnalysis~~ | +| `NORM` | The normalized form of the token text. ~~str~~ | +| `ORTH` | The exact verbatim text of a token. ~~str~~ | +| `POS` | The token's universal part of speech (UPOS). ~~str~~ | +| `SENT_START` | Token is start of sentence. ~~bool~~ | +| `SHAPE` | The token's shape. ~~str~~ | +| `SPACY` | Token has a trailing space. ~~bool~~ | +| `TAG` | The token's fine-grained part of speech. ~~str~~ | diff --git a/website/docs/api/cli.md b/website/docs/api/cli.md index e801ff0a6..cbd1f794a 100644 --- a/website/docs/api/cli.md +++ b/website/docs/api/cli.md @@ -466,6 +466,18 @@ takes the same arguments as `train` and reads settings off the + + +If your pipeline contains a `spancat` component, then this command will also +report span characteristics such as the average span length and the span (or +span boundary) distinctiveness. The distinctiveness measure shows how different +the tokens are with respect to the rest of the corpus using the KL-divergence of +the token distributions. To learn more, you can check out Papay et al.'s work on +[*Dissecting Span Identification Tasks with Performance Prediction* (EMNLP +2020)](https://aclanthology.org/2020.emnlp-main.396/). + + + ```cli $ python -m spacy debug data [config_path] [--code] [--ignore-warnings] [--verbose] [--no-format] [overrides] ``` @@ -1323,7 +1335,7 @@ $ python -m spacy project run [subcommand] [project_dir] [--force] [--dry] | `subcommand` | Name of the command or workflow to run. ~~str (positional)~~ | | `project_dir` | Path to project directory. Defaults to current working directory. ~~Path (positional)~~ | | `--force`, `-F` | Force re-running steps, even if nothing changed. ~~bool (flag)~~ | -| `--dry`, `-D` |  Perform a dry run and don't execute scripts. ~~bool (flag)~~ | +| `--dry`, `-D` | Perform a dry run and don't execute scripts. ~~bool (flag)~~ | | `--help`, `-h` | Show help message and available arguments. ~~bool (flag)~~ | | **EXECUTES** | The command defined in the `project.yml`. | @@ -1441,12 +1453,12 @@ For more examples, see the templates in our -| Name | Description | -| -------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `project_dir` | Path to project directory. Defaults to current working directory. ~~Path (positional)~~ | -| `--output`, `-o` | Path to output file or `-` for stdout (default). If a file is specified and it already exists and contains auto-generated docs, only the auto-generated docs section is replaced. ~~Path (positional)~~ | -|  `--no-emoji`, `-NE` | Don't use emoji in the titles. ~~bool (flag)~~ | -| **CREATES** | The Markdown-formatted project documentation. | +| Name | Description | +| ------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `project_dir` | Path to project directory. Defaults to current working directory. ~~Path (positional)~~ | +| `--output`, `-o` | Path to output file or `-` for stdout (default). If a file is specified and it already exists and contains auto-generated docs, only the auto-generated docs section is replaced. ~~Path (positional)~~ | +| `--no-emoji`, `-NE` | Don't use emoji in the titles. ~~bool (flag)~~ | +| **CREATES** | The Markdown-formatted project documentation. | ### project dvc {#project-dvc tag="command"} @@ -1485,7 +1497,7 @@ $ python -m spacy project dvc [project_dir] [workflow] [--force] [--verbose] | `project_dir` | Path to project directory. Defaults to current working directory. ~~Path (positional)~~ | | `workflow` | Name of workflow defined in `project.yml`. Defaults to first workflow if not set. ~~Optional[str] \(option)~~ | | `--force`, `-F` | Force-updating config file. ~~bool (flag)~~ | -| `--verbose`, `-V` |  Print more output generated by DVC. ~~bool (flag)~~ | +| `--verbose`, `-V` | Print more output generated by DVC. ~~bool (flag)~~ | | `--help`, `-h` | Show help message and available arguments. ~~bool (flag)~~ | | **CREATES** | A `dvc.yaml` file in the project directory, based on the steps defined in the given workflow. | @@ -1576,5 +1588,5 @@ $ python -m spacy huggingface-hub push [whl_path] [--org] [--msg] [--local-repo] | `--org`, `-o` | Optional name of organization to which the pipeline should be uploaded. ~~str (option)~~ | | `--msg`, `-m` | Commit message to use for update. Defaults to `"Update spaCy pipeline"`. ~~str (option)~~ | | `--local-repo`, `-l` | Local path to the model repository (will be created if it doesn't exist). Defaults to `hub` in the current working directory. ~~Path (option)~~ | -| `--verbose`, `-V` | Output additional info for debugging, e.g. the full generated hub metadata. ~~bool (flag)~~  | +| `--verbose`, `-V` | Output additional info for debugging, e.g. the full generated hub metadata. ~~bool (flag)~~ | | **UPLOADS** | The pipeline to the hub. | diff --git a/website/docs/api/corpus.md b/website/docs/api/corpus.md index 35afc8fea..88c4befd7 100644 --- a/website/docs/api/corpus.md +++ b/website/docs/api/corpus.md @@ -37,13 +37,13 @@ streaming. > augmenter = null > ``` -| Name | Description | -| --------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `path` | The directory or filename to read from. Expects data in spaCy's binary [`.spacy` format](/api/data-formats#binary-training). ~~Path~~ | -|  `gold_preproc` | Whether to set up the Example object with gold-standard sentences and tokens for the predictions. See [`Corpus`](/api/corpus#init) for details. ~~bool~~ | -| `max_length` | Maximum document length. Longer documents will be split into sentences, if sentence boundaries are available. Defaults to `0` for no limit. ~~int~~ | -| `limit` | Limit corpus to a subset of examples, e.g. for debugging. Defaults to `0` for no limit. ~~int~~ | -| `augmenter` | Apply some simply data augmentation, where we replace tokens with variations. This is especially useful for punctuation and case replacement, to help generalize beyond corpora that don't have smart-quotes, or only have smart quotes, etc. Defaults to `None`. ~~Optional[Callable]~~ | +| Name | Description | +| -------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `path` | The directory or filename to read from. Expects data in spaCy's binary [`.spacy` format](/api/data-formats#binary-training). ~~Path~~ | +| `gold_preproc` | Whether to set up the Example object with gold-standard sentences and tokens for the predictions. See [`Corpus`](/api/corpus#init) for details. ~~bool~~ | +| `max_length` | Maximum document length. Longer documents will be split into sentences, if sentence boundaries are available. Defaults to `0` for no limit. ~~int~~ | +| `limit` | Limit corpus to a subset of examples, e.g. for debugging. Defaults to `0` for no limit. ~~int~~ | +| `augmenter` | Apply some simply data augmentation, where we replace tokens with variations. This is especially useful for punctuation and case replacement, to help generalize beyond corpora that don't have smart-quotes, or only have smart quotes, etc. Defaults to `None`. ~~Optional[Callable]~~ | ```python %%GITHUB_SPACY/spacy/training/corpus.py @@ -71,15 +71,15 @@ train/test skew. > corpus = Corpus("./data", limit=10) > ``` -| Name | Description | -| --------------- | --------------------------------------------------------------------------------------------------------------------------------------------------- | -| `path` | The directory or filename to read from. ~~Union[str, Path]~~ | -| _keyword-only_ | | -|  `gold_preproc` | Whether to set up the Example object with gold-standard sentences and tokens for the predictions. Defaults to `False`. ~~bool~~ | -| `max_length` | Maximum document length. Longer documents will be split into sentences, if sentence boundaries are available. Defaults to `0` for no limit. ~~int~~ | -| `limit` | Limit corpus to a subset of examples, e.g. for debugging. Defaults to `0` for no limit. ~~int~~ | -| `augmenter` | Optional data augmentation callback. ~~Callable[[Language, Example], Iterable[Example]]~~ | -| `shuffle` | Whether to shuffle the examples. Defaults to `False`. ~~bool~~ | +| Name | Description | +| -------------- | --------------------------------------------------------------------------------------------------------------------------------------------------- | +| `path` | The directory or filename to read from. ~~Union[str, Path]~~ | +| _keyword-only_ | | +| `gold_preproc` | Whether to set up the Example object with gold-standard sentences and tokens for the predictions. Defaults to `False`. ~~bool~~ | +| `max_length` | Maximum document length. Longer documents will be split into sentences, if sentence boundaries are available. Defaults to `0` for no limit. ~~int~~ | +| `limit` | Limit corpus to a subset of examples, e.g. for debugging. Defaults to `0` for no limit. ~~int~~ | +| `augmenter` | Optional data augmentation callback. ~~Callable[[Language, Example], Iterable[Example]]~~ | +| `shuffle` | Whether to shuffle the examples. Defaults to `False`. ~~bool~~ | ## Corpus.\_\_call\_\_ {#call tag="method"} diff --git a/website/docs/api/dependencymatcher.md b/website/docs/api/dependencymatcher.md index 356adcda7..cae4221bf 100644 --- a/website/docs/api/dependencymatcher.md +++ b/website/docs/api/dependencymatcher.md @@ -62,7 +62,7 @@ of relations, see the usage guide on -### Operators +### Operators {#operators} The following operators are supported by the `DependencyMatcher`, most of which come directly from @@ -82,6 +82,11 @@ come directly from | `A $- B` | `B` is a left immediate sibling of `A`, i.e. `A` and `B` have the same parent and `A.i == B.i + 1`. | | `A $++ B` | `B` is a right sibling of `A`, i.e. `A` and `B` have the same parent and `A.i < B.i`. | | `A $-- B` | `B` is a left sibling of `A`, i.e. `A` and `B` have the same parent and `A.i > B.i`. | +| `A >++ B` | `B` is a right child of `A`, i.e. `A` is a parent of `B` and `A.i < B.i` _(not in Semgrex)_. | +| `A >-- B` | `B` is a left child of `A`, i.e. `A` is a parent of `B` and `A.i > B.i` _(not in Semgrex)_. | +| `A <++ B` | `B` is a right parent of `A`, i.e. `A` is a child of `B` and `A.i < B.i` _(not in Semgrex)_. | +| `A <-- B` | `B` is a left parent of `A`, i.e. `A` is a child of `B` and `A.i > B.i` _(not in Semgrex)_. | + ## DependencyMatcher.\_\_init\_\_ {#init tag="method"} diff --git a/website/docs/api/dependencyparser.md b/website/docs/api/dependencyparser.md index 103e0826e..27e315592 100644 --- a/website/docs/api/dependencyparser.md +++ b/website/docs/api/dependencyparser.md @@ -158,10 +158,10 @@ applied to the `Doc` in order. Both [`__call__`](/api/dependencyparser#call) and ## DependencyParser.initialize {#initialize tag="method" new="3"} Initialize the component for training. `get_examples` should be a function that -returns an iterable of [`Example`](/api/example) objects. The data examples are -used to **initialize the model** of the component and can either be the full -training data or a representative sample. Initialization includes validating the -network, +returns an iterable of [`Example`](/api/example) objects. **At least one example +should be supplied.** The data examples are used to **initialize the model** of +the component and can either be the full training data or a representative +sample. Initialization includes validating the network, [inferring missing shapes](https://thinc.ai/docs/usage-models#validation) and setting up the label scheme based on the data. This method is typically called by [`Language.initialize`](/api/language#initialize) and lets you customize @@ -179,7 +179,7 @@ This method was previously called `begin_training`. > > ```python > parser = nlp.add_pipe("parser") -> parser.initialize(lambda: [], nlp=nlp) +> parser.initialize(lambda: examples, nlp=nlp) > ``` > > ```ini @@ -193,7 +193,7 @@ This method was previously called `begin_training`. | Name | Description | | -------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `get_examples` | Function that returns gold-standard annotations in the form of [`Example`](/api/example) objects. ~~Callable[[], Iterable[Example]]~~ | +| `get_examples` | Function that returns gold-standard annotations in the form of [`Example`](/api/example) objects. Must contain at least one `Example`. ~~Callable[[], Iterable[Example]]~~ | | _keyword-only_ | | | `nlp` | The current `nlp` object. Defaults to `None`. ~~Optional[Language]~~ | | `labels` | The label information to add to the component, as provided by the [`label_data`](#label_data) property after initialization. To generate a reusable JSON file from your data, you should run the [`init labels`](/api/cli#init-labels) command. If no labels are provided, the `get_examples` callback is used to extract the labels from the data, which may be a lot slower. ~~Optional[Dict[str, Dict[str, int]]]~~ | diff --git a/website/docs/api/doc.md b/website/docs/api/doc.md index 0008cde31..f97f4ad83 100644 --- a/website/docs/api/doc.md +++ b/website/docs/api/doc.md @@ -481,6 +481,45 @@ Deserialize, i.e. import the document contents from a binary string. | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | | **RETURNS** | The `Doc` object. ~~Doc~~ | +## Doc.to_json {#to_json tag="method"} + +Serializes a document to JSON. Note that this is format differs from the +deprecated [`JSON training format`](/api/data-formats#json-input). + +> #### Example +> +> ```python +> doc = nlp("All we have to decide is what to do with the time that is given us.") +> assert doc.to_json()["text"] == doc.text +> ``` + +| Name | Description | +| ------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `underscore` | Optional list of string names of custom `Doc` attributes. Attribute values need to be JSON-serializable. Values will be added to an `"_"` key in the data, e.g. `"_": {"foo": "bar"}`. ~~Optional[List[str]]~~ | +| **RETURNS** | The data in JSON format. ~~Dict[str, Any]~~ | + +## Doc.from_json {#from_json tag="method" new="3.3.1"} + +Deserializes a document from JSON, i.e. generates a document from the provided +JSON data as generated by [`Doc.to_json()`](/api/doc#to_json). + +> #### Example +> +> ```python +> from spacy.tokens import Doc +> doc = nlp("All we have to decide is what to do with the time that is given us.") +> doc_json = doc.to_json() +> deserialized_doc = Doc(nlp.vocab).from_json(doc_json) +> assert deserialized_doc.text == doc.text == doc_json["text"] +> ``` + +| Name | Description | +| -------------- | -------------------------------------------------------------------------------------------------------------------- | +| `doc_json` | The Doc data in JSON format from [`Doc.to_json`](#to_json). ~~Dict[str, Any]~~ | +| _keyword-only_ | | +| `validate` | Whether to validate the JSON input against the expected schema for detailed debugging. Defaults to `False`. ~~bool~~ | +| **RETURNS** | A `Doc` corresponding to the provided JSON. ~~Doc~~ | + ## Doc.retokenize {#retokenize tag="contextmanager" new="2.1"} Context manager to handle retokenization of the `Doc`. Modifications to the diff --git a/website/docs/api/edittreelemmatizer.md b/website/docs/api/edittreelemmatizer.md index 99a705f5e..63e4bf910 100644 --- a/website/docs/api/edittreelemmatizer.md +++ b/website/docs/api/edittreelemmatizer.md @@ -141,10 +141,10 @@ and [`pipe`](/api/edittreelemmatizer#pipe) delegate to the ## EditTreeLemmatizer.initialize {#initialize tag="method" new="3"} Initialize the component for training. `get_examples` should be a function that -returns an iterable of [`Example`](/api/example) objects. The data examples are -used to **initialize the model** of the component and can either be the full -training data or a representative sample. Initialization includes validating the -network, +returns an iterable of [`Example`](/api/example) objects. **At least one example +should be supplied.** The data examples are used to **initialize the model** of +the component and can either be the full training data or a representative +sample. Initialization includes validating the network, [inferring missing shapes](https://thinc.ai/docs/usage-models#validation) and setting up the label scheme based on the data. This method is typically called by [`Language.initialize`](/api/language#initialize) and lets you customize @@ -156,7 +156,7 @@ config. > > ```python > lemmatizer = nlp.add_pipe("trainable_lemmatizer", name="lemmatizer") -> lemmatizer.initialize(lambda: [], nlp=nlp) +> lemmatizer.initialize(lambda: examples, nlp=nlp) > ``` > > ```ini @@ -170,7 +170,7 @@ config. | Name | Description | | -------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `get_examples` | Function that returns gold-standard annotations in the form of [`Example`](/api/example) objects. ~~Callable[[], Iterable[Example]]~~ | +| `get_examples` | Function that returns gold-standard annotations in the form of [`Example`](/api/example) objects. Must contain at least one `Example`. ~~Callable[[], Iterable[Example]]~~ | | _keyword-only_ | | | `nlp` | The current `nlp` object. Defaults to `None`. ~~Optional[Language]~~ | | `labels` | The label information to add to the component, as provided by the [`label_data`](#label_data) property after initialization. To generate a reusable JSON file from your data, you should run the [`init labels`](/api/cli#init-labels) command. If no labels are provided, the `get_examples` callback is used to extract the labels from the data, which may be a lot slower. ~~Optional[Iterable[str]]~~ | diff --git a/website/docs/api/entitylinker.md b/website/docs/api/entitylinker.md index 8e0d6087a..43e08a39c 100644 --- a/website/docs/api/entitylinker.md +++ b/website/docs/api/entitylinker.md @@ -47,22 +47,24 @@ architectures and their arguments and hyperparameters. > "model": DEFAULT_NEL_MODEL, > "entity_vector_length": 64, > "get_candidates": {'@misc': 'spacy.CandidateGenerator.v1'}, +> "threshold": None, > } > nlp.add_pipe("entity_linker", config=config) > ``` -| Setting | Description | -| ---------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -| `labels_discard` | NER labels that will automatically get a "NIL" prediction. Defaults to `[]`. ~~Iterable[str]~~ | -| `n_sents` | The number of neighbouring sentences to take into account. Defaults to 0. ~~int~~ | -| `incl_prior` | Whether or not to include prior probabilities from the KB in the model. Defaults to `True`. ~~bool~~ | -| `incl_context` | Whether or not to include the local context in the model. Defaults to `True`. ~~bool~~ | -| `model` | The [`Model`](https://thinc.ai/docs/api-model) powering the pipeline component. Defaults to [EntityLinker](/api/architectures#EntityLinker). ~~Model~~ | -| `entity_vector_length` | Size of encoding vectors in the KB. Defaults to `64`. ~~int~~ | -| `use_gold_ents` | Whether to copy entities from the gold docs or not. Defaults to `True`. If `False`, entities must be set in the training data or by an annotating component in the pipeline. ~~int~~ | -| `get_candidates` | Function that generates plausible candidates for a given `Span` object. Defaults to [CandidateGenerator](/api/architectures#CandidateGenerator), a function looking up exact, case-dependent aliases in the KB. ~~Callable[[KnowledgeBase, Span], Iterable[Candidate]]~~ | -| `overwrite` 3.2 | Whether existing annotation is overwritten. Defaults to `True`. ~~bool~~ | -| `scorer` 3.2 | The scoring method. Defaults to [`Scorer.score_links`](/api/scorer#score_links). ~~Optional[Callable]~~ | +| Setting | Description | +| ---------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `labels_discard` | NER labels that will automatically get a "NIL" prediction. Defaults to `[]`. ~~Iterable[str]~~ | +| `n_sents` | The number of neighbouring sentences to take into account. Defaults to 0. ~~int~~ | +| `incl_prior` | Whether or not to include prior probabilities from the KB in the model. Defaults to `True`. ~~bool~~ | +| `incl_context` | Whether or not to include the local context in the model. Defaults to `True`. ~~bool~~ | +| `model` | The [`Model`](https://thinc.ai/docs/api-model) powering the pipeline component. Defaults to [EntityLinker](/api/architectures#EntityLinker). ~~Model~~ | +| `entity_vector_length` | Size of encoding vectors in the KB. Defaults to `64`. ~~int~~ | +| `use_gold_ents` | Whether to copy entities from the gold docs or not. Defaults to `True`. If `False`, entities must be set in the training data or by an annotating component in the pipeline. ~~int~~ | +| `get_candidates` | Function that generates plausible candidates for a given `Span` object. Defaults to [CandidateGenerator](/api/architectures#CandidateGenerator), a function looking up exact, case-dependent aliases in the KB. ~~Callable[[KnowledgeBase, Span], Iterable[Candidate]]~~ | +| `overwrite` 3.2 | Whether existing annotation is overwritten. Defaults to `True`. ~~bool~~ | +| `scorer` 3.2 | The scoring method. Defaults to [`Scorer.score_links`](/api/scorer#score_links). ~~Optional[Callable]~~ | +| `threshold` 3.4 | Confidence threshold for entity predictions. The default of `None` implies that all predictions are accepted, otherwise those with a score beneath the treshold are discarded. If there are no predictions with scores above the threshold, the linked entity is `NIL`. ~~Optional[float]~~ | ```python %%GITHUB_SPACY/spacy/pipeline/entity_linker.py @@ -95,20 +97,21 @@ custom knowledge base, you should either call [`set_kb`](/api/entitylinker#set_kb) or provide a `kb_loader` in the [`initialize`](/api/entitylinker#initialize) call. -| Name | Description | -| ---------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------- | -| `vocab` | The shared vocabulary. ~~Vocab~~ | -| `model` | The [`Model`](https://thinc.ai/docs/api-model) powering the pipeline component. ~~Model~~ | -| `name` | String name of the component instance. Used to add entries to the `losses` during training. ~~str~~ | -| _keyword-only_ | | -| `entity_vector_length` | Size of encoding vectors in the KB. ~~int~~ | -| `get_candidates` | Function that generates plausible candidates for a given `Span` object. ~~Callable[[KnowledgeBase, Span], Iterable[Candidate]]~~ | -| `labels_discard` | NER labels that will automatically get a `"NIL"` prediction. ~~Iterable[str]~~ | -| `n_sents` | The number of neighbouring sentences to take into account. ~~int~~ | -| `incl_prior` | Whether or not to include prior probabilities from the KB in the model. ~~bool~~ | -| `incl_context` | Whether or not to include the local context in the model. ~~bool~~ | -| `overwrite` 3.2 | Whether existing annotation is overwritten. Defaults to `True`. ~~bool~~ | -| `scorer` 3.2 | The scoring method. Defaults to [`Scorer.score_links`](/api/scorer#score_links). ~~Optional[Callable]~~ | +| Name | Description | +| ---------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `vocab` | The shared vocabulary. ~~Vocab~~ | +| `model` | The [`Model`](https://thinc.ai/docs/api-model) powering the pipeline component. ~~Model~~ | +| `name` | String name of the component instance. Used to add entries to the `losses` during training. ~~str~~ | +| _keyword-only_ | | +| `entity_vector_length` | Size of encoding vectors in the KB. ~~int~~ | +| `get_candidates` | Function that generates plausible candidates for a given `Span` object. ~~Callable[[KnowledgeBase, Span], Iterable[Candidate]]~~ | +| `labels_discard` | NER labels that will automatically get a `"NIL"` prediction. ~~Iterable[str]~~ | +| `n_sents` | The number of neighbouring sentences to take into account. ~~int~~ | +| `incl_prior` | Whether or not to include prior probabilities from the KB in the model. ~~bool~~ | +| `incl_context` | Whether or not to include the local context in the model. ~~bool~~ | +| `overwrite` 3.2 | Whether existing annotation is overwritten. Defaults to `True`. ~~bool~~ | +| `scorer` 3.2 | The scoring method. Defaults to [`Scorer.score_links`](/api/scorer#score_links). ~~Optional[Callable]~~ | +| `threshold` 3.4 | Confidence threshold for entity predictions. The default of `None` implies that all predictions are accepted, otherwise those with a score beneath the treshold are discarded. If there are no predictions with scores above the threshold, the linked entity is `NIL`. ~~Optional[float]~~ | ## EntityLinker.\_\_call\_\_ {#call tag="method"} @@ -182,10 +185,10 @@ with the current vocab. ## EntityLinker.initialize {#initialize tag="method" new="3"} Initialize the component for training. `get_examples` should be a function that -returns an iterable of [`Example`](/api/example) objects. The data examples are -used to **initialize the model** of the component and can either be the full -training data or a representative sample. Initialization includes validating the -network, +returns an iterable of [`Example`](/api/example) objects. **At least one example +should be supplied.** The data examples are used to **initialize the model** of +the component and can either be the full training data or a representative +sample. Initialization includes validating the network, [inferring missing shapes](https://thinc.ai/docs/usage-models#validation) and setting up the label scheme based on the data. This method is typically called by [`Language.initialize`](/api/language#initialize). @@ -205,15 +208,15 @@ This method was previously called `begin_training`. > > ```python > entity_linker = nlp.add_pipe("entity_linker") -> entity_linker.initialize(lambda: [], nlp=nlp, kb_loader=my_kb) +> entity_linker.initialize(lambda: examples, nlp=nlp, kb_loader=my_kb) > ``` -| Name | Description | -| -------------- | ------------------------------------------------------------------------------------------------------------------------------------- | -| `get_examples` | Function that returns gold-standard annotations in the form of [`Example`](/api/example) objects. ~~Callable[[], Iterable[Example]]~~ | -| _keyword-only_ | | -| `nlp` | The current `nlp` object. Defaults to `None`. ~~Optional[Language]~~ | -| `kb_loader` | Function that creates a [`KnowledgeBase`](/api/kb) from a `Vocab` instance. ~~Callable[[Vocab], KnowledgeBase]~~ | +| Name | Description | +| -------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `get_examples` | Function that returns gold-standard annotations in the form of [`Example`](/api/example) objects. Must contain at least one `Example`. ~~Callable[[], Iterable[Example]]~~ | +| _keyword-only_ | | +| `nlp` | The current `nlp` object. Defaults to `None`. ~~Optional[Language]~~ | +| `kb_loader` | Function that creates a [`KnowledgeBase`](/api/kb) from a `Vocab` instance. ~~Callable[[Vocab], KnowledgeBase]~~ | ## EntityLinker.predict {#predict tag="method"} diff --git a/website/docs/api/entityrecognizer.md b/website/docs/api/entityrecognizer.md index 7c153f064..a535e8316 100644 --- a/website/docs/api/entityrecognizer.md +++ b/website/docs/api/entityrecognizer.md @@ -154,10 +154,10 @@ applied to the `Doc` in order. Both [`__call__`](/api/entityrecognizer#call) and ## EntityRecognizer.initialize {#initialize tag="method" new="3"} Initialize the component for training. `get_examples` should be a function that -returns an iterable of [`Example`](/api/example) objects. The data examples are -used to **initialize the model** of the component and can either be the full -training data or a representative sample. Initialization includes validating the -network, +returns an iterable of [`Example`](/api/example) objects. **At least one example +should be supplied.** The data examples are used to **initialize the model** of +the component and can either be the full training data or a representative +sample. Initialization includes validating the network, [inferring missing shapes](https://thinc.ai/docs/usage-models#validation) and setting up the label scheme based on the data. This method is typically called by [`Language.initialize`](/api/language#initialize) and lets you customize @@ -175,7 +175,7 @@ This method was previously called `begin_training`. > > ```python > ner = nlp.add_pipe("ner") -> ner.initialize(lambda: [], nlp=nlp) +> ner.initialize(lambda: examples, nlp=nlp) > ``` > > ```ini @@ -189,7 +189,7 @@ This method was previously called `begin_training`. | Name | Description | | -------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `get_examples` | Function that returns gold-standard annotations in the form of [`Example`](/api/example) objects. ~~Callable[[], Iterable[Example]]~~ | +| `get_examples` | Function that returns gold-standard annotations in the form of [`Example`](/api/example) objects. Must contain at least one `Example`. ~~Callable[[], Iterable[Example]]~~ | | _keyword-only_ | | | `nlp` | The current `nlp` object. Defaults to `None`. ~~Optional[Language]~~ | | `labels` | The label information to add to the component, as provided by the [`label_data`](#label_data) property after initialization. To generate a reusable JSON file from your data, you should run the [`init labels`](/api/cli#init-labels) command. If no labels are provided, the `get_examples` callback is used to extract the labels from the data, which may be a lot slower. ~~Optional[Dict[str, Dict[str, int]]]~~ | diff --git a/website/docs/api/entityruler.md b/website/docs/api/entityruler.md index 1ef283870..c2ba33f01 100644 --- a/website/docs/api/entityruler.md +++ b/website/docs/api/entityruler.md @@ -290,7 +290,7 @@ Load the pipe from a bytestring. Modifies the object in place and returns it. > > ```python > ruler_bytes = ruler.to_bytes() -> ruler = nlp.add_pipe("enity_ruler") +> ruler = nlp.add_pipe("entity_ruler") > ruler.from_bytes(ruler_bytes) > ``` diff --git a/website/docs/api/language.md b/website/docs/api/language.md index 8d7686243..9a413efaf 100644 --- a/website/docs/api/language.md +++ b/website/docs/api/language.md @@ -1123,7 +1123,7 @@ instance and factory instance. | `factory` | The name of the registered component factory. ~~str~~ | | `default_config` | The default config, describing the default values of the factory arguments. ~~Dict[str, Any]~~ | | `assigns` | `Doc` or `Token` attributes assigned by this component, e.g. `["token.ent_id"]`. Used for [pipe analysis](/usage/processing-pipelines#analysis). ~~Iterable[str]~~ | -| `requires` | `Doc` or `Token` attributes required by this component, e.g. `["token.ent_id"]`. Used for [pipe analysis](/usage/processing-pipelines#analysis). ~~Iterable[str]~~  | -| `retokenizes` | Whether the component changes tokenization. Used for [pipe analysis](/usage/processing-pipelines#analysis). ~~bool~~  | +| `requires` | `Doc` or `Token` attributes required by this component, e.g. `["token.ent_id"]`. Used for [pipe analysis](/usage/processing-pipelines#analysis). ~~Iterable[str]~~ | +| `retokenizes` | Whether the component changes tokenization. Used for [pipe analysis](/usage/processing-pipelines#analysis). ~~bool~~ | | `default_score_weights` | The scores to report during training, and their default weight towards the final score used to select the best model. Weights should sum to `1.0` per component and will be combined and normalized for the whole pipeline. If a weight is set to `None`, the score will not be logged or weighted. ~~Dict[str, Optional[float]]~~ | | `scores` | All scores set by the components if it's trainable, e.g. `["ents_f", "ents_r", "ents_p"]`. Based on the `default_score_weights` and used for [pipe analysis](/usage/processing-pipelines#analysis). ~~Iterable[str]~~ | diff --git a/website/docs/api/legacy.md b/website/docs/api/legacy.md index e24c37d77..31d178b67 100644 --- a/website/docs/api/legacy.md +++ b/website/docs/api/legacy.md @@ -103,11 +103,22 @@ and residual connections. | `depth` | The number of convolutional layers. Recommended value is `4`. ~~int~~ | | **CREATES** | The model using the architecture. ~~Model[Floats2d, Floats2d]~~ | -### spacy.TransitionBasedParser.v1 {#TransitionBasedParser_v1} +### spacy.HashEmbedCNN.v1 {#HashEmbedCNN_v1} -Identical to -[`spacy.TransitionBasedParser.v2`](/api/architectures#TransitionBasedParser) -except the `use_upper` was set to `True` by default. +Identical to [`spacy.HashEmbedCNN.v2`](/api/architectures#HashEmbedCNN) except +using [`spacy.StaticVectors.v1`](#StaticVectors_v1) if vectors are included. + +### spacy.MultiHashEmbed.v1 {#MultiHashEmbed_v1} + +Identical to [`spacy.MultiHashEmbed.v2`](/api/architectures#MultiHashEmbed) +except with [`spacy.StaticVectors.v1`](#StaticVectors_v1) if vectors are +included. + +### spacy.CharacterEmbed.v1 {#CharacterEmbed_v1} + +Identical to [`spacy.CharacterEmbed.v2`](/api/architectures#CharacterEmbed) +except using [`spacy.StaticVectors.v1`](#StaticVectors_v1) if vectors are +included. ### spacy.TextCatEnsemble.v1 {#TextCatEnsemble_v1} @@ -147,41 +158,6 @@ network has an internal CNN Tok2Vec layer and uses attention. | `nO` | Output dimension, determined by the number of different labels. If not set, the [`TextCategorizer`](/api/textcategorizer) component will set it when `initialize` is called. ~~Optional[int]~~ | | **CREATES** | The model using the architecture. ~~Model[List[Doc], Floats2d]~~ | -### spacy.HashEmbedCNN.v1 {#HashEmbedCNN_v1} - -Identical to [`spacy.HashEmbedCNN.v2`](/api/architectures#HashEmbedCNN) except -using [`spacy.StaticVectors.v1`](#StaticVectors_v1) if vectors are included. - -### spacy.MultiHashEmbed.v1 {#MultiHashEmbed_v1} - -Identical to [`spacy.MultiHashEmbed.v2`](/api/architectures#MultiHashEmbed) -except with [`spacy.StaticVectors.v1`](#StaticVectors_v1) if vectors are -included. - -### spacy.CharacterEmbed.v1 {#CharacterEmbed_v1} - -Identical to [`spacy.CharacterEmbed.v2`](/api/architectures#CharacterEmbed) -except using [`spacy.StaticVectors.v1`](#StaticVectors_v1) if vectors are -included. - -## Layers {#layers} - -These functions are available from `@spacy.registry.layers`. - -### spacy.StaticVectors.v1 {#StaticVectors_v1} - -Identical to [`spacy.StaticVectors.v2`](/api/architectures#StaticVectors) except -for the handling of tokens without vectors. - - - -`spacy.StaticVectors.v1` maps tokens without vectors to the final row in the -vectors table, which causes the model predictions to change if new vectors are -added to an existing vectors table. See more details in -[issue #7662](https://github.com/explosion/spaCy/issues/7662#issuecomment-813925655). - - - ### spacy.TextCatCNN.v1 {#TextCatCNN_v1} Since `spacy.TextCatCNN.v2`, this architecture has become resizable, which means @@ -246,8 +222,35 @@ the others, but may not be as accurate, especially if texts are short. | `nO` | Output dimension, determined by the number of different labels. If not set, the [`TextCategorizer`](/api/textcategorizer) component will set it when `initialize` is called. ~~Optional[int]~~ | | **CREATES** | The model using the architecture. ~~Model[List[Doc], Floats2d]~~ | +### spacy.TransitionBasedParser.v1 {#TransitionBasedParser_v1} + +Identical to +[`spacy.TransitionBasedParser.v2`](/api/architectures#TransitionBasedParser) +except the `use_upper` was set to `True` by default. + +## Layers {#layers} + +These functions are available from `@spacy.registry.layers`. + +### spacy.StaticVectors.v1 {#StaticVectors_v1} + +Identical to [`spacy.StaticVectors.v2`](/api/architectures#StaticVectors) except +for the handling of tokens without vectors. + + + +`spacy.StaticVectors.v1` maps tokens without vectors to the final row in the +vectors table, which causes the model predictions to change if new vectors are +added to an existing vectors table. See more details in +[issue #7662](https://github.com/explosion/spaCy/issues/7662#issuecomment-813925655). + + + ## Loggers {#loggers} -Logging utilities for spaCy are implemented in the [`spacy-loggers`](https://github.com/explosion/spacy-loggers) repo, and the functions are typically available from `@spacy.registry.loggers`. +Logging utilities for spaCy are implemented in the +[`spacy-loggers`](https://github.com/explosion/spacy-loggers) repo, and the +functions are typically available from `@spacy.registry.loggers`. -More documentation can be found in that repo's [readme](https://github.com/explosion/spacy-loggers/blob/main/README.md) file. +More documentation can be found in that repo's +[readme](https://github.com/explosion/spacy-loggers/blob/main/README.md) file. diff --git a/website/docs/api/lemmatizer.md b/website/docs/api/lemmatizer.md index 75387305a..422f34040 100644 --- a/website/docs/api/lemmatizer.md +++ b/website/docs/api/lemmatizer.md @@ -118,7 +118,7 @@ shortcut for this and instantiate the component using its string name and | `name` | String name of the component instance. Used to add entries to the `losses` during training. ~~str~~ | | _keyword-only_ | | | mode | The lemmatizer mode, e.g. `"lookup"` or `"rule"`. Defaults to `"lookup"`. ~~str~~ | -| overwrite | Whether to overwrite existing lemmas. ~~bool~ | +| overwrite | Whether to overwrite existing lemmas. ~~bool~~ | ## Lemmatizer.\_\_call\_\_ {#call tag="method"} diff --git a/website/docs/api/matcher.md b/website/docs/api/matcher.md index 273c202ca..8cc446c6a 100644 --- a/website/docs/api/matcher.md +++ b/website/docs/api/matcher.md @@ -30,26 +30,26 @@ pattern keys correspond to a number of [`Token` attributes](/api/token#attributes). The supported attributes for rule-based matching are: -| Attribute |  Description | -| ----------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------- | -| `ORTH` | The exact verbatim text of a token. ~~str~~ | -| `TEXT` 2.1 | The exact verbatim text of a token. ~~str~~ | -| `NORM` | The normalized form of the token text. ~~str~~ | -| `LOWER` | The lowercase form of the token text. ~~str~~ | -|  `LENGTH` | The length of the token text. ~~int~~ | -|  `IS_ALPHA`, `IS_ASCII`, `IS_DIGIT` | Token text consists of alphabetic characters, ASCII characters, digits. ~~bool~~ | -|  `IS_LOWER`, `IS_UPPER`, `IS_TITLE` | Token text is in lowercase, uppercase, titlecase. ~~bool~~ | -|  `IS_PUNCT`, `IS_SPACE`, `IS_STOP` | Token is punctuation, whitespace, stop word. ~~bool~~ | -|  `IS_SENT_START` | Token is start of sentence. ~~bool~~ | -|  `LIKE_NUM`, `LIKE_URL`, `LIKE_EMAIL` | Token text resembles a number, URL, email. ~~bool~~ | -| `SPACY` | Token has a trailing space. ~~bool~~ | -|  `POS`, `TAG`, `MORPH`, `DEP`, `LEMMA`, `SHAPE` | The token's simple and extended part-of-speech tag, morphological analysis, dependency label, lemma, shape. ~~str~~ | -| `ENT_TYPE` | The token's entity label. ~~str~~ | -| `ENT_IOB` | The IOB part of the token's entity tag. ~~str~~ | -| `ENT_ID` | The token's entity ID (`ent_id`). ~~str~~ | -| `ENT_KB_ID` | The token's entity knowledge base ID (`ent_kb_id`). ~~str~~ | -| `_` 2.1 | Properties in [custom extension attributes](/usage/processing-pipelines#custom-components-attributes). ~~Dict[str, Any]~~ | -| `OP` | Operator or quantifier to determine how often to match a token pattern. ~~str~~ | +| Attribute | Description | +| ---------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------- | +| `ORTH` | The exact verbatim text of a token. ~~str~~ | +| `TEXT` 2.1 | The exact verbatim text of a token. ~~str~~ | +| `NORM` | The normalized form of the token text. ~~str~~ | +| `LOWER` | The lowercase form of the token text. ~~str~~ | +| `LENGTH` | The length of the token text. ~~int~~ | +| `IS_ALPHA`, `IS_ASCII`, `IS_DIGIT` | Token text consists of alphabetic characters, ASCII characters, digits. ~~bool~~ | +| `IS_LOWER`, `IS_UPPER`, `IS_TITLE` | Token text is in lowercase, uppercase, titlecase. ~~bool~~ | +| `IS_PUNCT`, `IS_SPACE`, `IS_STOP` | Token is punctuation, whitespace, stop word. ~~bool~~ | +| `IS_SENT_START` | Token is start of sentence. ~~bool~~ | +| `LIKE_NUM`, `LIKE_URL`, `LIKE_EMAIL` | Token text resembles a number, URL, email. ~~bool~~ | +| `SPACY` | Token has a trailing space. ~~bool~~ | +| `POS`, `TAG`, `MORPH`, `DEP`, `LEMMA`, `SHAPE` | The token's simple and extended part-of-speech tag, morphological analysis, dependency label, lemma, shape. ~~str~~ | +| `ENT_TYPE` | The token's entity label. ~~str~~ | +| `ENT_IOB` | The IOB part of the token's entity tag. ~~str~~ | +| `ENT_ID` | The token's entity ID (`ent_id`). ~~str~~ | +| `ENT_KB_ID` | The token's entity knowledge base ID (`ent_kb_id`). ~~str~~ | +| `_` 2.1 | Properties in [custom extension attributes](/usage/processing-pipelines#custom-components-attributes). ~~Dict[str, Any]~~ | +| `OP` | Operator or quantifier to determine how often to match a token pattern. ~~str~~ | Operators and quantifiers define **how often** a token pattern should be matched: @@ -59,15 +59,20 @@ matched: > [ > {"POS": "ADJ", "OP": "*"}, > {"POS": "NOUN", "OP": "+"} +> {"POS": "PROPN", "OP": "{2}"} > ] > ``` -| OP | Description | -| --- | ---------------------------------------------------------------- | -| `!` | Negate the pattern, by requiring it to match exactly 0 times. | -| `?` | Make the pattern optional, by allowing it to match 0 or 1 times. | -| `+` | Require the pattern to match 1 or more times. | -| `*` | Allow the pattern to match 0 or more times. | +| OP | Description | +|---------|------------------------------------------------------------------------| +| `!` | Negate the pattern, by requiring it to match exactly 0 times. | +| `?` | Make the pattern optional, by allowing it to match 0 or 1 times. | +| `+` | Require the pattern to match 1 or more times. | +| `*` | Allow the pattern to match 0 or more times. | +| `{n}` | Require the pattern to match exactly _n_ times. | +| `{n,m}` | Require the pattern to match at least _n_ but not more than _m_ times. | +| `{n,}` | Require the pattern to match at least _n_ times. | +| `{,m}` | Require the pattern to match at most _m_ times. | Token patterns can also map to a **dictionary of properties** instead of a single value to indicate whether the expected value is a member of a list or how @@ -113,6 +118,10 @@ string where an integer is expected) or unexpected property names. Find all token sequences matching the supplied patterns on the `Doc` or `Span`. +Note that if a single label has multiple patterns associated with it, the +returned matches don't provide a way to tell which pattern was responsible for +the match. + > #### Example > > ```python @@ -131,7 +140,7 @@ Find all token sequences matching the supplied patterns on the `Doc` or `Span`. | _keyword-only_ | | | `as_spans` 3 | Instead of tuples, return a list of [`Span`](/api/span) objects of the matches, with the `match_id` assigned as the span label. Defaults to `False`. ~~bool~~ | | `allow_missing` 3 | Whether to skip checks for missing annotation for attributes included in patterns. Defaults to `False`. ~~bool~~ | -| `with_alignments` 3.0.6 | Return match alignment information as part of the match tuple as `List[int]` with the same length as the matched span. Each entry denotes the corresponding index of the token pattern. If `as_spans` is set to `True`, this setting is ignored. Defaults to `False`. ~~bool~~ | +| `with_alignments` 3.0.6 | Return match alignment information as part of the match tuple as `List[int]` with the same length as the matched span. Each entry denotes the corresponding index of the token in the pattern. If `as_spans` is set to `True`, this setting is ignored. Defaults to `False`. ~~bool~~ | | **RETURNS** | A list of `(match_id, start, end)` tuples, describing the matches. A match tuple describes a span `doc[start:end`]. The `match_id` is the ID of the added match pattern. If `as_spans` is set to `True`, a list of `Span` objects is returned instead. ~~Union[List[Tuple[int, int, int]], List[Span]]~~ | ## Matcher.\_\_len\_\_ {#len tag="method" new="2"} @@ -190,7 +199,7 @@ will be overwritten. > [{"LOWER": "hello"}, {"LOWER": "world"}], > [{"ORTH": "Google"}, {"ORTH": "Maps"}] > ] -> matcher.add("TEST_PATTERNS", patterns) +> matcher.add("TEST_PATTERNS", patterns, on_match=on_match) > doc = nlp("HELLO WORLD on Google Maps.") > matches = matcher(doc) > ``` diff --git a/website/docs/api/morphologizer.md b/website/docs/api/morphologizer.md index 434c56833..f874e8bea 100644 --- a/website/docs/api/morphologizer.md +++ b/website/docs/api/morphologizer.md @@ -147,10 +147,10 @@ applied to the `Doc` in order. Both [`__call__`](/api/morphologizer#call) and ## Morphologizer.initialize {#initialize tag="method"} Initialize the component for training. `get_examples` should be a function that -returns an iterable of [`Example`](/api/example) objects. The data examples are -used to **initialize the model** of the component and can either be the full -training data or a representative sample. Initialization includes validating the -network, +returns an iterable of [`Example`](/api/example) objects. **At least one example +should be supplied.** The data examples are used to **initialize the model** of +the component and can either be the full training data or a representative +sample. Initialization includes validating the network, [inferring missing shapes](https://thinc.ai/docs/usage-models#validation) and setting up the label scheme based on the data. This method is typically called by [`Language.initialize`](/api/language#initialize) and lets you customize @@ -162,7 +162,7 @@ config. > > ```python > morphologizer = nlp.add_pipe("morphologizer") -> morphologizer.initialize(lambda: [], nlp=nlp) +> morphologizer.initialize(lambda: examples, nlp=nlp) > ``` > > ```ini @@ -176,7 +176,7 @@ config. | Name | Description | | -------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `get_examples` | Function that returns gold-standard annotations in the form of [`Example`](/api/example) objects. ~~Callable[[], Iterable[Example]]~~ | +| `get_examples` | Function that returns gold-standard annotations in the form of [`Example`](/api/example) objects. Must contain at least one `Example`. ~~Callable[[], Iterable[Example]]~~ | | _keyword-only_ | | | `nlp` | The current `nlp` object. Defaults to `None`. ~~Optional[Language]~~ | | `labels` | The label information to add to the component, as provided by the [`label_data`](#label_data) property after initialization. To generate a reusable JSON file from your data, you should run the [`init labels`](/api/cli#init-labels) command. If no labels are provided, the `get_examples` callback is used to extract the labels from the data, which may be a lot slower. ~~Optional[dict]~~ | diff --git a/website/docs/api/pipeline-functions.md b/website/docs/api/pipeline-functions.md index ff19d3e71..1b7017ca7 100644 --- a/website/docs/api/pipeline-functions.md +++ b/website/docs/api/pipeline-functions.md @@ -7,6 +7,7 @@ menu: - ['merge_entities', 'merge_entities'] - ['merge_subtokens', 'merge_subtokens'] - ['token_splitter', 'token_splitter'] + - ['doc_cleaner', 'doc_cleaner'] --- ## merge_noun_chunks {#merge_noun_chunks tag="function"} diff --git a/website/docs/api/sentencerecognizer.md b/website/docs/api/sentencerecognizer.md index 29bf10393..2f50350ae 100644 --- a/website/docs/api/sentencerecognizer.md +++ b/website/docs/api/sentencerecognizer.md @@ -132,10 +132,10 @@ and [`pipe`](/api/sentencerecognizer#pipe) delegate to the ## SentenceRecognizer.initialize {#initialize tag="method"} Initialize the component for training. `get_examples` should be a function that -returns an iterable of [`Example`](/api/example) objects. The data examples are -used to **initialize the model** of the component and can either be the full -training data or a representative sample. Initialization includes validating the -network, +returns an iterable of [`Example`](/api/example) objects. **At least one example +should be supplied.** The data examples are used to **initialize the model** of +the component and can either be the full training data or a representative +sample. Initialization includes validating the network, [inferring missing shapes](https://thinc.ai/docs/usage-models#validation) and setting up the label scheme based on the data. This method is typically called by [`Language.initialize`](/api/language#initialize). @@ -144,14 +144,14 @@ by [`Language.initialize`](/api/language#initialize). > > ```python > senter = nlp.add_pipe("senter") -> senter.initialize(lambda: [], nlp=nlp) +> senter.initialize(lambda: examples, nlp=nlp) > ``` -| Name | Description | -| -------------- | ------------------------------------------------------------------------------------------------------------------------------------- | -| `get_examples` | Function that returns gold-standard annotations in the form of [`Example`](/api/example) objects. ~~Callable[[], Iterable[Example]]~~ | -| _keyword-only_ | | -| `nlp` | The current `nlp` object. Defaults to `None`. ~~Optional[Language]~~ | +| Name | Description | +| -------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `get_examples` | Function that returns gold-standard annotations in the form of [`Example`](/api/example) objects. Must contain at least one `Example`. ~~Callable[[], Iterable[Example]]~~ | +| _keyword-only_ | | +| `nlp` | The current `nlp` object. Defaults to `None`. ~~Optional[Language]~~ | ## SentenceRecognizer.predict {#predict tag="method"} diff --git a/website/docs/api/span.md b/website/docs/api/span.md index d765a199c..89f608994 100644 --- a/website/docs/api/span.md +++ b/website/docs/api/span.md @@ -27,6 +27,7 @@ Create a `Span` object from the slice `doc[start : end]`. | `vector` | A meaning representation of the span. ~~numpy.ndarray[ndim=1, dtype=float32]~~ | | `vector_norm` | The L2 norm of the document's vector representation. ~~float~~ | | `kb_id` | A knowledge base ID to attach to the span, e.g. for named entities. ~~Union[str, int]~~ | +| `span_id` | An ID to associate with the span. ~~Union[str, int]~~ | ## Span.\_\_getitem\_\_ {#getitem tag="method"} @@ -560,7 +561,9 @@ overlaps with will be returned. | `lemma_` | The span's lemma. Equivalent to `"".join(token.text_with_ws for token in span)`. ~~str~~ | | `kb_id` | The hash value of the knowledge base ID referred to by the span. ~~int~~ | | `kb_id_` | The knowledge base ID referred to by the span. ~~str~~ | -| `ent_id` | The hash value of the named entity the token is an instance of. ~~int~~ | -| `ent_id_` | The string ID of the named entity the token is an instance of. ~~str~~ | +| `ent_id` | The hash value of the named entity the root token is an instance of. ~~int~~ | +| `ent_id_` | The string ID of the named entity the root token is an instance of. ~~str~~ | +| `id` | The hash value of the span's ID. ~~int~~ | +| `id_` | The span's ID. ~~str~~ | | `sentiment` | A scalar value indicating the positivity or negativity of the span. ~~float~~ | | `_` | User space for adding custom [attribute extensions](/usage/processing-pipelines#custom-components-attributes). ~~Underscore~~ | diff --git a/website/docs/api/spancategorizer.md b/website/docs/api/spancategorizer.md index f09ac8bdb..58a06bcf5 100644 --- a/website/docs/api/spancategorizer.md +++ b/website/docs/api/spancategorizer.md @@ -56,7 +56,7 @@ architectures and their arguments and hyperparameters. | -------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | `suggester` | A function that [suggests spans](#suggesters). Spans are returned as a ragged array with two integer columns, for the start and end positions. Defaults to [`ngram_suggester`](#ngram_suggester). ~~Callable[[Iterable[Doc], Optional[Ops]], Ragged]~~ | | `model` | A model instance that is given a a list of documents and `(start, end)` indices representing candidate span offsets. The model predicts a probability for each category for each span. Defaults to [SpanCategorizer](/api/architectures#SpanCategorizer). ~~Model[Tuple[List[Doc], Ragged], Floats2d]~~ | -| `spans_key` | Key of the [`Doc.spans`](/api/doc#spans) dict to save the spans under. During initialization and training, the component will look for spans on the reference document under the same key. Defaults to `"sc"`. ~~str~~ | +| `spans_key` | Key of the [`Doc.spans`](/api/doc#spans) dict to save the spans under. During initialization and training, the component will look for spans on the reference document under the same key. Defaults to `"sc"`. ~~str~~ | | `threshold` | Minimum probability to consider a prediction positive. Spans with a positive prediction will be saved on the Doc. Defaults to `0.5`. ~~float~~ | | `max_positive` | Maximum number of labels to consider positive per span. Defaults to `None`, indicating no limit. ~~Optional[int]~~ | | `scorer` | The scoring method. Defaults to [`Scorer.score_spans`](/api/scorer#score_spans) for `Doc.spans[spans_key]` with overlapping spans allowed. ~~Optional[Callable]~~ | @@ -93,7 +93,7 @@ shortcut for this and instantiate the component using its string name and | `suggester` | A function that [suggests spans](#suggesters). Spans are returned as a ragged array with two integer columns, for the start and end positions. ~~Callable[[Iterable[Doc], Optional[Ops]], Ragged]~~ | | `name` | String name of the component instance. Used to add entries to the `losses` during training. ~~str~~ | | _keyword-only_ | | -| `spans_key` | Key of the [`Doc.spans`](/api/doc#sans) dict to save the spans under. During initialization and training, the component will look for spans on the reference document under the same key. Defaults to `"sc"`. ~~str~~ | +| `spans_key` | Key of the [`Doc.spans`](/api/doc#sans) dict to save the spans under. During initialization and training, the component will look for spans on the reference document under the same key. Defaults to `"sc"`. ~~str~~ | | `threshold` | Minimum probability to consider a prediction positive. Spans with a positive prediction will be saved on the Doc. Defaults to `0.5`. ~~float~~ | | `max_positive` | Maximum number of labels to consider positive per span. Defaults to `None`, indicating no limit. ~~Optional[int]~~ | @@ -147,10 +147,10 @@ applied to the `Doc` in order. Both [`__call__`](/api/spancategorizer#call) and ## SpanCategorizer.initialize {#initialize tag="method"} Initialize the component for training. `get_examples` should be a function that -returns an iterable of [`Example`](/api/example) objects. The data examples are -used to **initialize the model** of the component and can either be the full -training data or a representative sample. Initialization includes validating the -network, +returns an iterable of [`Example`](/api/example) objects. **At least one example +should be supplied.** The data examples are used to **initialize the model** of +the component and can either be the full training data or a representative +sample. Initialization includes validating the network, [inferring missing shapes](https://thinc.ai/docs/usage-models#validation) and setting up the label scheme based on the data. This method is typically called by [`Language.initialize`](/api/language#initialize) and lets you customize @@ -162,7 +162,7 @@ config. > > ```python > spancat = nlp.add_pipe("spancat") -> spancat.initialize(lambda: [], nlp=nlp) +> spancat.initialize(lambda: examples, nlp=nlp) > ``` > > ```ini @@ -176,7 +176,7 @@ config. | Name | Description | | -------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `get_examples` | Function that returns gold-standard annotations in the form of [`Example`](/api/example) objects. ~~Callable[[], Iterable[Example]]~~ | +| `get_examples` | Function that returns gold-standard annotations in the form of [`Example`](/api/example) objects. Must contain at least one `Example`. ~~Callable[[], Iterable[Example]]~~ | | _keyword-only_ | | | `nlp` | The current `nlp` object. Defaults to `None`. ~~Optional[Language]~~ | | `labels` | The label information to add to the component, as provided by the [`label_data`](#label_data) property after initialization. To generate a reusable JSON file from your data, you should run the [`init labels`](/api/cli#init-labels) command. If no labels are provided, the `get_examples` callback is used to extract the labels from the data, which may be a lot slower. ~~Optional[Iterable[str]]~~ | diff --git a/website/docs/api/spangroup.md b/website/docs/api/spangroup.md index 1e2d18a82..8dbdefc01 100644 --- a/website/docs/api/spangroup.md +++ b/website/docs/api/spangroup.md @@ -233,7 +233,7 @@ group. > doc.spans["errors"] = [] > doc.spans["errors"].extend([doc[1:3], doc[0:1]]) > assert len(doc.spans["errors"]) == 2 -> span_group = SpanGroup([doc[1:4], doc[0:3]) +> span_group = SpanGroup(doc, spans=[doc[1:4], doc[0:3]]) > doc.spans["errors"].extend(span_group) > ``` diff --git a/website/docs/api/spanruler.md b/website/docs/api/spanruler.md new file mode 100644 index 000000000..b573f7c58 --- /dev/null +++ b/website/docs/api/spanruler.md @@ -0,0 +1,351 @@ +--- +title: SpanRuler +tag: class +source: spacy/pipeline/span_ruler.py +new: 3.3 +teaser: 'Pipeline component for rule-based span and named entity recognition' +api_string_name: span_ruler +api_trainable: false +--- + +The span ruler lets you add spans to [`Doc.spans`](/api/doc#spans) and/or +[`Doc.ents`](/api/doc#ents) using token-based rules or exact phrase matches. For +usage examples, see the docs on +[rule-based span matching](/usage/rule-based-matching#spanruler). + +## Assigned Attributes {#assigned-attributes} + +Matches will be saved to `Doc.spans[spans_key]` as a +[`SpanGroup`](/api/spangroup) and/or to `Doc.ents`, where the annotation is +saved in the `Token.ent_type` and `Token.ent_iob` fields. + +| Location | Value | +| ---------------------- | ----------------------------------------------------------------- | +| `Doc.spans[spans_key]` | The annotated spans. ~~SpanGroup~~ | +| `Doc.ents` | The annotated spans. ~~Tuple[Span]~~ | +| `Token.ent_iob` | An enum encoding of the IOB part of the named entity tag. ~~int~~ | +| `Token.ent_iob_` | The IOB part of the named entity tag. ~~str~~ | +| `Token.ent_type` | The label part of the named entity tag (hash). ~~int~~ | +| `Token.ent_type_` | The label part of the named entity tag. ~~str~~ | + +## Config and implementation {#config} + +The default config is defined by the pipeline component factory and describes +how the component should be configured. You can override its settings via the +`config` argument on [`nlp.add_pipe`](/api/language#add_pipe) or in your +[`config.cfg`](/usage/training#config). + +> #### Example +> +> ```python +> config = { +> "spans_key": "my_spans", +> "validate": True, +> "overwrite": False, +> } +> nlp.add_pipe("span_ruler", config=config) +> ``` + +| Setting | Description | +| --------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `spans_key` | The spans key to save the spans under. If `None`, no spans are saved. Defaults to `"ruler"`. ~~Optional[str]~~ | +| `spans_filter` | The optional method to filter spans before they are assigned to doc.spans. Defaults to `None`. ~~Optional[Callable[[Iterable[Span], Iterable[Span]], List[Span]]]~~ | +| `annotate_ents` | Whether to save spans to doc.ents. Defaults to `False`. ~~bool~~ | +| `ents_filter` | The method to filter spans before they are assigned to doc.ents. Defaults to `util.filter_chain_spans`. ~~Callable[[Iterable[Span], Iterable[Span]], List[Span]]~~ | +| `phrase_matcher_attr` | Token attribute to match on, passed to the internal PhraseMatcher as `attr`. Defaults to `None`. ~~Optional[Union[int, str]]~~ | +| `validate` | Whether patterns should be validated, passed to Matcher and PhraseMatcher as `validate`. Defaults to `False`. ~~bool~~ | +| `overwrite` | Whether to remove any existing spans under `Doc.spans[spans key]` if `spans_key` is set, or to remove any ents under `Doc.ents` if `annotate_ents` is set. Defaults to `True`. ~~bool~~ | +| `scorer` | The scoring method. Defaults to [`Scorer.score_spans`](/api/scorer#score_spans) for `Doc.spans[spans_key]` with overlapping spans allowed. ~~Optional[Callable]~~ | + +```python +%%GITHUB_SPACY/spacy/pipeline/span_ruler.py +``` + +## SpanRuler.\_\_init\_\_ {#init tag="method"} + +Initialize the span ruler. If patterns are supplied here, they need to be a list +of dictionaries with a `"label"` and `"pattern"` key. A pattern can either be a +token pattern (list) or a phrase pattern (string). For example: +`{"label": "ORG", "pattern": "Apple"}`. + +> #### Example +> +> ```python +> # Construction via add_pipe +> ruler = nlp.add_pipe("span_ruler") +> +> # Construction from class +> from spacy.pipeline import SpanRuler +> ruler = SpanRuler(nlp, overwrite=True) +> ``` + +| Name | Description | +| --------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `nlp` | The shared nlp object to pass the vocab to the matchers and process phrase patterns. ~~Language~~ | +| `name` | Instance name of the current pipeline component. Typically passed in automatically from the factory when the component is added. Used to disable the current span ruler while creating phrase patterns with the nlp object. ~~str~~ | +| _keyword-only_ | | +| `spans_key` | The spans key to save the spans under. If `None`, no spans are saved. Defaults to `"ruler"`. ~~Optional[str]~~ | +| `spans_filter` | The optional method to filter spans before they are assigned to doc.spans. Defaults to `None`. ~~Optional[Callable[[Iterable[Span], Iterable[Span]], List[Span]]]~~ | +| `annotate_ents` | Whether to save spans to doc.ents. Defaults to `False`. ~~bool~~ | +| `ents_filter` | The method to filter spans before they are assigned to doc.ents. Defaults to `util.filter_chain_spans`. ~~Callable[[Iterable[Span], Iterable[Span]], List[Span]]~~ | +| `phrase_matcher_attr` | Token attribute to match on, passed to the internal PhraseMatcher as `attr`. Defaults to `None`. ~~Optional[Union[int, str]]~~ | +| `validate` | Whether patterns should be validated, passed to Matcher and PhraseMatcher as `validate`. Defaults to `False`. ~~bool~~ | +| `overwrite` | Whether to remove any existing spans under `Doc.spans[spans key]` if `spans_key` is set, or to remove any ents under `Doc.ents` if `annotate_ents` is set. Defaults to `True`. ~~bool~~ | +| `scorer` | The scoring method. Defaults to [`Scorer.score_spans`](/api/scorer#score_spans) for `Doc.spans[spans_key]` with overlapping spans allowed. ~~Optional[Callable]~~ | + +## SpanRuler.initialize {#initialize tag="method"} + +Initialize the component with data and used before training to load in rules +from a [pattern file](/usage/rule-based-matching/#spanruler-files). This method +is typically called by [`Language.initialize`](/api/language#initialize) and +lets you customize arguments it receives via the +[`[initialize.components]`](/api/data-formats#config-initialize) block in the +config. Any existing patterns are removed on initialization. + +> #### Example +> +> ```python +> span_ruler = nlp.add_pipe("span_ruler") +> span_ruler.initialize(lambda: [], nlp=nlp, patterns=patterns) +> ``` +> +> ```ini +> ### config.cfg +> [initialize.components.span_ruler] +> +> [initialize.components.span_ruler.patterns] +> @readers = "srsly.read_jsonl.v1" +> path = "corpus/span_ruler_patterns.jsonl +> ``` + +| Name | Description | +| -------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `get_examples` | Function that returns gold-standard annotations in the form of [`Example`](/api/example) objects. Not used by the `SpanRuler`. ~~Callable[[], Iterable[Example]]~~ | +| _keyword-only_ | | +| `nlp` | The current `nlp` object. Defaults to `None`. ~~Optional[Language]~~ | +| `patterns` | The list of patterns. Defaults to `None`. ~~Optional[Sequence[Dict[str, Union[str, List[Dict[str, Any]]]]]]~~ | + +## SpanRuler.\_\len\_\_ {#len tag="method"} + +The number of all patterns added to the span ruler. + +> #### Example +> +> ```python +> ruler = nlp.add_pipe("span_ruler") +> assert len(ruler) == 0 +> ruler.add_patterns([{"label": "ORG", "pattern": "Apple"}]) +> assert len(ruler) == 1 +> ``` + +| Name | Description | +| ----------- | ------------------------------- | +| **RETURNS** | The number of patterns. ~~int~~ | + +## SpanRuler.\_\_contains\_\_ {#contains tag="method"} + +Whether a label is present in the patterns. + +> #### Example +> +> ```python +> ruler = nlp.add_pipe("span_ruler") +> ruler.add_patterns([{"label": "ORG", "pattern": "Apple"}]) +> assert "ORG" in ruler +> assert not "PERSON" in ruler +> ``` + +| Name | Description | +| ----------- | --------------------------------------------------- | +| `label` | The label to check. ~~str~~ | +| **RETURNS** | Whether the span ruler contains the label. ~~bool~~ | + +## SpanRuler.\_\_call\_\_ {#call tag="method"} + +Find matches in the `Doc` and add them to `doc.spans[span_key]` and/or +`doc.ents`. Typically, this happens automatically after the component has been +added to the pipeline using [`nlp.add_pipe`](/api/language#add_pipe). If the +span ruler was initialized with `overwrite=True`, existing spans and entities +will be removed. + +> #### Example +> +> ```python +> ruler = nlp.add_pipe("span_ruler") +> ruler.add_patterns([{"label": "ORG", "pattern": "Apple"}]) +> +> doc = nlp("A text about Apple.") +> spans = [(span.text, span.label_) for span in doc.spans["ruler"]] +> assert spans == [("Apple", "ORG")] +> ``` + +| Name | Description | +| ----------- | -------------------------------------------------------------------- | +| `doc` | The `Doc` object to process, e.g. the `Doc` in the pipeline. ~~Doc~~ | +| **RETURNS** | The modified `Doc` with added spans/entities. ~~Doc~~ | + +## SpanRuler.add_patterns {#add_patterns tag="method"} + +Add patterns to the span ruler. A pattern can either be a token pattern (list of +dicts) or a phrase pattern (string). For more details, see the usage guide on +[rule-based matching](/usage/rule-based-matching). + +> #### Example +> +> ```python +> patterns = [ +> {"label": "ORG", "pattern": "Apple"}, +> {"label": "GPE", "pattern": [{"lower": "san"}, {"lower": "francisco"}]} +> ] +> ruler = nlp.add_pipe("span_ruler") +> ruler.add_patterns(patterns) +> ``` + +| Name | Description | +| ---------- | ---------------------------------------------------------------- | +| `patterns` | The patterns to add. ~~List[Dict[str, Union[str, List[dict]]]]~~ | + +## SpanRuler.remove {#remove tag="method"} + +Remove patterns by label from the span ruler. A `ValueError` is raised if the +label does not exist in any patterns. + +> #### Example +> +> ```python +> patterns = [{"label": "ORG", "pattern": "Apple", "id": "apple"}] +> ruler = nlp.add_pipe("span_ruler") +> ruler.add_patterns(patterns) +> ruler.remove("ORG") +> ``` + +| Name | Description | +| ------- | -------------------------------------- | +| `label` | The label of the pattern rule. ~~str~~ | + +## SpanRuler.remove_by_id {#remove_by_id tag="method"} + +Remove patterns by ID from the span ruler. A `ValueError` is raised if the ID +does not exist in any patterns. + +> #### Example +> +> ```python +> patterns = [{"label": "ORG", "pattern": "Apple", "id": "apple"}] +> ruler = nlp.add_pipe("span_ruler") +> ruler.add_patterns(patterns) +> ruler.remove_by_id("apple") +> ``` + +| Name | Description | +| ------------ | ----------------------------------- | +| `pattern_id` | The ID of the pattern rule. ~~str~~ | + +## SpanRuler.clear {#clear tag="method"} + +Remove all patterns the span ruler. + +> #### Example +> +> ```python +> patterns = [{"label": "ORG", "pattern": "Apple", "id": "apple"}] +> ruler = nlp.add_pipe("span_ruler") +> ruler.add_patterns(patterns) +> ruler.clear() +> ``` + +## SpanRuler.to_disk {#to_disk tag="method"} + +Save the span ruler patterns to a directory. The patterns will be saved as +newline-delimited JSON (JSONL). + +> #### Example +> +> ```python +> ruler = nlp.add_pipe("span_ruler") +> ruler.to_disk("/path/to/span_ruler") +> ``` + +| Name | Description | +| ------ | ------------------------------------------------------------------------------------------------------------------------------------------ | +| `path` | A path to a directory, which will be created if it doesn't exist. Paths may be either strings or `Path`-like objects. ~~Union[str, Path]~~ | + +## SpanRuler.from_disk {#from_disk tag="method"} + +Load the span ruler from a path. + +> #### Example +> +> ```python +> ruler = nlp.add_pipe("span_ruler") +> ruler.from_disk("/path/to/span_ruler") +> ``` + +| Name | Description | +| ----------- | ----------------------------------------------------------------------------------------------- | +| `path` | A path to a directory. Paths may be either strings or `Path`-like objects. ~~Union[str, Path]~~ | +| **RETURNS** | The modified `SpanRuler` object. ~~SpanRuler~~ | + +## SpanRuler.to_bytes {#to_bytes tag="method"} + +Serialize the span ruler to a bytestring. + +> #### Example +> +> ```python +> ruler = nlp.add_pipe("span_ruler") +> ruler_bytes = ruler.to_bytes() +> ``` + +| Name | Description | +| ----------- | ---------------------------------- | +| **RETURNS** | The serialized patterns. ~~bytes~~ | + +## SpanRuler.from_bytes {#from_bytes tag="method"} + +Load the pipe from a bytestring. Modifies the object in place and returns it. + +> #### Example +> +> ```python +> ruler_bytes = ruler.to_bytes() +> ruler = nlp.add_pipe("span_ruler") +> ruler.from_bytes(ruler_bytes) +> ``` + +| Name | Description | +| ------------ | ---------------------------------------------- | +| `bytes_data` | The bytestring to load. ~~bytes~~ | +| **RETURNS** | The modified `SpanRuler` object. ~~SpanRuler~~ | + +## SpanRuler.labels {#labels tag="property"} + +All labels present in the match patterns. + +| Name | Description | +| ----------- | -------------------------------------- | +| **RETURNS** | The string labels. ~~Tuple[str, ...]~~ | + +## SpanRuler.ids {#ids tag="property"} + +All IDs present in the `id` property of the match patterns. + +| Name | Description | +| ----------- | ----------------------------------- | +| **RETURNS** | The string IDs. ~~Tuple[str, ...]~~ | + +## SpanRuler.patterns {#patterns tag="property"} + +All patterns that were added to the span ruler. + +| Name | Description | +| ----------- | ---------------------------------------------------------------------------------------- | +| **RETURNS** | The original patterns, one dictionary per pattern. ~~List[Dict[str, Union[str, dict]]]~~ | + +## Attributes {#attributes} + +| Name | Description | +| ---------------- | -------------------------------------------------------------------------------- | +| `key` | The spans key that spans are saved under. ~~Optional[str]~~ | +| `matcher` | The underlying matcher used to process token patterns. ~~Matcher~~ | +| `phrase_matcher` | The underlying phrase matcher used to process phrase patterns. ~~PhraseMatcher~~ | diff --git a/website/docs/api/stringstore.md b/website/docs/api/stringstore.md index d5f78dbab..cd414b1f0 100644 --- a/website/docs/api/stringstore.md +++ b/website/docs/api/stringstore.md @@ -161,7 +161,7 @@ Load state from a binary string. > #### Example > > ```python -> fron spacy.strings import StringStore +> from spacy.strings import StringStore > store_bytes = stringstore.to_bytes() > new_store = StringStore().from_bytes(store_bytes) > ``` diff --git a/website/docs/api/tagger.md b/website/docs/api/tagger.md index b51864d3a..90a49b197 100644 --- a/website/docs/api/tagger.md +++ b/website/docs/api/tagger.md @@ -130,10 +130,10 @@ applied to the `Doc` in order. Both [`__call__`](/api/tagger#call) and ## Tagger.initialize {#initialize tag="method" new="3"} Initialize the component for training. `get_examples` should be a function that -returns an iterable of [`Example`](/api/example) objects. The data examples are -used to **initialize the model** of the component and can either be the full -training data or a representative sample. Initialization includes validating the -network, +returns an iterable of [`Example`](/api/example) objects. **At least one example +should be supplied.** The data examples are used to **initialize the model** of +the component and can either be the full training data or a representative +sample. Initialization includes validating the network, [inferring missing shapes](https://thinc.ai/docs/usage-models#validation) and setting up the label scheme based on the data. This method is typically called by [`Language.initialize`](/api/language#initialize) and lets you customize @@ -151,7 +151,7 @@ This method was previously called `begin_training`. > > ```python > tagger = nlp.add_pipe("tagger") -> tagger.initialize(lambda: [], nlp=nlp) +> tagger.initialize(lambda: examples, nlp=nlp) > ``` > > ```ini @@ -165,7 +165,7 @@ This method was previously called `begin_training`. | Name | Description | | -------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `get_examples` | Function that returns gold-standard annotations in the form of [`Example`](/api/example) objects. ~~Callable[[], Iterable[Example]]~~ | +| `get_examples` | Function that returns gold-standard annotations in the form of [`Example`](/api/example) objects. Must contain at least one `Example`. ~~Callable[[], Iterable[Example]]~~ | | _keyword-only_ | | | `nlp` | The current `nlp` object. Defaults to `None`. ~~Optional[Language]~~ | | `labels` | The label information to add to the component, as provided by the [`label_data`](#label_data) property after initialization. To generate a reusable JSON file from your data, you should run the [`init labels`](/api/cli#init-labels) command. If no labels are provided, the `get_examples` callback is used to extract the labels from the data, which may be a lot slower. ~~Optional[Iterable[str]]~~ | diff --git a/website/docs/api/textcategorizer.md b/website/docs/api/textcategorizer.md index 2ff569bad..042b4ab76 100644 --- a/website/docs/api/textcategorizer.md +++ b/website/docs/api/textcategorizer.md @@ -84,6 +84,7 @@ architectures and their arguments and hyperparameters. | ----------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------- | | `threshold` | Cutoff to consider a prediction "positive", relevant when printing accuracy results. ~~float~~ | | `model` | A model instance that predicts scores for each category. Defaults to [TextCatEnsemble](/api/architectures#TextCatEnsemble). ~~Model[List[Doc], List[Floats2d]]~~ | +| `scorer` | The scoring method. Defaults to [`Scorer.score_cats`](/api/scorer#score_cats) for the attribute `"cats"`. ~~Optional[Callable]~~ | ```python %%GITHUB_SPACY/spacy/pipeline/textcat.py @@ -175,10 +176,10 @@ applied to the `Doc` in order. Both [`__call__`](/api/textcategorizer#call) and ## TextCategorizer.initialize {#initialize tag="method" new="3"} Initialize the component for training. `get_examples` should be a function that -returns an iterable of [`Example`](/api/example) objects. The data examples are -used to **initialize the model** of the component and can either be the full -training data or a representative sample. Initialization includes validating the -network, +returns an iterable of [`Example`](/api/example) objects. **At least one example +should be supplied.** The data examples are used to **initialize the model** of +the component and can either be the full training data or a representative +sample. Initialization includes validating the network, [inferring missing shapes](https://thinc.ai/docs/usage-models#validation) and setting up the label scheme based on the data. This method is typically called by [`Language.initialize`](/api/language#initialize) and lets you customize @@ -196,7 +197,7 @@ This method was previously called `begin_training`. > > ```python > textcat = nlp.add_pipe("textcat") -> textcat.initialize(lambda: [], nlp=nlp) +> textcat.initialize(lambda: examples, nlp=nlp) > ``` > > ```ini @@ -211,7 +212,7 @@ This method was previously called `begin_training`. | Name | Description | | ---------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `get_examples` | Function that returns gold-standard annotations in the form of [`Example`](/api/example) objects. ~~Callable[[], Iterable[Example]]~~ | +| `get_examples` | Function that returns gold-standard annotations in the form of [`Example`](/api/example) objects. Must contain at least one `Example`. ~~Callable[[], Iterable[Example]]~~ | | _keyword-only_ | | | `nlp` | The current `nlp` object. Defaults to `None`. ~~Optional[Language]~~ | | `labels` | The label information to add to the component, as provided by the [`label_data`](#label_data) property after initialization. To generate a reusable JSON file from your data, you should run the [`init labels`](/api/cli#init-labels) command. If no labels are provided, the `get_examples` callback is used to extract the labels from the data, which may be a lot slower. ~~Optional[Iterable[str]]~~ | diff --git a/website/docs/api/tok2vec.md b/website/docs/api/tok2vec.md index 70c352b4d..2dcb1a013 100644 --- a/website/docs/api/tok2vec.md +++ b/website/docs/api/tok2vec.md @@ -127,10 +127,10 @@ and [`set_annotations`](/api/tok2vec#set_annotations) methods. Initialize the component for training and return an [`Optimizer`](https://thinc.ai/docs/api-optimizers). `get_examples` should be a -function that returns an iterable of [`Example`](/api/example) objects. The data -examples are used to **initialize the model** of the component and can either be -the full training data or a representative sample. Initialization includes -validating the network, +function that returns an iterable of [`Example`](/api/example) objects. **At +least one example should be supplied.** The data examples are used to +**initialize the model** of the component and can either be the full training +data or a representative sample. Initialization includes validating the network, [inferring missing shapes](https://thinc.ai/docs/usage-models#validation) and setting up the label scheme based on the data. This method is typically called by [`Language.initialize`](/api/language#initialize). @@ -139,14 +139,14 @@ by [`Language.initialize`](/api/language#initialize). > > ```python > tok2vec = nlp.add_pipe("tok2vec") -> tok2vec.initialize(lambda: [], nlp=nlp) +> tok2vec.initialize(lambda: examples, nlp=nlp) > ``` -| Name | Description | -| -------------- | ------------------------------------------------------------------------------------------------------------------------------------- | -| `get_examples` | Function that returns gold-standard annotations in the form of [`Example`](/api/example) objects. ~~Callable[[], Iterable[Example]]~~ | -| _keyword-only_ | | -| `nlp` | The current `nlp` object. Defaults to `None`. ~~Optional[Language]~~ | +| Name | Description | +| -------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `get_examples` | Function that returns gold-standard annotations in the form of [`Example`](/api/example) objects. Must contain at least one `Example`. ~~Callable[[], Iterable[Example]]~~ | +| _keyword-only_ | | +| `nlp` | The current `nlp` object. Defaults to `None`. ~~Optional[Language]~~ | ## Tok2Vec.predict {#predict tag="method"} diff --git a/website/docs/api/token.md b/website/docs/api/token.md index 3c3d12d54..d43cd3ff1 100644 --- a/website/docs/api/token.md +++ b/website/docs/api/token.md @@ -221,7 +221,7 @@ dependency tree. ## Token.ancestors {#ancestors tag="property" model="parser"} -The rightmost token of this token's syntactic descendants. +A sequence of the token's syntactic ancestors (parents, grandparents, etc). > #### Example > diff --git a/website/docs/api/top-level.md b/website/docs/api/top-level.md index f2fd1415f..c96c571e9 100644 --- a/website/docs/api/top-level.md +++ b/website/docs/api/top-level.md @@ -51,6 +51,7 @@ specified separately using the new `exclude` keyword argument. | _keyword-only_ | | | `vocab` | Optional shared vocab to pass in on initialization. If `True` (default), a new `Vocab` object will be created. ~~Union[Vocab, bool]~~ | | `disable` | Names of pipeline components to [disable](/usage/processing-pipelines#disabling). Disabled pipes will be loaded but they won't be run unless you explicitly enable them by calling [nlp.enable_pipe](/api/language#enable_pipe). ~~List[str]~~ | +| `enable` | Names of pipeline components to [enable](/usage/processing-pipelines#disabling). All other pipes will be disabled. ~~List[str]~~ | | `exclude` 3 | Names of pipeline components to [exclude](/usage/processing-pipelines#disabling). Excluded components won't be loaded. ~~List[str]~~ | | `config` 3 | Optional config overrides, either as nested dict or dict keyed by section value in dot notation, e.g. `"components.name.value"`. ~~Union[Dict[str, Any], Config]~~ | | **RETURNS** | A `Language` object with the loaded pipeline. ~~Language~~ | @@ -239,7 +240,7 @@ browser. Will run a simple web server. | Name | Description | | --------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------- | | `docs` | Document(s) or span(s) to visualize. ~~Union[Iterable[Union[Doc, Span]], Doc, Span]~~ | -| `style` | Visualization style, `"dep"` or `"ent"`. Defaults to `"dep"`. ~~str~~ | +| `style` | Visualization style, `"dep"`, `"ent"` or `"span"` 3.3. Defaults to `"dep"`. ~~str~~ | | `page` | Render markup as full HTML page. Defaults to `True`. ~~bool~~ | | `minify` | Minify HTML markup. Defaults to `False`. ~~bool~~ | | `options` | [Visualizer-specific options](#displacy_options), e.g. colors. ~~Dict[str, Any]~~ | @@ -264,7 +265,7 @@ Render a dependency parse tree or named entity visualization. | Name | Description | | ----------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | `docs` | Document(s) or span(s) to visualize. ~~Union[Iterable[Union[Doc, Span, dict]], Doc, Span, dict]~~ | -| `style` | Visualization style, `"dep"` or `"ent"`. Defaults to `"dep"`. ~~str~~ | +| `style` | Visualization style,`"dep"`, `"ent"` or `"span"` 3.3. Defaults to `"dep"`. ~~str~~ | | `page` | Render markup as full HTML page. Defaults to `True`. ~~bool~~ | | `minify` | Minify HTML markup. Defaults to `False`. ~~bool~~ | | `options` | [Visualizer-specific options](#displacy_options), e.g. colors. ~~Dict[str, Any]~~ | @@ -320,7 +321,6 @@ If a setting is not present in the options, the default value will be used. | `template` 2.2 | Optional template to overwrite the HTML used to render entity spans. Should be a format string and can use `{bg}`, `{text}` and `{label}`. See [`templates.py`](%%GITHUB_SPACY/spacy/displacy/templates.py) for examples. ~~Optional[str]~~ | | `kb_url_template` 3.2.1 | Optional template to construct the KB url for the entity to link to. Expects a python f-string format with single field to fill in. ~~Optional[str]~~ | - #### Span Visualizer options {#displacy_options-span} > #### Example @@ -330,21 +330,19 @@ If a setting is not present in the options, the default value will be used. > displacy.serve(doc, style="span", options=options) > ``` -| Name | Description | -|-----------------|---------------------------------------------------------------------------------------------------------------------------------------------------------| -| `spans_key` | Which spans key to render spans from. Default is `"sc"`. ~~str~~ | +| Name | Description | +| ----------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `spans_key` | Which spans key to render spans from. Default is `"sc"`. ~~str~~ | | `templates` | Dictionary containing the keys `"span"`, `"slice"`, and `"start"`. These dictate how the overall span, a span slice, and the starting token will be rendered. ~~Optional[Dict[str, str]~~ | -| `kb_url_template` | Optional template to construct the KB url for the entity to link to. Expects a python f-string format with single field to fill in ~~Optional[str]~~ | -| `colors` | Color overrides. Entity types should be mapped to color names or values. ~~Dict[str, str]~~ | +| `kb_url_template` | Optional template to construct the KB url for the entity to link to. Expects a python f-string format with single field to fill in ~~Optional[str]~~ | +| `colors` | Color overrides. Entity types should be mapped to color names or values. ~~Dict[str, str]~~ | - -By default, displaCy comes with colors for all entity types used by [spaCy's -trained pipelines](/models) for both entity and span visualizer. If you're -using custom entity types, you can use the `colors` setting to add your own -colors for them. Your application or pipeline package can also expose a -[`spacy_displacy_colors` entry -point](/usage/saving-loading#entry-points-displacy) to add custom labels and -their colors automatically. +By default, displaCy comes with colors for all entity types used by +[spaCy's trained pipelines](/models) for both entity and span visualizer. If +you're using custom entity types, you can use the `colors` setting to add your +own colors for them. Your application or pipeline package can also expose a +[`spacy_displacy_colors` entry point](/usage/saving-loading#entry-points-displacy) +to add custom labels and their colors automatically. By default, displaCy links to `#` for entities without a `kb_id` set on their span. If you wish to link an entity to their URL then consider using the @@ -354,7 +352,6 @@ span. If you wish to link an entity to their URL then consider using the should redirect you to their Wikidata page, in this case `https://www.wikidata.org/wiki/Q95`. - ## registry {#registry source="spacy/util.py" new="3"} spaCy's function registry extends @@ -443,8 +440,8 @@ and the accuracy scores on the development set. The built-in, default logger is the ConsoleLogger, which prints results to the console in tabular format. The [spacy-loggers](https://github.com/explosion/spacy-loggers) package, included as -a dependency of spaCy, enables other loggers, such as one that -sends results to a [Weights & Biases](https://www.wandb.com/) dashboard. +a dependency of spaCy, enables other loggers, such as one that sends results to +a [Weights & Biases](https://www.wandb.com/) dashboard. Instead of using one of the built-in loggers, you can [implement your own](/usage/training#custom-logging). @@ -583,14 +580,14 @@ the [`Corpus`](/api/corpus) class. > limit = 0 > ``` -| Name | Description | -| --------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `path` | The directory or filename to read from. Expects data in spaCy's binary [`.spacy` format](/api/data-formats#binary-training). ~~Union[str, Path]~~ | -|  `gold_preproc` | Whether to set up the Example object with gold-standard sentences and tokens for the predictions. See [`Corpus`](/api/corpus#init) for details. ~~bool~~ | -| `max_length` | Maximum document length. Longer documents will be split into sentences, if sentence boundaries are available. Defaults to `0` for no limit. ~~int~~ | -| `limit` | Limit corpus to a subset of examples, e.g. for debugging. Defaults to `0` for no limit. ~~int~~ | -| `augmenter` | Apply some simply data augmentation, where we replace tokens with variations. This is especially useful for punctuation and case replacement, to help generalize beyond corpora that don't have smart-quotes, or only have smart quotes, etc. Defaults to `None`. ~~Optional[Callable]~~ | -| **CREATES** | The corpus reader. ~~Corpus~~ | +| Name | Description | +| -------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `path` | The directory or filename to read from. Expects data in spaCy's binary [`.spacy` format](/api/data-formats#binary-training). ~~Union[str, Path]~~ | +| `gold_preproc` | Whether to set up the Example object with gold-standard sentences and tokens for the predictions. See [`Corpus`](/api/corpus#init) for details. ~~bool~~ | +| `max_length` | Maximum document length. Longer documents will be split into sentences, if sentence boundaries are available. Defaults to `0` for no limit. ~~int~~ | +| `limit` | Limit corpus to a subset of examples, e.g. for debugging. Defaults to `0` for no limit. ~~int~~ | +| `augmenter` | Apply some simply data augmentation, where we replace tokens with variations. This is especially useful for punctuation and case replacement, to help generalize beyond corpora that don't have smart-quotes, or only have smart quotes, etc. Defaults to `None`. ~~Optional[Callable]~~ | +| **CREATES** | The corpus reader. ~~Corpus~~ | #### spacy.JsonlCorpus.v1 {#jsonlcorpus tag="registered function"} diff --git a/website/docs/api/transformer.md b/website/docs/api/transformer.md index b1673cdbe..e747ad383 100644 --- a/website/docs/api/transformer.md +++ b/website/docs/api/transformer.md @@ -175,10 +175,10 @@ applied to the `Doc` in order. Both [`__call__`](/api/transformer#call) and Initialize the component for training and return an [`Optimizer`](https://thinc.ai/docs/api-optimizers). `get_examples` should be a -function that returns an iterable of [`Example`](/api/example) objects. The data -examples are used to **initialize the model** of the component and can either be -the full training data or a representative sample. Initialization includes -validating the network, +function that returns an iterable of [`Example`](/api/example) objects. **At +least one example should be supplied.** The data examples are used to +**initialize the model** of the component and can either be the full training +data or a representative sample. Initialization includes validating the network, [inferring missing shapes](https://thinc.ai/docs/usage-models#validation) and setting up the label scheme based on the data. This method is typically called by [`Language.initialize`](/api/language#initialize). @@ -187,14 +187,14 @@ by [`Language.initialize`](/api/language#initialize). > > ```python > trf = nlp.add_pipe("transformer") -> trf.initialize(lambda: iter([]), nlp=nlp) +> trf.initialize(lambda: examples, nlp=nlp) > ``` -| Name | Description | -| -------------- | ------------------------------------------------------------------------------------------------------------------------------------- | -| `get_examples` | Function that returns gold-standard annotations in the form of [`Example`](/api/example) objects. ~~Callable[[], Iterable[Example]]~~ | -| _keyword-only_ | | -| `nlp` | The current `nlp` object. Defaults to `None`. ~~Optional[Language]~~ | +| Name | Description | +| -------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `get_examples` | Function that returns gold-standard annotations in the form of [`Example`](/api/example) objects. Must contain at least one `Example`. ~~Callable[[], Iterable[Example]]~~ | +| _keyword-only_ | | +| `nlp` | The current `nlp` object. Defaults to `None`. ~~Optional[Language]~~ | ## Transformer.predict {#predict tag="method"} diff --git a/website/docs/models/index.md b/website/docs/models/index.md index 9ee96528e..203555651 100644 --- a/website/docs/models/index.md +++ b/website/docs/models/index.md @@ -115,7 +115,7 @@ The Finnish, Korean and Swedish `md` and `lg` pipelines use running a trained pipeline on texts and working with [`Doc`](/api/doc) objects, you shouldn't notice any difference with floret vectors. With floret vectors no tokens are out-of-vocabulary, so [`Token.is_oov`](/api/token#attributes) will -return `True` for all tokens. +return `False` for all tokens. If you access vectors directly for similarity comparisons, there are a few differences because floret vectors don't include a fixed word list like the diff --git a/website/docs/usage/embeddings-transformers.md b/website/docs/usage/embeddings-transformers.md index 70fa95099..a487371de 100644 --- a/website/docs/usage/embeddings-transformers.md +++ b/website/docs/usage/embeddings-transformers.md @@ -530,7 +530,8 @@ models, which can **improve the accuracy** of your components. Word vectors in spaCy are "static" in the sense that they are not learned parameters of the statistical models, and spaCy itself does not feature any algorithms for learning word vector tables. You can train a word vectors table -using tools such as [Gensim](https://radimrehurek.com/gensim/), +using tools such as [floret](https://github.com/explosion/floret), +[Gensim](https://radimrehurek.com/gensim/), [FastText](https://fasttext.cc/) or [GloVe](https://nlp.stanford.edu/projects/glove/), or download existing pretrained vectors. The [`init vectors`](/api/cli#init-vectors) command lets you diff --git a/website/docs/usage/index.md b/website/docs/usage/index.md index 54ab62467..1f4869606 100644 --- a/website/docs/usage/index.md +++ b/website/docs/usage/index.md @@ -129,15 +129,14 @@ machine learning library, [Thinc](https://thinc.ai). For GPU support, we've been grateful to use the work of Chainer's [CuPy](https://cupy.chainer.org) module, which provides a numpy-compatible interface for GPU arrays. -spaCy can be installed on GPU by specifying `spacy[cuda]`, `spacy[cuda90]`, -`spacy[cuda91]`, `spacy[cuda92]`, `spacy[cuda100]`, `spacy[cuda101]`, -`spacy[cuda102]`, `spacy[cuda110]`, `spacy[cuda111]` or `spacy[cuda112]`. If you -know your cuda version, using the more explicit specifier allows cupy to be -installed via wheel, saving some compilation time. The specifiers should install +spaCy can be installed for a CUDA-compatible GPU by specifying `spacy[cuda]`, +`spacy[cuda102]`, `spacy[cuda112]`, `spacy[cuda113]`, etc. If you know your CUDA +version, using the more explicit specifier allows CuPy to be installed via +wheel, saving some compilation time. The specifiers should install [`cupy`](https://cupy.chainer.org). ```bash -$ pip install -U %%SPACY_PKG_NAME[cuda92]%%SPACY_PKG_FLAGS +$ pip install -U %%SPACY_PKG_NAME[cuda113]%%SPACY_PKG_FLAGS ``` Once you have a GPU-enabled installation, the best way to activate it is to call @@ -196,29 +195,73 @@ How to install compilers and related build tools: [Visual Studio Express](https://www.visualstudio.com/vs/visual-studio-express/) that matches the version that was used to compile your Python interpreter. +#### Using build constraints when compiling from source + +If you install spaCy from source or with `pip` for platforms where there are not +binary wheels on PyPI, you may need to use build constraints if any package in +your environment requires an older version of `numpy`. + +If `numpy` gets downgraded from the most recent release at any point after +you've compiled `spacy`, you might see an error that looks like this: + +```none +numpy.ndarray size changed, may indicate binary incompatibility. +``` + +To fix this, create a new virtual environment and install `spacy` and all of its +dependencies using build constraints. +[Build constraints](https://pip.pypa.io/en/stable/user_guide/#constraints-files) +specify an older version of `numpy` that is only used while compiling `spacy`, +and then your runtime environment can use any newer version of `numpy` and still +be compatible. In addition, use `--no-cache-dir` to ignore any previously cached +wheels so that all relevant packages are recompiled from scratch: + +```shell +PIP_CONSTRAINT=https://raw.githubusercontent.com/explosion/spacy/master/build-constraints.txt \ +pip install spacy --no-cache-dir +``` + +Our build constraints currently specify the oldest supported `numpy` available +on PyPI for `x86_64` and `aarch64`. Depending on your platform and environment, +you may want to customize the specific versions of `numpy`. For other platforms, +you can have a look at SciPy's +[`oldest-supported-numpy`](https://github.com/scipy/oldest-supported-numpy/blob/main/setup.cfg) +package to see what the oldest recommended versions of `numpy` are. + +(_Warning_: don't use `pip install -c constraints.txt` instead of +`PIP_CONSTRAINT`, since this isn't applied to the isolated build environments.) + #### Additional options for developers {#source-developers} Some additional options may be useful for spaCy developers who are editing the source code and recompiling frequently. -- Install in editable mode. Changes to `.py` files will be reflected as soon as - the files are saved, but edits to Cython files (`.pxd`, `.pyx`) will require - the `pip install` or `python setup.py build_ext` command below to be run - again. Before installing in editable mode, be sure you have removed any - previous installs with `pip uninstall spacy`, which you may need to run - multiple times to remove all traces of earlier installs. +- Install in editable mode. Changes to `.py` files will be reflected as soon + as the files are saved, but edits to Cython files (`.pxd`, `.pyx`) will + require the `pip install` command below to be run again. Before installing in + editable mode, be sure you have removed any previous installs with + `pip uninstall spacy`, which you may need to run multiple times to remove all + traces of earlier installs. ```bash $ pip install -r requirements.txt $ pip install --no-build-isolation --editable . ``` -- Build in parallel using `N` CPUs to speed up compilation and then install in - editable mode: +- Build in parallel. Starting in v3.4.0, you can specify the number of + build jobs with the environment variable `SPACY_NUM_BUILD_JOBS`: ```bash $ pip install -r requirements.txt - $ python setup.py build_ext --inplace -j N + $ SPACY_NUM_BUILD_JOBS=4 pip install --no-build-isolation --editable . + ``` + +- For editable mode and parallel builds with `python setup.py` instead of `pip` + (no longer recommended): + + ```bash + $ pip install -r requirements.txt + $ python setup.py build_ext --inplace -j 4 $ python setup.py develop ``` diff --git a/website/docs/usage/linguistic-features.md b/website/docs/usage/linguistic-features.md index b3b896a54..9dae6f2ee 100644 --- a/website/docs/usage/linguistic-features.md +++ b/website/docs/usage/linguistic-features.md @@ -48,7 +48,7 @@ but do not change its part-of-speech. We say that a **lemma** (root form) is **inflected** (modified/combined) with one or more **morphological features** to create a surface form. Here are some examples: -| Context | Surface | Lemma | POS |  Morphological Features | +| Context | Surface | Lemma | POS | Morphological Features | | ---------------------------------------- | ------- | ----- | ------ | ---------------------------------------- | | I was reading the paper | reading | read | `VERB` | `VerbForm=Ger` | | I don't watch the news, I read the paper | read | read | `VERB` | `VerbForm=Fin`, `Mood=Ind`, `Tense=Pres` | @@ -430,7 +430,7 @@ for token in doc: print(token.text, token.pos_, token.dep_, token.head.text) ``` -| Text |  POS | Dep | Head text | +| Text | POS | Dep | Head text | | ----------------------------------- | ------ | ------- | --------- | | Credit and mortgage account holders | `NOUN` | `nsubj` | submit | | must | `VERB` | `aux` | submit | @@ -1899,7 +1899,7 @@ access to some nice Latin vectors. You can then pass the directory path to > ``` ```cli -$ wget https://s3-us-west-1.amazonaws.com/fasttext-vectors/word-vectors-v2/cc.la.300.vec.gz +$ wget https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.la.300.vec.gz $ python -m spacy init vectors en cc.la.300.vec.gz /tmp/la_vectors_wiki_lg ``` diff --git a/website/docs/usage/processing-pipelines.md b/website/docs/usage/processing-pipelines.md index 4f75b5193..bd28810ae 100644 --- a/website/docs/usage/processing-pipelines.md +++ b/website/docs/usage/processing-pipelines.md @@ -362,6 +362,18 @@ nlp = spacy.load("en_core_web_sm", disable=["tagger", "parser"]) nlp.enable_pipe("tagger") ``` +In addition to `disable`, `spacy.load()` also accepts `enable`. If `enable` is +set, all components except for those in `enable` are disabled. + +```python +# Load the complete pipeline, but disable all components except for tok2vec and tagger +nlp = spacy.load("en_core_web_sm", enable=["tok2vec", "tagger"]) +# Has the same effect, as NER is already not part of enabled set of components +nlp = spacy.load("en_core_web_sm", enable=["tok2vec", "tagger"], disable=["ner"]) +# Will raise an error, as the sets of enabled and disabled components are conflicting +nlp = spacy.load("en_core_web_sm", enable=["ner"], disable=["ner"]) +``` + As of v3.0, the `disable` keyword argument specifies components to load but diff --git a/website/docs/usage/projects.md b/website/docs/usage/projects.md index 57d226913..566ae561b 100644 --- a/website/docs/usage/projects.md +++ b/website/docs/usage/projects.md @@ -94,9 +94,8 @@ also use any private repo you have access to with Git. Assets are data files your project needs – for example, the training and evaluation data or pretrained vectors and embeddings to initialize your model with. Each project template comes with a `project.yml` that defines the assets -to download and where to put them. The -[`spacy project assets`](/api/cli#project-assets) will fetch the project assets -for you: +to download and where to put them. The [`spacy project assets`](/api/cli#run) +will fetch the project assets for you: ```cli $ cd some_example_project @@ -108,6 +107,11 @@ even cloud storage such as GCS and S3. You can also fetch assets using git, by replacing the `url` string with a `git` block. spaCy will use Git's "sparse checkout" feature to avoid downloading the whole repository. +Sometimes your project configuration may include large assets that you don't +necessarily want to download when you run `spacy project assets`. That's why +assets can be marked as [`extra`](#data-assets-url) - by default, these assets +are not downloaded. If they should be, run `spacy project assets --extra`. + ### 3. Run a command {#run} > #### project.yml @@ -215,19 +219,27 @@ pipelines. > #### Tip: Multi-line YAML syntax for long values > -> YAML has [multi-line syntax](https://yaml-multiline.info/) that can be -> helpful for readability with longer values such as project descriptions or -> commands that take several arguments. +> YAML has [multi-line syntax](https://yaml-multiline.info/) that can be helpful +> for readability with longer values such as project descriptions or commands +> that take several arguments. ```yaml %%GITHUB_PROJECTS/pipelines/tagger_parser_ud/project.yml ``` +> #### Tip: Overriding variables on the CLI +> +> If you want to override one or more variables on the CLI and are not already specifying a +> project directory, you need to add `.` as a placeholder: +> +> ``` +> python -m spacy project run test . --vars.foo bar +> ``` | Section | Description | | --------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | | `title` | An optional project title used in `--help` message and [auto-generated docs](#custom-docs). | | `description` | An optional project description used in [auto-generated docs](#custom-docs). | -| `vars` | A dictionary of variables that can be referenced in paths, URLs and scripts, just like [`config.cfg` variables](/usage/training#config-interpolation). For example, `${vars.name}` will use the value of the variable `name`. Variables need to be defined in the section `vars`, but can be a nested dict, so you're able to reference `${vars.model.name}`. | +| `vars` | A dictionary of variables that can be referenced in paths, URLs and scripts and overriden on the CLI, just like [`config.cfg` variables](/usage/training#config-interpolation). For example, `${vars.name}` will use the value of the variable `name`. Variables need to be defined in the section `vars`, but can be a nested dict, so you're able to reference `${vars.model.name}`. | | `env` | A dictionary of variables, mapped to the names of environment variables that will be read in when running the project. For example, `${env.name}` will use the value of the environment variable defined as `name`. | | `directories` | An optional list of [directories](#project-files) that should be created in the project for assets, training outputs, metrics etc. spaCy will make sure that these directories always exist. | | `assets` | A list of assets that can be fetched with the [`project assets`](/api/cli#project-assets) command. `url` defines a URL or local path, `dest` is the destination file relative to the project directory, and an optional `checksum` ensures that an error is raised if the file's checksum doesn't match. Instead of `url`, you can also provide a `git` block with the keys `repo`, `branch` and `path`, to download from a Git repo. | @@ -261,8 +273,9 @@ dependencies to use certain protocols. > - dest: 'assets/training.spacy' > url: 'https://example.com/data.spacy' > checksum: '63373dd656daa1fd3043ce166a59474c' -> # Download from Google Cloud Storage bucket +> # Optional download from Google Cloud Storage bucket > - dest: 'assets/development.spacy' +> extra: True > url: 'gs://your-bucket/corpora' > checksum: '5113dc04e03f079525edd8df3f4f39e3' > ``` @@ -270,6 +283,7 @@ dependencies to use certain protocols. | Name | Description | | ------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | `dest` | The destination path to save the downloaded asset to (relative to the project directory), including the file name. | +| `extra` | Optional flag determining whether this asset is downloaded only if `spacy project assets` is run with `--extra`. `False` by default. | | `url` | The URL to download from, using the respective protocol. | | `checksum` | Optional checksum of the file. If provided, it will be used to verify that the file matches and downloads will be skipped if a local file with the same checksum already exists. | | `description` | Optional asset description, used in [auto-generated docs](#custom-docs). | @@ -294,12 +308,12 @@ files you need and not the whole repo. > description: 'The training data (5000 examples)' > ``` -| Name | Description | -| ------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `dest` | The destination path to save the downloaded asset to (relative to the project directory), including the file name. | +| Name | Description | +| ------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `dest` | The destination path to save the downloaded asset to (relative to the project directory), including the file name. | | `git` | `repo`: The URL of the repo to download from.
`path`: Path of the file or directory to download, relative to the repo root. "" specifies the root directory.
`branch`: The branch to download from. Defaults to `"master"`. | -| `checksum` | Optional checksum of the file. If provided, it will be used to verify that the file matches and downloads will be skipped if a local file with the same checksum already exists. | -| `description` | Optional asset description, used in [auto-generated docs](#custom-docs). | +| `checksum` | Optional checksum of the file. If provided, it will be used to verify that the file matches and downloads will be skipped if a local file with the same checksum already exists. | +| `description` | Optional asset description, used in [auto-generated docs](#custom-docs). | #### Working with private assets {#data-asets-private} diff --git a/website/docs/usage/rule-based-matching.md b/website/docs/usage/rule-based-matching.md index be9a56dc8..f096890cb 100644 --- a/website/docs/usage/rule-based-matching.md +++ b/website/docs/usage/rule-based-matching.md @@ -6,6 +6,7 @@ menu: - ['Phrase Matcher', 'phrasematcher'] - ['Dependency Matcher', 'dependencymatcher'] - ['Entity Ruler', 'entityruler'] + - ['Span Ruler', 'spanruler'] - ['Models & Rules', 'models-rules'] --- @@ -158,23 +159,23 @@ The available token pattern keys correspond to a number of [`Token` attributes](/api/token#attributes). The supported attributes for rule-based matching are: -| Attribute |  Description | -| ----------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `ORTH` | The exact verbatim text of a token. ~~str~~ | -| `TEXT` 2.1 | The exact verbatim text of a token. ~~str~~ | -| `NORM` | The normalized form of the token text. ~~str~~ | -| `LOWER` | The lowercase form of the token text. ~~str~~ | -|  `LENGTH` | The length of the token text. ~~int~~ | -|  `IS_ALPHA`, `IS_ASCII`, `IS_DIGIT` | Token text consists of alphabetic characters, ASCII characters, digits. ~~bool~~ | -|  `IS_LOWER`, `IS_UPPER`, `IS_TITLE` | Token text is in lowercase, uppercase, titlecase. ~~bool~~ | -|  `IS_PUNCT`, `IS_SPACE`, `IS_STOP` | Token is punctuation, whitespace, stop word. ~~bool~~ | -|  `IS_SENT_START` | Token is start of sentence. ~~bool~~ | -|  `LIKE_NUM`, `LIKE_URL`, `LIKE_EMAIL` | Token text resembles a number, URL, email. ~~bool~~ | -| `SPACY` | Token has a trailing space. ~~bool~~ | -|  `POS`, `TAG`, `MORPH`, `DEP`, `LEMMA`, `SHAPE` | The token's simple and extended part-of-speech tag, morphological analysis, dependency label, lemma, shape. Note that the values of these attributes are case-sensitive. For a list of available part-of-speech tags and dependency labels, see the [Annotation Specifications](/api/annotation). ~~str~~ | -| `ENT_TYPE` | The token's entity label. ~~str~~ | -| `_` 2.1 | Properties in [custom extension attributes](/usage/processing-pipelines#custom-components-attributes). ~~Dict[str, Any]~~ | -| `OP` | [Operator or quantifier](#quantifiers) to determine how often to match a token pattern. ~~str~~ | +| Attribute | Description | +| ---------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `ORTH` | The exact verbatim text of a token. ~~str~~ | +| `TEXT` 2.1 | The exact verbatim text of a token. ~~str~~ | +| `NORM` | The normalized form of the token text. ~~str~~ | +| `LOWER` | The lowercase form of the token text. ~~str~~ | +| `LENGTH` | The length of the token text. ~~int~~ | +| `IS_ALPHA`, `IS_ASCII`, `IS_DIGIT` | Token text consists of alphabetic characters, ASCII characters, digits. ~~bool~~ | +| `IS_LOWER`, `IS_UPPER`, `IS_TITLE` | Token text is in lowercase, uppercase, titlecase. ~~bool~~ | +| `IS_PUNCT`, `IS_SPACE`, `IS_STOP` | Token is punctuation, whitespace, stop word. ~~bool~~ | +| `IS_SENT_START` | Token is start of sentence. ~~bool~~ | +| `LIKE_NUM`, `LIKE_URL`, `LIKE_EMAIL` | Token text resembles a number, URL, email. ~~bool~~ | +| `SPACY` | Token has a trailing space. ~~bool~~ | +| `POS`, `TAG`, `MORPH`, `DEP`, `LEMMA`, `SHAPE` | The token's simple and extended part-of-speech tag, morphological analysis, dependency label, lemma, shape. Note that the values of these attributes are case-sensitive. For a list of available part-of-speech tags and dependency labels, see the [Annotation Specifications](/api/annotation). ~~str~~ | +| `ENT_TYPE` | The token's entity label. ~~str~~ | +| `_` 2.1 | Properties in [custom extension attributes](/usage/processing-pipelines#custom-components-attributes). ~~Dict[str, Any]~~ | +| `OP` | [Operator or quantifier](#quantifiers) to determine how often to match a token pattern. ~~str~~ | @@ -373,12 +374,16 @@ punctuation marks, or specify optional tokens. Note that there are no nested or scoped quantifiers – instead, you can build those behaviors with `on_match` callbacks. -| OP | Description | -| --- | ---------------------------------------------------------------- | -| `!` | Negate the pattern, by requiring it to match exactly 0 times. | -| `?` | Make the pattern optional, by allowing it to match 0 or 1 times. | -| `+` | Require the pattern to match 1 or more times. | -| `*` | Allow the pattern to match zero or more times. | +| OP | Description | +|---------|------------------------------------------------------------------------| +| `!` | Negate the pattern, by requiring it to match exactly 0 times. | +| `?` | Make the pattern optional, by allowing it to match 0 or 1 times. | +| `+` | Require the pattern to match 1 or more times. | +| `*` | Allow the pattern to match zero or more times. | +| `{n}` | Require the pattern to match exactly _n_ times. | +| `{n,m}` | Require the pattern to match at least _n_ but not more than _m_ times. | +| `{n,}` | Require the pattern to match at least _n_ times. | +| `{,m}` | Require the pattern to match at most _m_ times. | > #### Example > @@ -1446,6 +1451,108 @@ with nlp.select_pipes(enable="tagger"): ruler.add_patterns(patterns) ``` +## Rule-based span matching {#spanruler new="3.3.1"} + +The [`SpanRuler`](/api/spanruler) is a generalized version of the entity ruler +that lets you add spans to `doc.spans` or `doc.ents` based on pattern +dictionaries, which makes it easy to combine rule-based and statistical pipeline +components. + +### Span patterns {#spanruler-patterns} + +The [pattern format](#entityruler-patterns) is the same as for the entity ruler: + +1. **Phrase patterns** for exact string matches (string). + + ```python + {"label": "ORG", "pattern": "Apple"} + ``` + +2. **Token patterns** with one dictionary describing one token (list). + + ```python + {"label": "GPE", "pattern": [{"LOWER": "san"}, {"LOWER": "francisco"}]} + ``` + +### Using the span ruler {#spanruler-usage} + +The [`SpanRuler`](/api/spanruler) is a pipeline component that's typically added +via [`nlp.add_pipe`](/api/language#add_pipe). When the `nlp` object is called on +a text, it will find matches in the `doc` and add them as spans to +`doc.spans["ruler"]`, using the specified pattern label as the entity label. +Unlike in `doc.ents`, overlapping matches are allowed in `doc.spans`, so no +filtering is required, but optional filtering and sorting can be applied to the +spans before they're saved. + +```python +### {executable="true"} +import spacy + +nlp = spacy.blank("en") +ruler = nlp.add_pipe("span_ruler") +patterns = [{"label": "ORG", "pattern": "Apple"}, + {"label": "GPE", "pattern": [{"LOWER": "san"}, {"LOWER": "francisco"}]}] +ruler.add_patterns(patterns) + +doc = nlp("Apple is opening its first big office in San Francisco.") +print([(span.text, span.label_) for span in doc.spans["ruler"]]) +``` + +The span ruler is designed to integrate with spaCy's existing pipeline +components and enhance the [SpanCategorizer](/api/spancat) and +[EntityRecognizer](/api/entityrecognizer). The `overwrite` setting determines +whether the existing annotation in `doc.spans` or `doc.ents` is preserved. +Because overlapping entities are not allowed for `doc.ents`, the entities are +always filtered, using [`util.filter_spans`](/api/top-level#util.filter_spans) +by default. See the [`SpanRuler` API docs](/api/spanruler) for more information +about how to customize the sorting and filtering of matched spans. + +```python +### {executable="true"} +import spacy + +nlp = spacy.load("en_core_web_sm") +# only annotate doc.ents, not doc.spans +config = {"spans_key": None, "annotate_ents": True, "overwrite": False} +ruler = nlp.add_pipe("span_ruler", config=config) +patterns = [{"label": "ORG", "pattern": "MyCorp Inc."}] +ruler.add_patterns(patterns) + +doc = nlp("MyCorp Inc. is a company in the U.S.") +print([(ent.text, ent.label_) for ent in doc.ents]) +``` + +### Using pattern files {#spanruler-files} + +You can save patterns in a JSONL file (newline-delimited JSON) to load with +[`SpanRuler.initialize`](/api/spanruler#initialize) or +[`SpanRuler.add_patterns`](/api/spanruler#add_patterns). + +```json +### patterns.jsonl +{"label": "ORG", "pattern": "Apple"} +{"label": "GPE", "pattern": [{"LOWER": "san"}, {"LOWER": "francisco"}]} +``` + +```python +import srsly + +patterns = srsly.read_jsonl("patterns.jsonl") +ruler = nlp.add_pipe("span_ruler") +ruler.add_patterns(patterns) +``` + + + +Unlike the entity ruler, the span ruler cannot load patterns on initialization +with `SpanRuler(patterns=patterns)` or directly from a JSONL file path with +`SpanRuler.from_disk(jsonl_path)`. Patterns should be loaded from the JSONL file +separately and then added through +[`SpanRuler.initialize`](/api/spanruler#initialize]) or +[`SpanRuler.add_patterns`](/api/spanruler#add_patterns) as shown above. + + + ## Combining models and rules {#models-rules} You can combine statistical and rule-based components in a variety of ways. diff --git a/website/docs/usage/saving-loading.md b/website/docs/usage/saving-loading.md index af140e7a7..0fd713a49 100644 --- a/website/docs/usage/saving-loading.md +++ b/website/docs/usage/saving-loading.md @@ -203,11 +203,14 @@ the data to and from a JSON file. ```python ### {highlight="16-23,25-30"} +import json +from spacy import Language from spacy.util import ensure_path @Language.factory("my_component") class CustomComponent: - def __init__(self): + def __init__(self, nlp: Language, name: str = "my_component"): + self.name = name self.data = [] def __call__(self, doc): @@ -231,7 +234,7 @@ class CustomComponent: # This will receive the directory path + /my_component data_path = path / "data.json" with data_path.open("r", encoding="utf8") as f: - self.data = json.loads(f) + self.data = json.load(f) return self ``` diff --git a/website/docs/usage/v3-1.md b/website/docs/usage/v3-1.md index 1bac8fd81..2725cacb9 100644 --- a/website/docs/usage/v3-1.md +++ b/website/docs/usage/v3-1.md @@ -132,13 +132,13 @@ your own. > contributions for Catalan and to Kenneth Enevoldsen for Danish. For additional > Danish pipelines, check out [DaCy](https://github.com/KennethEnevoldsen/DaCy). -| Package | Language | UPOS | Parser LAS |  NER F | -| ------------------------------------------------- | -------- | ---: | ---------: | -----: | -| [`ca_core_news_sm`](/models/ca#ca_core_news_sm) | Catalan | 98.2 | 87.4 | 79.8 | -| [`ca_core_news_md`](/models/ca#ca_core_news_md) | Catalan | 98.3 | 88.2 | 84.0 | -| [`ca_core_news_lg`](/models/ca#ca_core_news_lg) | Catalan | 98.5 | 88.4 | 84.2 | -| [`ca_core_news_trf`](/models/ca#ca_core_news_trf) | Catalan | 98.9 | 93.0 | 91.2 | -| [`da_core_news_trf`](/models/da#da_core_news_trf) | Danish | 98.0 | 85.0 | 82.9 | +| Package | Language | UPOS | Parser LAS | NER F | +| ------------------------------------------------- | -------- | ---: | ---------: | ----: | +| [`ca_core_news_sm`](/models/ca#ca_core_news_sm) | Catalan | 98.2 | 87.4 | 79.8 | +| [`ca_core_news_md`](/models/ca#ca_core_news_md) | Catalan | 98.3 | 88.2 | 84.0 | +| [`ca_core_news_lg`](/models/ca#ca_core_news_lg) | Catalan | 98.5 | 88.4 | 84.2 | +| [`ca_core_news_trf`](/models/ca#ca_core_news_trf) | Catalan | 98.9 | 93.0 | 91.2 | +| [`da_core_news_trf`](/models/da#da_core_news_trf) | Danish | 98.0 | 85.0 | 82.9 | ### Resizable text classification architectures {#resizable-textcat} diff --git a/website/docs/usage/v3-4.md b/website/docs/usage/v3-4.md new file mode 100644 index 000000000..7cc4570d5 --- /dev/null +++ b/website/docs/usage/v3-4.md @@ -0,0 +1,143 @@ +--- +title: What's New in v3.4 +teaser: New features and how to upgrade +menu: + - ['New Features', 'features'] + - ['Upgrading Notes', 'upgrading'] +--- + +## New features {#features hidden="true"} + +spaCy v3.4 brings typing and speed improvements along with new vectors for +English CNN pipelines and new trained pipelines for Croatian. This release also +includes prebuilt linux aarch64 wheels for all spaCy dependencies distributed by +Explosion. + +### Typing improvements {#typing} + +spaCy v3.4 supports pydantic v1.9 and mypy 0.950+ through extensive updates to +types in Thinc v8.1. + +### Speed improvements {#speed} + +- For the parser, use C `saxpy`/`sgemm` provided by the `Ops` implementation in + order to use Accelerate through `thinc-apple-ops`. +- Improved speed of vector lookups. +- Improved speed for `Example.get_aligned_parse` and `Example.get_aligned`. + +## Additional features and improvements + +- Min/max `{n,m}` operator for `Matcher` patterns. +- Language updates: + - Improve tokenization for Cyrillic combining diacritics. + - Improve English tokenizer exceptions for contractions with + this/that/these/those. +- Updated `spacy project clone` to try both `main` and `master` branches by + default. +- Added confidence threshold for named entity linker. +- Improved handling of Typer optional default values for `init_config_cli`. +- Added cycle detection in parser projectivization methods. +- Added counts for NER labels in `debug data`. +- Support for adding NVTX ranges to `TrainablePipe` components. +- Support env variable `SPACY_NUM_BUILD_JOBS` to specify the number of build + jobs to run in parallel with `pip`. + +## Trained pipelines {#pipelines} + +### New trained pipelines {#new-pipelines} + +v3.4 introduces new CPU/CNN pipelines for Croatian, which use the trainable +lemmatizer and [floret vectors](https://github.com/explosion/floret). Due to the +use of [Bloom embeddings](https://explosion.ai/blog/bloom-embeddings) and +subwords, the pipelines have compact vectors with no out-of-vocabulary words. + +| Package | UPOS | Parser LAS | NER F | +| ----------------------------------------------- | ---: | ---------: | ----: | +| [`hr_core_news_sm`](/models/hr#hr_core_news_sm) | 96.6 | 77.5 | 76.1 | +| [`hr_core_news_md`](/models/hr#hr_core_news_md) | 97.3 | 80.1 | 81.8 | +| [`hr_core_news_lg`](/models/hr#hr_core_news_lg) | 97.5 | 80.4 | 83.0 | + +### Pipeline updates {#pipeline-updates} + +All CNN pipelines have been extended with whitespace augmentation. + +The English CNN pipelines have new word vectors: + +| Package | Model Version | TAG | Parser LAS | NER F | +| ----------------------------------------------- | ------------- | ---: | ---------: | ----: | +| [`en_core_news_md`](/models/en#en_core_news_md) | v3.3.0 | 97.3 | 90.1 | 84.6 | +| [`en_core_news_md`](/models/en#en_core_news_lg) | v3.4.0 | 97.2 | 90.3 | 85.5 | +| [`en_core_news_lg`](/models/en#en_core_news_md) | v3.3.0 | 97.4 | 90.1 | 85.3 | +| [`en_core_news_lg`](/models/en#en_core_news_lg) | v3.4.0 | 97.3 | 90.2 | 85.6 | + +## Notes about upgrading from v3.3 {#upgrading} + +### Doc.has_vector + +`Doc.has_vector` now matches `Token.has_vector` and `Span.has_vector`: it +returns `True` if at least one token in the doc has a vector rather than +checking only whether the vocab contains vectors. + +### Using trained pipelines with floret vectors + +If you're using a trained pipeline for Croatian, Finnish, Korean or Swedish with +new texts and working with `Doc` objects, you shouldn't notice any difference +between floret vectors and default vectors. + +If you use vectors for similarity comparisons, there are a few differences, +mainly because a floret pipeline doesn't include any kind of frequency-based +word list similar to the list of in-vocabulary vector keys with default vectors. + +- If your workflow iterates over the vector keys, you should use an external + word list instead: + + ```diff + - lexemes = [nlp.vocab[orth] for orth in nlp.vocab.vectors] + + lexemes = [nlp.vocab[word] for word in external_word_list] + ``` + +- `Vectors.most_similar` is not supported because there's no fixed list of + vectors to compare your vectors to. + +### Pipeline package version compatibility {#version-compat} + +> #### Using legacy implementations +> +> In spaCy v3, you'll still be able to load and reference legacy implementations +> via [`spacy-legacy`](https://github.com/explosion/spacy-legacy), even if the +> components or architectures change and newer versions are available in the +> core library. + +When you're loading a pipeline package trained with an earlier version of spaCy +v3, you will see a warning telling you that the pipeline may be incompatible. +This doesn't necessarily have to be true, but we recommend running your +pipelines against your test suite or evaluation data to make sure there are no +unexpected results. + +If you're using one of the [trained pipelines](/models) we provide, you should +run [`spacy download`](/api/cli#download) to update to the latest version. To +see an overview of all installed packages and their compatibility, you can run +[`spacy validate`](/api/cli#validate). + +If you've trained your own custom pipeline and you've confirmed that it's still +working as expected, you can update the spaCy version requirements in the +[`meta.json`](/api/data-formats#meta): + +```diff +- "spacy_version": ">=3.3.0,<3.4.0", ++ "spacy_version": ">=3.3.0,<3.5.0", +``` + +### Updating v3.3 configs + +To update a config from spaCy v3.3 with the new v3.4 settings, run +[`init fill-config`](/api/cli#init-fill-config): + +```cli +$ python -m spacy init fill-config config-v3.3.cfg config-v3.4.cfg +``` + +In many cases ([`spacy train`](/api/cli#train), +[`spacy.load`](/api/top-level#spacy.load)), the new defaults will be filled in +automatically, but you'll need to fill in the new settings to run +[`debug config`](/api/cli#debug) and [`debug data`](/api/cli#debug-data). diff --git a/website/docs/usage/v3.md b/website/docs/usage/v3.md index 980f06172..971779ed3 100644 --- a/website/docs/usage/v3.md +++ b/website/docs/usage/v3.md @@ -116,7 +116,7 @@ import Benchmarks from 'usage/\_benchmarks-models.md' > corpus that had both syntactic and entity annotations, so the transformer > models for those languages do not include NER. -| Package | Language | Transformer | Tagger | Parser |  NER | +| Package | Language | Transformer | Tagger | Parser | NER | | ------------------------------------------------ | -------- | --------------------------------------------------------------------------------------------- | -----: | -----: | ---: | | [`en_core_web_trf`](/models/en#en_core_web_trf) | English | [`roberta-base`](https://huggingface.co/roberta-base) | 97.8 | 95.2 | 89.9 | | [`de_dep_news_trf`](/models/de#de_dep_news_trf) | German | [`bert-base-german-cased`](https://huggingface.co/bert-base-german-cased) | 99.0 | 95.8 | - | @@ -856,9 +856,9 @@ attribute ruler before training using the `[initialize]` block of your config. ### Using Lexeme Tables -To use tables like `lexeme_prob` when training a model from scratch, you need -to add an entry to the `initialize` block in your config. Here's what that -looks like for the existing trained pipelines: +To use tables like `lexeme_prob` when training a model from scratch, you need to +add an entry to the `initialize` block in your config. Here's what that looks +like for the existing trained pipelines: ```ini [initialize.lookups] diff --git a/website/meta/languages.json b/website/meta/languages.json index 64ca7a082..6bc2309ed 100644 --- a/website/meta/languages.json +++ b/website/meta/languages.json @@ -162,7 +162,12 @@ { "code": "hr", "name": "Croatian", - "has_examples": true + "has_examples": true, + "models": [ + "hr_core_news_sm", + "hr_core_news_md", + "hr_core_news_lg" + ] }, { "code": "hsb", diff --git a/website/meta/sidebars.json b/website/meta/sidebars.json index cf3f1398e..1b743636c 100644 --- a/website/meta/sidebars.json +++ b/website/meta/sidebars.json @@ -12,7 +12,9 @@ { "text": "New in v3.0", "url": "/usage/v3" }, { "text": "New in v3.1", "url": "/usage/v3-1" }, { "text": "New in v3.2", "url": "/usage/v3-2" }, - { "text": "New in v3.3", "url": "/usage/v3-3" } + { "text": "New in v3.2", "url": "/usage/v3-2" }, + { "text": "New in v3.3", "url": "/usage/v3-3" }, + { "text": "New in v3.4", "url": "/usage/v3-4" } ] }, { @@ -103,6 +105,7 @@ { "text": "SentenceRecognizer", "url": "/api/sentencerecognizer" }, { "text": "Sentencizer", "url": "/api/sentencizer" }, { "text": "SpanCategorizer", "url": "/api/spancategorizer" }, + { "text": "SpanRuler", "url": "/api/spanruler" }, { "text": "Tagger", "url": "/api/tagger" }, { "text": "TextCategorizer", "url": "/api/textcategorizer" }, { "text": "Tok2Vec", "url": "/api/tok2vec" }, @@ -123,6 +126,7 @@ { "label": "Other", "items": [ + { "text": "Attributes", "url": "/api/attributes" }, { "text": "Corpus", "url": "/api/corpus" }, { "text": "KnowledgeBase", "url": "/api/kb" }, { "text": "Lookups", "url": "/api/lookups" }, diff --git a/website/meta/site.json b/website/meta/site.json index 97051011f..360a72178 100644 --- a/website/meta/site.json +++ b/website/meta/site.json @@ -28,7 +28,7 @@ }, "binderUrl": "explosion/spacy-io-binder", "binderBranch": "spacy.io", - "binderVersion": "3.0", + "binderVersion": "3.4", "sections": [ { "id": "usage", "title": "Usage Documentation", "theme": "blue" }, { "id": "models", "title": "Models Documentation", "theme": "blue" }, diff --git a/website/meta/universe.json b/website/meta/universe.json index e37c918ca..6c8caa6a6 100644 --- a/website/meta/universe.json +++ b/website/meta/universe.json @@ -1,5 +1,112 @@ { "resources": [ + { + "id": "concepcy", + "title": "concepCy", + "slogan": "A multilingual knowledge graph in spaCy", + "description": "A spaCy wrapper for ConceptNet, a freely-available semantic network designed to help computers understand the meaning of words.", + "github": "JulesBelveze/concepcy", + "pip": "concepcy", + "code_example": [ + "import spacy", + "import concepcy", + "", + "nlp = spacy.load('en_core_web_sm')", + "# Using default concepCy configuration", + "nlp.add_pipe('concepcy')", + "", + "doc = nlp('WHO is a lovely company')", + "", + "# Access all the 'RelatedTo' relations from the Doc", + "for word, relations in doc._.relatedto.items():", + " print(f'Word: {word}\n{relations}')", + "", + "# Access the 'RelatedTo' relations word by word", + "for token in doc:", + " print(f'Word: {token}\n{token._.relatedto}')" + ], + "category": ["pipeline"], + "image": "https://github.com/JulesBelveze/concepcy/blob/main/figures/concepcy.png", + "tags": ["semantic", "ConceptNet"], + "author": "Jules Belveze", + "author_links": { + "github": "JulesBelveze", + "website": "https://www.linkedin.com/in/jules-belveze/" + } + }, + { + "id": "spacyfishing", + "title": "spaCy fishing", + "slogan": "Named entity disambiguation and linking on Wikidata in spaCy with Entity-Fishing.", + "description": "A spaCy wrapper of Entity-Fishing for named entity disambiguation and linking against a Wikidata knowledge base.", + "github": "Lucaterre/spacyfishing", + "pip": "spacyfishing", + "code_example": [ + "import spacy", + "text = 'Victor Hugo and Honoré de Balzac are French writers who lived in Paris.'", + "nlp = spacy.load('en_core_web_sm')", + "nlp.add_pipe('entityfishing')", + "doc = nlp(text)", + "for span in doc.ents:", + " print((ent.text, ent.label_, ent._.kb_qid, ent._.url_wikidata, ent._.nerd_score))", + "# ('Victor Hugo', 'PERSON', 'Q535', 'https://www.wikidata.org/wiki/Q535', 0.972)", + "# ('Honoré de Balzac', 'PERSON', 'Q9711', 'https://www.wikidata.org/wiki/Q9711', 0.9724)", + "# ('French', 'NORP', 'Q121842', 'https://www.wikidata.org/wiki/Q121842', 0.3739)", + "# ('Paris', 'GPE', 'Q90', 'https://www.wikidata.org/wiki/Q90', 0.5652)", + "## Set parameter `extra_info` to `True` and check also span._.description, span._.src_description, span._.normal_term, span._.other_ids" + ], + "category": ["models", "pipeline"], + "image": "https://raw.githubusercontent.com/Lucaterre/spacyfishing/main/docs/spacyfishing-logo-resized.png", + "tags": ["NER", "NEL"], + "author": "Lucas Terriel", + "author_links": { + "twitter": "TerreLuca", + "github": "Lucaterre" + } + }, + { + "id": "aim-spacy", + "title": "Aim-spaCy", + "slogan": "Aim-spaCy is an Aim-based spaCy experiment tracker.", + "description": "Aim-spaCy helps to easily collect, store and explore training logs for spaCy, including: hyper-parameters, metrics and displaCy visualizations", + "github": "aimhubio/aim-spacy", + "pip": "aim-spacy", + "code_example": [ + "https://github.com/aimhubio/aim-spacy/tree/master/examples" + ], + "code_language": "python", + "url": "https://aimstack.io/spacy", + "thumb": "https://user-images.githubusercontent.com/13848158/172912427-ee9327ea-3cd8-47fa-8427-6c0d36cd831f.png", + "image": "https://user-images.githubusercontent.com/13848158/136364717-0939222c-55b6-44f0-ad32-d9ab749546e4.png", + "author": "AimStack", + "author_links": { + "twitter": "aimstackio", + "github": "aimhubio", + "website": "https://aimstack.io" + }, + "category": ["visualizers"], + "tags": ["experiment-tracking", "visualization"] + }, + { + "id": "spacy-report", + "title": "spacy-report", + "slogan": "Generates interactive reports for spaCy models.", + "description": "The goal of spacy-report is to offer static reports for spaCy models that help users make better decisions on how the models can be used.", + "github": "koaning/spacy-report", + "pip": "spacy-report", + "thumb": "https://github.com/koaning/spacy-report/raw/main/icon.png", + "image": "https://raw.githubusercontent.com/koaning/spacy-report/main/gif.gif", + "code_example": [ + "python -m spacy report textcat training/model-best/ corpus/train.spacy corpus/dev.spacy" + ], + "category": ["visualizers", "research"], + "author": "Vincent D. Warmerdam", + "author_links": { + "twitter": "fishnets88", + "github": "koaning", + "website": "https://koaning.io" + } + }, { "id": "scrubadub_spacy", "title": "scrubadub_spacy", @@ -12,7 +119,7 @@ "code_language": "python", "author": "Leap Beyond", "author_links": { - "github": "https://github.com/LeapBeyond", + "github": "LeapBeyond", "website": "https://leapbeyond.ai" }, "code_example": [ @@ -35,8 +142,8 @@ "code_language": "python", "author": "Peter Baumgartner", "author_links": { - "twitter" : "https://twitter.com/pmbaumgartner", - "github": "https://github.com/pmbaumgartner", + "twitter" : "pmbaumgartner", + "github": "pmbaumgartner", "website": "https://www.peterbaumgartner.com/" }, "code_example": [ @@ -55,8 +162,8 @@ "code_language": "python", "author": "Explosion", "author_links": { - "twitter" : "https://twitter.com/explosion_ai", - "github": "https://github.com/explosion", + "twitter" : "explosion_ai", + "github": "explosion", "website": "https://explosion.ai/" }, "code_example": [ @@ -298,6 +405,10 @@ "github": "SamEdwardes/spacytextblob", "pip": "spacytextblob", "code_example": [ + "# the following installations are required", + "# python -m textblob.download_corpora", + "# python -m spacy download en_core_web_sm", + "", "import spacy", "from spacytextblob.spacytextblob import SpacyTextBlob", "", @@ -468,6 +579,37 @@ "website": "https://koaning.io" } }, + { + "id": "bertopic", + "title": "BERTopic", + "slogan": "Leveraging BERT and c-TF-IDF to create easily interpretable topics.", + "description": "BERTopic is a topic modeling technique that leverages embedding models and c-TF-IDF to create dense clusters allowing for easily interpretable topics whilst keeping important words in the topic descriptions. BERTopic supports guided, (semi-) supervised, hierarchical, and dynamic topic modeling.", + "github": "maartengr/bertopic", + "pip": "bertopic", + "thumb": "https://i.imgur.com/Rx2LfBm.png", + "image": "https://raw.githubusercontent.com/MaartenGr/BERTopic/master/images/topic_visualization.gif", + "code_example": [ + "import spacy", + "from bertopic import BERTopic", + "from sklearn.datasets import fetch_20newsgroups", + "", + "docs = fetch_20newsgroups(subset='all', remove=('headers', 'footers', 'quotes'))['data']", + "nlp = spacy.load('en_core_web_md', exclude=['tagger', 'parser', 'ner', 'attribute_ruler', 'lemmatizer'])", + "", + "topic_model = BERTopic(embedding_model=nlp)", + "topics, probs = topic_model.fit_transform(docs)", + "", + "fig = topic_model.visualize_topics()", + "fig.show()" + ], + "category": ["visualizers", "training"], + "author": "Maarten Grootendorst", + "author_links": { + "twitter": "maartengr", + "github": "maartengr", + "website": "https://maartengrootendorst.com" + } + }, { "id": "tokenwiser", "title": "tokenwiser", @@ -524,8 +666,8 @@ "code_language": "python", "author": "Keith Rozario", "author_links": { - "twitter" : "https://twitter.com/keithrozario", - "github": "https://github.com/keithrozario", + "twitter" : "keithrozario", + "github": "keithrozario", "website": "https://www.keithrozario.com" }, "code_example": [ @@ -673,43 +815,6 @@ "category": ["standalone", "research"], "tags": ["pytorch"] }, - { - "id": "NeuroNER", - "title": "NeuroNER", - "slogan": "Named-entity recognition using neural networks", - "github": "Franck-Dernoncourt/NeuroNER", - "category": ["models"], - "pip": "pyneuroner[cpu]", - "code_example": [ - "from neuroner import neuromodel", - "nn = neuromodel.NeuroNER(train_model=False, use_pretrained_model=True)" - ], - "tags": ["standalone"] - }, - { - "id": "NLPre", - "title": "NLPre", - "slogan": "Natural Language Preprocessing Library for health data and more", - "github": "NIHOPA/NLPre", - "pip": "nlpre", - "code_example": [ - "from nlpre import titlecaps, dedash, identify_parenthetical_phrases", - "from nlpre import replace_acronyms, replace_from_dictionary", - "ABBR = identify_parenthetical_phrases()(text)", - "parsers = [dedash(), titlecaps(), replace_acronyms(ABBR),", - " replace_from_dictionary(prefix='MeSH_')]", - "for f in parsers:", - " text = f(text)", - "print(text)" - ], - "category": ["scientific", "biomedical"], - "author": "Travis Hoppe", - "author_links": { - "github": "thoppe", - "twitter": "metasemantic", - "website": "http://thoppe.github.io/" - } - }, { "id": "Chatterbot", "title": "Chatterbot", @@ -812,78 +917,6 @@ "github": "shigapov" } }, - { - "id": "spacy_hunspell", - "slogan": "Add spellchecking and spelling suggestions to your spaCy pipeline using Hunspell", - "description": "This package uses the [spaCy 2.0 extensions](https://spacy.io/usage/processing-pipelines#extensions) to add [Hunspell](http://hunspell.github.io) support for spellchecking.", - "github": "tokestermw/spacy_hunspell", - "pip": "spacy_hunspell", - "code_example": [ - "import spacy", - "from spacy_hunspell import spaCyHunSpell", - "", - "nlp = spacy.load('en_core_web_sm')", - "hunspell = spaCyHunSpell(nlp, 'mac')", - "nlp.add_pipe(hunspell)", - "doc = nlp('I can haz cheezeburger.')", - "haz = doc[2]", - "haz._.hunspell_spell # False", - "haz._.hunspell_suggest # ['ha', 'haze', 'hazy', 'has', 'hat', 'had', 'hag', 'ham', 'hap', 'hay', 'haw', 'ha z']" - ], - "author": "Motoki Wu", - "author_links": { - "github": "tokestermw", - "twitter": "plusepsilon" - }, - "category": ["pipeline"], - "tags": ["spellcheck"] - }, - { - "id": "spacy_grammar", - "slogan": "Language Tool style grammar handling with spaCy", - "description": "This packages leverages the [Matcher API](https://spacy.io/docs/usage/rule-based-matching) in spaCy to quickly match on spaCy tokens not dissimilar to regex. It reads a `grammar.yml` file to load up custom patterns and returns the results inside `Doc`, `Span`, and `Token`. It is extensible through adding rules to `grammar.yml` (though currently only the simple string matching is implemented).", - "github": "tokestermw/spacy_grammar", - "code_example": [ - "import spacy", - "from spacy_grammar.grammar import Grammar", - "", - "nlp = spacy.load('en')", - "grammar = Grammar(nlp)", - "nlp.add_pipe(grammar)", - "doc = nlp('I can haz cheeseburger.')", - "doc._.has_grammar_error # True" - ], - "author": "Motoki Wu", - "author_links": { - "github": "tokestermw", - "twitter": "plusepsilon" - }, - "category": ["pipeline"] - }, - { - "id": "spacy_kenlm", - "slogan": "KenLM extension for spaCy 2.0", - "github": "tokestermw/spacy_kenlm", - "pip": "spacy_kenlm", - "code_example": [ - "import spacy", - "from spacy_kenlm import spaCyKenLM", - "", - "nlp = spacy.load('en_core_web_sm')", - "spacy_kenlm = spaCyKenLM() # default model from test.arpa", - "nlp.add_pipe(spacy_kenlm)", - "doc = nlp('How are you?')", - "doc._.kenlm_score # doc score", - "doc[:2]._.kenlm_score # span score", - "doc[2]._.kenlm_score # token score" - ], - "author": "Motoki Wu", - "author_links": { - "github": "tokestermw", - "twitter": "plusepsilon" - }, - "category": ["pipeline"] - }, { "id": "spacy_readability", "slogan": "Add text readability meta data to Doc objects", @@ -952,34 +985,6 @@ }, "category": ["pipeline"] }, - { - "id": "spacy-lookup", - "slogan": "A powerful entity matcher for very large dictionaries, using the FlashText module", - "description": "spaCy v2.0 extension and pipeline component for adding Named Entities metadata to `Doc` objects. Detects Named Entities using dictionaries. The extension sets the custom `Doc`, `Token` and `Span` attributes `._.is_entity`, `._.entity_type`, `._.has_entities` and `._.entities`. Named Entities are matched using the python module `flashtext`, and looked up in the data provided by different dictionaries.", - "github": "mpuig/spacy-lookup", - "pip": "spacy-lookup", - "code_example": [ - "import spacy", - "from spacy_lookup import Entity", - "", - "nlp = spacy.load('en')", - "entity = Entity(keywords_list=['python', 'product manager', 'java platform'])", - "nlp.add_pipe(entity, last=True)", - "", - "doc = nlp(\"I am a product manager for a java and python.\")", - "assert doc._.has_entities == True", - "assert doc[0]._.is_entity == False", - "assert doc[3]._.entity_desc == 'product manager'", - "assert doc[3]._.is_entity == True", - "", - "print([(token.text, token._.canonical) for token in doc if token._.is_entity])" - ], - "author": "Marc Puig", - "author_links": { - "github": "mpuig" - }, - "category": ["pipeline"] - }, { "id": "spacy-iwnlp", "slogan": "German lemmatization with IWNLP", @@ -1181,6 +1186,46 @@ "category": ["pipeline", "models", "training"], "tags": ["pipeline", "models", "transformers"] }, + { + "id": "asent", + "title": "Asent", + "slogan": "Fast, flexible and transparent sentiment analysis", + "description": "Asent is a rule-based sentiment analysis library for Python made using spaCy. It is inspired by VADER, but uses a more modular ruleset, that allows the user to change e.g. the method for finding negations. Furthermore it includes visualisers to visualize the model predictions, making the model easily interpretable.", + "github": "kennethenevoldsen/asent", + "pip": "aseny", + "code_example": [ + "import spacy", + "import asent", + "", + "# load spacy pipeline", + "nlp = spacy.blank('en')", + "nlp.add_pipe('sentencizer')", + "", + "# add the rule-based sentiment model", + "nlp.add_pipe('asent_en_v1')", + "", + "# try an example", + "text = 'I am not very happy, but I am also not especially sad'", + "doc = nlp(text)", + "", + "# print polarity of document, scaled to be between -1, and 1", + "print(doc._.polarity)", + "# neg=0.0 neu=0.631 pos=0.369 compound=0.7526", + "", + "# Naturally, a simple score can be quite unsatisfying, thus Asent implements a series of visualizer to interpret the results:", + "asent.visualize(doc, style='prediction')", + " # or", + "asent.visualize(doc[:5], style='analysis')" + ], + "thumb": "https://github.com/KennethEnevoldsen/asent/raw/main/docs/img/logo_black_font.png?raw=true", + "author": "Kenneth Enevoldsen", + "author_links": { + "github": "KennethEnevoldsen", + "website": "https://www.kennethenevoldsen.com" + }, + "category": ["pipeline", "models"], + "tags": ["pipeline", "models", "sentiment"] + }, { "id": "textdescriptives", "title": "TextDescriptives", @@ -1246,21 +1291,6 @@ "github": "huggingface" } }, - { - "id": "spacy-vis", - "slogan": "A visualisation tool for spaCy using Hierplane", - "description": "A visualiser for spaCy annotations. This visualisation uses the [Hierplane](https://allenai.github.io/hierplane/) Library to render the dependency parse from spaCy's models. It also includes visualisation of entities and POS tags within nodes.", - "github": "DeNeutoy/spacy-vis", - "url": "http://spacyvis.allennlp.org/spacy-parser", - "thumb": "https://i.imgur.com/DAG9QFd.jpg", - "image": "https://raw.githubusercontent.com/DeNeutoy/spacy-vis/master/img/example.gif", - "author": "Mark Neumann", - "author_links": { - "twitter": "MarkNeumannnn", - "github": "DeNeutoy" - }, - "category": ["visualizers"] - }, { "id": "matcher-explorer", "title": "Rule-based Matcher Explorer", @@ -2248,7 +2278,7 @@ "author": "Daniel Whitenack & Chris Benson", "author_links": { "website": "https://changelog.com/practicalai", - "twitter": "https://twitter.com/PracticalAIFM" + "twitter": "PracticalAIFM" }, "category": ["podcasts"] }, @@ -2264,29 +2294,6 @@ "youtube": "8u57WSXVpmw", "category": ["videos"] }, - { - "id": "adam_qas", - "title": "ADAM: Question Answering System", - "slogan": "A question answering system that extracts answers from Wikipedia to questions posed in natural language.", - "github": "5hirish/adam_qas", - "pip": "qas", - "code_example": [ - "git clone https://github.com/5hirish/adam_qas.git", - "cd adam_qas", - "pip install -r requirements.txt", - "python -m qas.adam 'When was linux kernel version 4.0 released ?'" - ], - "code_language": "bash", - "thumb": "https://shirishkadam.files.wordpress.com/2018/04/mini_alleviate.png", - "author": "Shirish Kadam", - "author_links": { - "twitter": "5hirish", - "github": "5hirish", - "website": "https://shirishkadam.com/" - }, - "category": ["standalone"], - "tags": ["question-answering", "elasticsearch"] - }, { "id": "self-attentive-parser", "title": "Berkeley Neural Parser", @@ -2384,20 +2391,6 @@ "category": ["nonpython"], "tags": ["javascript"] }, - { - "id": "spacy-raspberry", - "title": "spacy-raspberry", - "slogan": "64bit Raspberry Pi image for spaCy and neuralcoref", - "github": "boehm-e/spacy-raspberry", - "thumb": "https://i.imgur.com/VCJMrE6.png", - "image": "https://raw.githubusercontent.com/boehm-e/spacy-raspberry/master/imgs/preview.png", - "author": "Erwan Boehm", - "author_links": { - "github": "boehm-e" - }, - "category": ["apis"], - "tags": ["raspberrypi"] - }, { "id": "spacy-wordnet", "title": "spacy-wordnet", @@ -2468,35 +2461,6 @@ "category": ["standalone", "pipeline"], "tags": ["linguistics", "computational linguistics", "conll", "conll-u"] }, - { - "id": "spacy-langdetect", - "title": "spacy-langdetect", - "slogan": "A fully customizable language detection pipeline for spaCy", - "description": "This module allows you to add language detection capabilites to your spaCy pipeline. Also supports custom language detectors!", - "pip": "spacy-langdetect", - "code_example": [ - "import spacy", - "from spacy_langdetect import LanguageDetector", - "nlp = spacy.load('en')", - "nlp.add_pipe(LanguageDetector(), name='language_detector', last=True)", - "text = 'This is an english text.'", - "doc = nlp(text)", - "# document level language detection. Think of it like average language of the document!", - "print(doc._.language)", - "# sentence level language detection", - "for sent in doc.sents:", - " print(sent, sent._.language)" - ], - "code_language": "python", - "author": "Abhijit Balaji", - "author_links": { - "github": "Abhijit-2592", - "website": "https://abhijit-2592.github.io/" - }, - "github": "Abhijit-2592/spacy-langdetect", - "category": ["pipeline"], - "tags": ["language-detection"] - }, { "id": "ludwig", "title": "Ludwig", @@ -2674,7 +2638,7 @@ " Add the courgette, garlic, red peppers and oregano and cook for 2–3 minutes.", " Later, add some oranges and chickens.\"\"\"", "", - "# use any model that has internal spacy embeddings", + "# use any model that has internal spacy embeddings", "nlp = spacy.load('en_core_web_lg')", "nlp.add_pipe(\"concise_concepts\", ", " config={\"data\": data}", @@ -2720,7 +2684,7 @@ " At that location, Nissin was founded.", " Many students survived by eating these noodles, but they don't even know him.\"\"\"", "", - "# use any model that has internal spacy embeddings", + "# use any model that has internal spacy embeddings", "nlp = spacy.load('en_core_web_sm')", "nlp.add_pipe(", " \"xx_coref\", config={\"chunk_size\": 2500, \"chunk_overlap\": 2, \"device\": 0})", @@ -2795,13 +2759,13 @@ "id": "holmes", "title": "Holmes", "slogan": "Information extraction from English and German texts based on predicate logic", - "github": "msg-systems/holmes-extractor", - "url": "https://github.com/msg-systems/holmes-extractor", - "description": "Holmes is a Python 3 library that supports a number of use cases involving information extraction from English and German texts, including chatbot, structural extraction, topic matching and supervised document classification. There is a [website demonstrating intelligent search based on topic matching](https://holmes-demo.xt.msg.team).", + "github": "explosion/holmes-extractor", + "url": "https://github.com/explosion/holmes-extractor", + "description": "Holmes is a Python 3 library that supports a number of use cases involving information extraction from English and German texts, including chatbot, structural extraction, topic matching and supervised document classification. There is a [website demonstrating intelligent search based on topic matching](https://holmes-demo.explosion.services).", "pip": "holmes-extractor", - "category": ["conversational", "standalone"], + "category": ["pipeline", "standalone"], "tags": ["chatbots", "text-processing"], - "thumb": "https://raw.githubusercontent.com/msg-systems/holmes-extractor/master/docs/holmes_thumbnail.png", + "thumb": "https://raw.githubusercontent.com/explosion/holmes-extractor/master/docs/holmes_thumbnail.png", "code_example": [ "import holmes_extractor as holmes", "holmes_manager = holmes.Manager(model='en_core_web_lg')", @@ -2903,7 +2867,7 @@ "doc = nlp(\"AE died in Princeton in 1955.\")", "", "print(doc._.clauses)", - "# Output:", + "# Output:", "# ", "", "propositions = doc._.clauses[0].to_propositions(as_text=True)", @@ -2995,35 +2959,6 @@ ], "author": "Stefan Daniel Dumitrescu, Andrei-Marius Avram" }, - { - "id": "num_fh", - "title": "Numeric Fused-Head", - "slogan": "Numeric Fused-Head Identificaiton and Resolution in English", - "description": "This package provide a wrapper for the Numeric Fused-Head in English. It provides another information layer on numbers that refer to another entity which is not obvious from the syntactic tree.", - "github": "yanaiela/num_fh", - "pip": "num_fh", - "category": ["pipeline", "research"], - "code_example": [ - "import spacy", - "from num_fh import NFH", - "nlp = spacy.load('en_core_web_sm')", - "nfh = NFH(nlp)", - "nlp.add_pipe(nfh, first=False)", - "doc = nlp(\"I told you two, that only one of them is the one who will get 2 or 3 icecreams\")", - "", - "assert doc[16]._.is_nfh == True", - "assert doc[18]._.is_nfh == False", - "assert doc[3]._.is_deter_nfh == True", - "assert doc[16]._.is_deter_nfh == False", - "assert len(doc._.nfh) == 4" - ], - "author": "Yanai Elazar", - "author_links": { - "github": "yanaiela", - "twitter": "yanaiela", - "website": "https://yanaiela.github.io" - } - }, { "id": "Healthsea", "title": "Healthsea", @@ -3114,6 +3049,7 @@ "from pysbd.utils import PySBDFactory", "", "nlp = spacy.blank('en')", + "# Caution: works with spaCy<=2.x.x", "nlp.add_pipe(PySBDFactory(nlp))", "", "doc = nlp('My name is Jonas E. Smith. Please turn to p. 55.')", @@ -3697,7 +3633,7 @@ "", "#Lexico Semantic (LxSem) Features", "TTRF = LingFeat.TTRF_() #Type Token Ratio Features", - "VarF = LingFeat.VarF_() #Noun/Verb/Adj/Adv Variation Features", + "VarF = LingFeat.VarF_() #Noun/Verb/Adj/Adv Variation Features", "PsyF = LingFeat.PsyF_() #Psycholinguistic Difficulty of Words (AoA Kuperman)", "WoLF = LingFeat.WorF_() #Word Familiarity from Frequency Count (SubtlexUS)", "", @@ -4033,6 +3969,21 @@ }, "category": ["biomedical", "scientific", "research", "pipeline"], "tags": ["clinical"] + }, + { + "id": "sent-pattern", + "title": "English Interpretation Sentence Pattern", + "slogan": "English interpretation for accurate translation from English to Japanese", + "description": "This package categorizes English sentences into one of five basic sentence patterns and identifies the subject, verb, object, and other components. The five basic sentence patterns are based on C. T. Onions's Advanced English Syntax and are frequently used when teaching English in Japan.", + "github": "lll-lll-lll-lll/sent-pattern", + "pip": "sent-pattern", + "author": "Shunpei Nakayama", + "author_links": { + "twitter": "ExZ79575296", + "github": "lll-lll-lll-lll" + }, + "category": ["pipeline"], + "tags": ["interpretation", "ja"] } ], diff --git a/website/src/templates/index.js b/website/src/templates/index.js index bdbdbd431..a0ba4503e 100644 --- a/website/src/templates/index.js +++ b/website/src/templates/index.js @@ -120,8 +120,8 @@ const AlertSpace = ({ nightly, legacy }) => { } const navAlert = ( - - 💥 Out now: spaCy v3.3 + + 💥 Out now: spaCy v3.4 ) diff --git a/website/src/templates/universe.js b/website/src/templates/universe.js index 10f2520d9..48ffa3add 100644 --- a/website/src/templates/universe.js +++ b/website/src/templates/universe.js @@ -142,10 +142,10 @@ const UniverseContent = ({ content = [], categories, theme, pageContext, mdxComp The Universe database is open-source and collected in a simple JSON file. For more details on the formats and available fields, see the documentation. Looking for inspiration your own spaCy plugin or extension? Check out the - - project idea + + project idea - label on the issue tracker. + section in Discussions.

diff --git a/website/src/widgets/quickstart-install.js b/website/src/widgets/quickstart-install.js index fbf043c7d..61c0678dd 100644 --- a/website/src/widgets/quickstart-install.js +++ b/website/src/widgets/quickstart-install.js @@ -23,6 +23,9 @@ const CUDA = { '11.2': 'cuda112', '11.3': 'cuda113', '11.4': 'cuda114', + '11.5': 'cuda115', + '11.6': 'cuda116', + '11.7': 'cuda117', } const LANG_EXTRAS = ['ja'] // only for languages with models @@ -48,7 +51,7 @@ const QuickstartInstall = ({ id, title }) => { const modelExtras = train ? selectedModels.filter(m => LANG_EXTRAS.includes(m)) : [] const apple = os === 'mac' && platform === 'arm' const pipExtras = [ - hardware === 'gpu' && cuda, + (hardware === 'gpu' && (platform !== 'arm' || os === 'linux')) && cuda, train && 'transformers', train && 'lookups', apple && 'apple',