diff --git a/.github/azure-steps.yml b/.github/azure-steps.yml index d0db75f9a..20d4582cb 100644 --- a/.github/azure-steps.yml +++ b/.github/azure-steps.yml @@ -52,51 +52,56 @@ steps: python -W error -c "import spacy" displayName: "Test import" -# - script: | -# python -m spacy download ca_core_news_sm -# python -m spacy download ca_core_news_md -# python -c "import spacy; nlp=spacy.load('ca_core_news_sm'); doc=nlp('test')" -# displayName: 'Test download CLI' -# condition: eq(variables['python_version'], '3.8') -# -# - script: | -# python -W error -c "import ca_core_news_sm; nlp = ca_core_news_sm.load(); doc=nlp('test')" -# displayName: 'Test no warnings on load (#11713)' -# condition: eq(variables['python_version'], '3.8') + - script: | + python -m spacy download ca_core_news_sm + python -m spacy download ca_core_news_md + python -c "import spacy; nlp=spacy.load('ca_core_news_sm'); doc=nlp('test')" + displayName: 'Test download CLI' + condition: eq(variables['python_version'], '3.9') + + - script: | + python -W error -m spacy info ca_core_news_sm | grep -q download_url + displayName: 'Test download_url in info CLI' + condition: eq(variables['python_version'], '3.9') + + - script: | + python -W error -c "import ca_core_news_sm; nlp = ca_core_news_sm.load(); doc=nlp('test')" + displayName: 'Test no warnings on load (#11713)' + condition: eq(variables['python_version'], '3.9') - script: | python -m spacy convert extra/example_data/ner_example_data/ner-token-per-line-conll2003.json . displayName: 'Test convert CLI' - condition: eq(variables['python_version'], '3.8') + condition: eq(variables['python_version'], '3.9') - script: | python -m spacy init config -p ner -l ca ner.cfg python -m spacy debug config ner.cfg --paths.train ner-token-per-line-conll2003.spacy --paths.dev ner-token-per-line-conll2003.spacy displayName: 'Test debug config CLI' - condition: eq(variables['python_version'], '3.8') + condition: eq(variables['python_version'], '3.9') - script: | # will have errors due to sparse data, check for summary in output python -m spacy debug data ner.cfg --paths.train ner-token-per-line-conll2003.spacy --paths.dev ner-token-per-line-conll2003.spacy | grep -q Summary displayName: 'Test debug data CLI' - condition: eq(variables['python_version'], '3.8') + condition: eq(variables['python_version'], '3.9') - script: | python -m spacy train ner.cfg --paths.train ner-token-per-line-conll2003.spacy --paths.dev ner-token-per-line-conll2003.spacy --training.max_steps 10 --gpu-id -1 displayName: 'Test train CLI' - condition: eq(variables['python_version'], '3.8') + condition: eq(variables['python_version'], '3.9') -# - script: | -# python -c "import spacy; config = spacy.util.load_config('ner.cfg'); config['components']['ner'] = {'source': 'ca_core_news_sm'}; config.to_disk('ner_source_sm.cfg')" -# PYTHONWARNINGS="error,ignore::DeprecationWarning" python -m spacy assemble ner_source_sm.cfg output_dir -# displayName: 'Test assemble CLI' -# condition: eq(variables['python_version'], '3.8') -# -# - script: | -# python -c "import spacy; config = spacy.util.load_config('ner.cfg'); config['components']['ner'] = {'source': 'ca_core_news_md'}; config.to_disk('ner_source_md.cfg')" -# python -m spacy assemble ner_source_md.cfg output_dir 2>&1 | grep -q W113 -# displayName: 'Test assemble CLI vectors warning' -# condition: eq(variables['python_version'], '3.8') + - script: | + python -c "import spacy; config = spacy.util.load_config('ner.cfg'); config['components']['ner'] = {'source': 'ca_core_news_sm'}; config.to_disk('ner_source_sm.cfg')" + PYTHONWARNINGS="error,ignore::DeprecationWarning" python -m spacy assemble ner_source_sm.cfg output_dir + displayName: 'Test assemble CLI' + condition: eq(variables['python_version'], '3.9') + + - script: | + python -c "import spacy; config = spacy.util.load_config('ner.cfg'); config['components']['ner'] = {'source': 'ca_core_news_md'}; config.to_disk('ner_source_md.cfg')" + python -m spacy assemble ner_source_md.cfg output_dir 2>&1 | grep -q W113 + displayName: 'Test assemble CLI vectors warning' + condition: eq(variables['python_version'], '3.9') - script: | python -m pip install -U -r requirements.txt @@ -111,9 +116,3 @@ steps: python -m pytest --pyargs spacy displayName: "Run CPU tests with thinc-apple-ops" condition: and(startsWith(variables['imageName'], 'macos'), eq(variables['python.version'], '3.11')) - - - script: | - python .github/validate_universe_json.py website/meta/universe.json - displayName: 'Test website/meta/universe.json' - condition: eq(variables['python_version'], '3.8') - diff --git a/.github/workflows/autoblack.yml b/.github/workflows/autoblack.yml deleted file mode 100644 index 70882c3cc..000000000 --- a/.github/workflows/autoblack.yml +++ /dev/null @@ -1,45 +0,0 @@ -# GitHub Action that uses Black to reformat all Python code and submits a PR -# in regular intervals. Inspired by: https://github.com/cclauss/autoblack - -name: autoblack -on: - workflow_dispatch: # allow manual trigger - schedule: - - cron: '0 8 * * 5' # every Friday at 8am UTC - -jobs: - autoblack: - if: github.repository_owner == 'explosion' - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - with: - ref: ${{ github.head_ref }} - - uses: actions/setup-python@v4 - - run: pip install black - - name: Auto-format code if needed - run: black spacy - # We can't run black --check here because that returns a non-zero excit - # code and makes GitHub think the action failed - - name: Check for modified files - id: git-check - run: echo modified=$(if git diff-index --quiet HEAD --; then echo "false"; else echo "true"; fi) >> $GITHUB_OUTPUT - - - name: Create Pull Request - if: steps.git-check.outputs.modified == 'true' - uses: peter-evans/create-pull-request@v4 - with: - title: Auto-format code with black - labels: meta - commit-message: Auto-format code with black - committer: GitHub - author: explosion-bot - body: _This PR is auto-generated._ - branch: autoblack - delete-branch: true - draft: false - - name: Check outputs - if: steps.git-check.outputs.modified == 'true' - run: | - echo "Pull Request Number - ${{ steps.cpr.outputs.pull-request-number }}" - echo "Pull Request URL - ${{ steps.cpr.outputs.pull-request-url }}" diff --git a/.github/workflows/explosionbot.yml b/.github/workflows/explosionbot.yml index 6b472cd12..910cfdc40 100644 --- a/.github/workflows/explosionbot.yml +++ b/.github/workflows/explosionbot.yml @@ -8,6 +8,7 @@ on: jobs: explosion-bot: + if: github.repository_owner == 'explosion' runs-on: ubuntu-latest steps: - name: Dump GitHub context diff --git a/.github/workflows/issue-manager.yml b/.github/workflows/issue-manager.yml index 8f3a151ea..6c7d7d5a6 100644 --- a/.github/workflows/issue-manager.yml +++ b/.github/workflows/issue-manager.yml @@ -13,6 +13,7 @@ on: jobs: issue-manager: + if: github.repository_owner == 'explosion' runs-on: ubuntu-latest steps: - uses: tiangolo/issue-manager@0.4.0 diff --git a/.github/workflows/lock.yml b/.github/workflows/lock.yml index 794adee85..6c3985a93 100644 --- a/.github/workflows/lock.yml +++ b/.github/workflows/lock.yml @@ -13,6 +13,7 @@ concurrency: jobs: action: + if: github.repository_owner == 'explosion' runs-on: ubuntu-latest steps: - uses: dessant/lock-threads@v4 diff --git a/.github/workflows/spacy_universe_alert.yml b/.github/workflows/spacy_universe_alert.yml index 837aaeb33..33851fbcc 100644 --- a/.github/workflows/spacy_universe_alert.yml +++ b/.github/workflows/spacy_universe_alert.yml @@ -7,6 +7,7 @@ on: jobs: build: + if: github.repository_owner == 'explosion' runs-on: ubuntu-latest steps: diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 000000000..f226057c9 --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,173 @@ +name: tests + +on: + push: + branches-ignore: + - "spacy.io" + - "nightly.spacy.io" + - "v2.spacy.io" + paths-ignore: + - "*.md" + - "*.mdx" + - "website/**" + - ".github/workflows/**" + pull_request: + types: [opened, synchronize, reopened, edited] + paths-ignore: + - "*.md" + - "*.mdx" + - "website/**" + +jobs: + validate: + name: Validate + if: github.repository_owner == 'explosion' + runs-on: ubuntu-latest + steps: + - name: Check out repo + uses: actions/checkout@v3 + + - name: Configure Python version + uses: actions/setup-python@v4 + with: + python-version: "3.7" + architecture: x64 + + - name: black + run: | + python -m pip install black -c requirements.txt + python -m black spacy --check + - name: flake8 + run: | + python -m pip install flake8==5.0.4 + python -m flake8 spacy --count --select=E901,E999,F821,F822,F823,W605 --show-source --statistics + tests: + name: Test + needs: Validate + strategy: + fail-fast: true + matrix: + os: [ubuntu-latest, windows-latest, macos-latest] + python_version: ["3.11"] + include: + - os: ubuntu-20.04 + python_version: "3.6" + - os: windows-latest + python_version: "3.7" + - os: macos-latest + python_version: "3.8" + - os: ubuntu-latest + python_version: "3.9" + - os: windows-latest + python_version: "3.10" + + runs-on: ${{ matrix.os }} + + steps: + - name: Check out repo + uses: actions/checkout@v3 + + - name: Configure Python version + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python_version }} + architecture: x64 + + - name: Install dependencies + run: | + python -m pip install -U build pip setuptools + python -m pip install -U -r requirements.txt + + - name: Build sdist + run: | + python -m build --sdist + + - name: Run mypy + run: | + python -m mypy spacy + if: matrix.python_version != '3.6' + + - name: Delete source directory and .egg-info + run: | + rm -rf spacy *.egg-info + shell: bash + + - name: Uninstall all packages + run: | + python -m pip freeze + python -m pip freeze --exclude pywin32 > installed.txt + python -m pip uninstall -y -r installed.txt + + - name: Install from sdist + run: | + SDIST=$(python -c "import os;print(os.listdir('./dist')[-1])" 2>&1) + SPACY_NUM_BUILD_JOBS=2 python -m pip install dist/$SDIST + shell: bash + + - name: Test import + run: python -W error -c "import spacy" + + - name: "Test download CLI" + run: | + python -m spacy download ca_core_news_sm + python -m spacy download ca_core_news_md + python -c "import spacy; nlp=spacy.load('ca_core_news_sm'); doc=nlp('test')" + if: matrix.python_version == '3.9' + + - name: "Test download_url in info CLI" + run: | + python -W error -m spacy info ca_core_news_sm | grep -q download_url + if: matrix.python_version == '3.9' + + - name: "Test no warnings on load (#11713)" + run: | + python -W error -c "import ca_core_news_sm; nlp = ca_core_news_sm.load(); doc=nlp('test')" + if: matrix.python_version == '3.9' + + - name: "Test convert CLI" + run: | + python -m spacy convert extra/example_data/ner_example_data/ner-token-per-line-conll2003.json . + if: matrix.python_version == '3.9' + + - name: "Test debug config CLI" + run: | + python -m spacy init config -p ner -l ca ner.cfg + python -m spacy debug config ner.cfg --paths.train ner-token-per-line-conll2003.spacy --paths.dev ner-token-per-line-conll2003.spacy + if: matrix.python_version == '3.9' + + - name: "Test debug data CLI" + run: | + # will have errors due to sparse data, check for summary in output + python -m spacy debug data ner.cfg --paths.train ner-token-per-line-conll2003.spacy --paths.dev ner-token-per-line-conll2003.spacy | grep -q Summary + if: matrix.python_version == '3.9' + + - name: "Test train CLI" + run: | + python -m spacy train ner.cfg --paths.train ner-token-per-line-conll2003.spacy --paths.dev ner-token-per-line-conll2003.spacy --training.max_steps 10 --gpu-id -1 + if: matrix.python_version == '3.9' + + - name: "Test assemble CLI" + run: | + python -c "import spacy; config = spacy.util.load_config('ner.cfg'); config['components']['ner'] = {'source': 'ca_core_news_sm'}; config.to_disk('ner_source_sm.cfg')" + PYTHONWARNINGS="error,ignore::DeprecationWarning" python -m spacy assemble ner_source_sm.cfg output_dir + if: matrix.python_version == '3.9' + + - name: "Test assemble CLI vectors warning" + run: | + python -c "import spacy; config = spacy.util.load_config('ner.cfg'); config['components']['ner'] = {'source': 'ca_core_news_md'}; config.to_disk('ner_source_md.cfg')" + python -m spacy assemble ner_source_md.cfg output_dir 2>&1 | grep -q W113 + if: matrix.python_version == '3.9' + + - name: "Install test requirements" + run: | + python -m pip install -U -r requirements.txt + + - name: "Run CPU tests" + run: | + python -m pytest --pyargs spacy -W error + + - name: "Run CPU tests with thinc-apple-ops" + run: | + python -m pip install 'spacy[apple]' + python -m pytest --pyargs spacy + if: startsWith(matrix.os, 'macos') && matrix.python_version == '3.11' diff --git a/.github/workflows/universe_validation.yml b/.github/workflows/universe_validation.yml new file mode 100644 index 000000000..a1e3253a9 --- /dev/null +++ b/.github/workflows/universe_validation.yml @@ -0,0 +1,33 @@ +name: universe validation + +on: + push: + branches-ignore: + - "spacy.io" + - "nightly.spacy.io" + - "v2.spacy.io" + paths: + - "website/meta/universe.json" + pull_request: + types: [opened, synchronize, reopened, edited] + paths: + - "website/meta/universe.json" + +jobs: + validate: + name: Validate + if: github.repository_owner == 'explosion' + runs-on: ubuntu-latest + steps: + - name: Check out repo + uses: actions/checkout@v3 + + - name: Configure Python version + uses: actions/setup-python@v4 + with: + python-version: "3.7" + architecture: x64 + + - name: Validate website/meta/universe.json + run: | + python .github/validate_universe_json.py website/meta/universe.json diff --git a/.gitignore b/.gitignore index ac333f958..af75a4d47 100644 --- a/.gitignore +++ b/.gitignore @@ -10,16 +10,6 @@ spacy/tests/package/setup.cfg spacy/tests/package/pyproject.toml spacy/tests/package/requirements.txt -# Website -website/.cache/ -website/public/ -website/node_modules -website/.npm -website/logs -*.log -npm-debug.log* -quickstart-training-generator.js - # Cython / C extensions cythonize.json spacy/*.html diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1f396bd71..f6f6dab59 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -173,6 +173,11 @@ formatting and [`flake8`](http://flake8.pycqa.org/en/latest/) for linting its Python modules. If you've built spaCy from source, you'll already have both tools installed. +As a general rule of thumb, we use f-strings for any formatting of strings. +One exception are calls to Python's `logging` functionality. +To avoid unnecessary string conversions in these cases, we use string formatting +templates with `%s` and `%d` etc. + **⚠️ Note that formatting and linting is currently only possible for Python modules in `.py` files, not Cython modules in `.pyx` and `.pxd` files.** diff --git a/README.md b/README.md index 195424551..36a015caf 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,10 @@ production-ready [**training system**](https://spacy.io/usage/training) and easy model packaging, deployment and workflow management. spaCy is commercial open-source software, released under the [MIT license](https://github.com/explosion/spaCy/blob/master/LICENSE). -💫 **Version 3.4 out now!** +💥 **We'd love to hear more about your experience with spaCy!** +[Fill out our survey here.](https://form.typeform.com/to/aMel9q9f) + +💫 **Version 3.5 out now!** [Check out the release notes here.](https://github.com/explosion/spaCy/releases) [![Azure Pipelines](https://img.shields.io/azure-devops/build/explosion-ai/public/8/master.svg?logo=azure-pipelines&style=flat-square&label=build)](https://dev.azure.com/explosion-ai/public/_build?definitionId=8) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 0f7ea91f9..83c57a164 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -11,18 +11,28 @@ trigger: exclude: - "website/*" - "*.md" + - "*.mdx" - ".github/workflows/*" pr: paths: exclude: - "*.md" + - "*.mdx" - "website/docs/*" - "website/src/*" + - "website/meta/*.tsx" + - "website/meta/*.mjs" + - "website/meta/languages.json" + - "website/meta/site.json" + - "website/meta/sidebars.json" + - "website/meta/type-annotations.json" + - "website/pages/*" - ".github/workflows/*" jobs: - # Perform basic checks for most important errors (syntax etc.) Uses the config - # defined in .flake8 and overwrites the selected codes. + # Check formatting and linting. Perform basic checks for most important errors + # (syntax etc.) Uses the config defined in setup.cfg and overwrites the + # selected codes. - job: "Validate" pool: vmImage: "ubuntu-latest" @@ -30,10 +40,17 @@ jobs: - task: UsePythonVersion@0 inputs: versionSpec: "3.7" + - script: | + pip install black -c requirements.txt + python -m black spacy --check + displayName: "black" - script: | pip install flake8==5.0.4 python -m flake8 spacy --count --select=E901,E999,F821,F822,F823,W605 --show-source --statistics displayName: "flake8" + - script: | + python .github/validate_universe_json.py website/meta/universe.json + displayName: 'Validate website/meta/universe.json' - job: "Test" dependsOn: "Validate" diff --git a/build-constraints.txt b/build-constraints.txt index 956973abf..c1e82f1b0 100644 --- a/build-constraints.txt +++ b/build-constraints.txt @@ -5,4 +5,5 @@ numpy==1.17.3; python_version=='3.8' and platform_machine!='aarch64' numpy==1.19.2; python_version=='3.8' and platform_machine=='aarch64' numpy==1.19.3; python_version=='3.9' numpy==1.21.3; python_version=='3.10' -numpy; python_version>='3.11' +numpy==1.23.2; python_version=='3.11' +numpy; python_version>='3.12' diff --git a/pyproject.toml b/pyproject.toml index 7abd7a96f..9cd96ac2d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ requires = [ "cymem>=2.0.2,<2.1.0", "preshed>=3.0.2,<3.1.0", "murmurhash>=0.28.0,<1.1.0", - "thinc>=8.1.0,<8.2.0", + "thinc>=8.1.8,<8.2.0", "numpy>=1.15.0", ] build-backend = "setuptools.build_meta" diff --git a/requirements.txt b/requirements.txt index 0440835f2..63e03d558 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,9 +1,9 @@ # Our libraries -spacy-legacy>=3.0.10,<3.1.0 +spacy-legacy>=3.0.11,<3.1.0 spacy-loggers>=1.0.0,<2.0.0 cymem>=2.0.2,<2.1.0 preshed>=3.0.2,<3.1.0 -thinc>=8.1.0,<8.2.0 +thinc>=8.1.8,<8.2.0 ml_datasets>=0.2.0,<0.3.0 murmurhash>=0.28.0,<1.1.0 wasabi>=0.9.1,<1.2.0 @@ -22,7 +22,7 @@ langcodes>=3.2.0,<4.0.0 # Official Python utilities setuptools packaging>=20.0 -typing_extensions>=3.7.4.1,<4.2.0; python_version < "3.8" +typing_extensions>=3.7.4.1,<4.5.0; python_version < "3.8" # Development dependencies pre-commit>=2.13.0 cython>=0.25,<3.0 @@ -31,10 +31,10 @@ pytest-timeout>=1.3.0,<2.0.0 mock>=2.0.0,<3.0.0 flake8>=3.8.0,<6.0.0 hypothesis>=3.27.0,<7.0.0 -mypy>=0.990,<0.1000; platform_machine != "aarch64" and python_version >= "3.7" +mypy>=0.990,<1.1.0; platform_machine != "aarch64" and python_version >= "3.7" types-dataclasses>=0.1.3; python_version < "3.7" types-mock>=0.1.1 types-setuptools>=57.0.0 types-requests types-setuptools>=57.0.0 -black>=22.0,<23.0 +black==22.3.0 diff --git a/setup.cfg b/setup.cfg index cf6e6f84b..eea557337 100644 --- a/setup.cfg +++ b/setup.cfg @@ -22,6 +22,7 @@ classifiers = Programming Language :: Python :: 3.8 Programming Language :: Python :: 3.9 Programming Language :: Python :: 3.10 + Programming Language :: Python :: 3.11 Topic :: Scientific/Engineering project_urls = Release notes = https://github.com/explosion/spaCy/releases @@ -38,15 +39,15 @@ setup_requires = cymem>=2.0.2,<2.1.0 preshed>=3.0.2,<3.1.0 murmurhash>=0.28.0,<1.1.0 - thinc>=8.1.0,<8.2.0 + thinc>=8.1.8,<8.2.0 install_requires = # Our libraries - spacy-legacy>=3.0.10,<3.1.0 + spacy-legacy>=3.0.11,<3.1.0 spacy-loggers>=1.0.0,<2.0.0 murmurhash>=0.28.0,<1.1.0 cymem>=2.0.2,<2.1.0 preshed>=3.0.2,<3.1.0 - thinc>=8.1.0,<8.2.0 + thinc>=8.1.8,<8.2.0 wasabi>=0.9.1,<1.2.0 srsly>=2.4.3,<3.0.0 catalogue>=2.0.6,<2.1.0 @@ -62,7 +63,7 @@ install_requires = # Official Python utilities setuptools packaging>=20.0 - typing_extensions>=3.7.4,<4.2.0; python_version < "3.8" + typing_extensions>=3.7.4.1,<4.5.0; python_version < "3.8" langcodes>=3.2.0,<4.0.0 [options.entry_points] @@ -73,45 +74,45 @@ console_scripts = lookups = spacy_lookups_data>=1.0.3,<1.1.0 transformers = - spacy_transformers>=1.1.2,<1.2.0 + spacy_transformers>=1.1.2,<1.3.0 ray = spacy_ray>=0.1.0,<1.0.0 cuda = - cupy>=5.0.0b4,<12.0.0 + cupy>=5.0.0b4,<13.0.0 cuda80 = - cupy-cuda80>=5.0.0b4,<12.0.0 + cupy-cuda80>=5.0.0b4,<13.0.0 cuda90 = - cupy-cuda90>=5.0.0b4,<12.0.0 + cupy-cuda90>=5.0.0b4,<13.0.0 cuda91 = - cupy-cuda91>=5.0.0b4,<12.0.0 + cupy-cuda91>=5.0.0b4,<13.0.0 cuda92 = - cupy-cuda92>=5.0.0b4,<12.0.0 + cupy-cuda92>=5.0.0b4,<13.0.0 cuda100 = - cupy-cuda100>=5.0.0b4,<12.0.0 + cupy-cuda100>=5.0.0b4,<13.0.0 cuda101 = - cupy-cuda101>=5.0.0b4,<12.0.0 + cupy-cuda101>=5.0.0b4,<13.0.0 cuda102 = - cupy-cuda102>=5.0.0b4,<12.0.0 + cupy-cuda102>=5.0.0b4,<13.0.0 cuda110 = - cupy-cuda110>=5.0.0b4,<12.0.0 + cupy-cuda110>=5.0.0b4,<13.0.0 cuda111 = - cupy-cuda111>=5.0.0b4,<12.0.0 + cupy-cuda111>=5.0.0b4,<13.0.0 cuda112 = - cupy-cuda112>=5.0.0b4,<12.0.0 + cupy-cuda112>=5.0.0b4,<13.0.0 cuda113 = - cupy-cuda113>=5.0.0b4,<12.0.0 + cupy-cuda113>=5.0.0b4,<13.0.0 cuda114 = - cupy-cuda114>=5.0.0b4,<12.0.0 + cupy-cuda114>=5.0.0b4,<13.0.0 cuda115 = - cupy-cuda115>=5.0.0b4,<12.0.0 + cupy-cuda115>=5.0.0b4,<13.0.0 cuda116 = - cupy-cuda116>=5.0.0b4,<12.0.0 + cupy-cuda116>=5.0.0b4,<13.0.0 cuda117 = - cupy-cuda117>=5.0.0b4,<12.0.0 + cupy-cuda117>=5.0.0b4,<13.0.0 cuda11x = - cupy-cuda11x>=11.0.0,<12.0.0 + cupy-cuda11x>=11.0.0,<13.0.0 cuda-autodetect = - cupy-wheel>=11.0.0,<12.0.0 + cupy-wheel>=11.0.0,<13.0.0 apple = thinc-apple-ops>=0.1.0.dev0,<1.0.0 # Language tokenizers with external dependencies diff --git a/spacy/cli/__init__.py b/spacy/cli/__init__.py index aab2c8d12..868526b42 100644 --- a/spacy/cli/__init__.py +++ b/spacy/cli/__init__.py @@ -4,6 +4,7 @@ from ._util import app, setup_cli # noqa: F401 # These are the actual functions, NOT the wrapped CLI commands. The CLI commands # are registered automatically and won't have to be imported here. +from .benchmark_speed import benchmark_speed_cli # noqa: F401 from .download import download # noqa: F401 from .info import info # noqa: F401 from .package import package # noqa: F401 @@ -16,6 +17,7 @@ from .debug_config import debug_config # noqa: F401 from .debug_model import debug_model # noqa: F401 from .debug_diff import debug_diff # noqa: F401 from .evaluate import evaluate # noqa: F401 +from .apply import apply # noqa: F401 from .convert import convert # noqa: F401 from .init_pipeline import init_pipeline_cli # noqa: F401 from .init_config import init_config, fill_config # noqa: F401 diff --git a/spacy/cli/_util.py b/spacy/cli/_util.py index 9b97a9f19..f104feff9 100644 --- a/spacy/cli/_util.py +++ b/spacy/cli/_util.py @@ -46,6 +46,7 @@ DEBUG_HELP = """Suite of helpful commands for debugging and profiling. Includes commands to check and validate your config files, training and evaluation data, and custom model implementations. """ +BENCHMARK_HELP = """Commands for benchmarking pipelines.""" INIT_HELP = """Commands for initializing configs and pipeline packages.""" # Wrappers for Typer's annotations. Initially created to set defaults and to @@ -54,12 +55,14 @@ Arg = typer.Argument Opt = typer.Option app = typer.Typer(name=NAME, help=HELP) +benchmark_cli = typer.Typer(name="benchmark", help=BENCHMARK_HELP, no_args_is_help=True) project_cli = typer.Typer(name="project", help=PROJECT_HELP, no_args_is_help=True) debug_cli = typer.Typer(name="debug", help=DEBUG_HELP, no_args_is_help=True) init_cli = typer.Typer(name="init", help=INIT_HELP, no_args_is_help=True) app.add_typer(project_cli) app.add_typer(debug_cli) +app.add_typer(benchmark_cli) app.add_typer(init_cli) @@ -87,9 +90,9 @@ def parse_config_overrides( cli_overrides = _parse_overrides(args, is_cli=True) if cli_overrides: keys = [k for k in cli_overrides if k not in env_overrides] - logger.debug(f"Config overrides from CLI: {keys}") + logger.debug("Config overrides from CLI: %s", keys) if env_overrides: - logger.debug(f"Config overrides from env variables: {list(env_overrides)}") + logger.debug("Config overrides from env variables: %s", list(env_overrides)) return {**cli_overrides, **env_overrides} @@ -582,6 +585,33 @@ def setup_gpu(use_gpu: int, silent=None) -> None: local_msg.info("To switch to GPU 0, use the option: --gpu-id 0") +def walk_directory(path: Path, suffix: Optional[str] = None) -> List[Path]: + """Given a directory and a suffix, recursively find all files matching the suffix. + Directories or files with names beginning with a . are ignored, but hidden flags on + filesystems are not checked. + When provided with a suffix `None`, there is no suffix-based filtering.""" + if not path.is_dir(): + return [path] + paths = [path] + locs = [] + seen = set() + for path in paths: + if str(path) in seen: + continue + seen.add(str(path)) + if path.parts[-1].startswith("."): + continue + elif path.is_dir(): + paths.extend(path.iterdir()) + elif suffix is not None and not path.parts[-1].endswith(suffix): + continue + else: + locs.append(path) + # It's good to sort these, in case the ordering messes up cache. + locs.sort() + return locs + + def _format_number(number: Union[int, float], ndigits: int = 2) -> str: """Formats a number (float or int) rounding to `ndigits`, without truncating trailing 0s, as happens with `round(number, ndigits)`""" diff --git a/spacy/cli/apply.py b/spacy/cli/apply.py new file mode 100644 index 000000000..f0df4e757 --- /dev/null +++ b/spacy/cli/apply.py @@ -0,0 +1,143 @@ +import tqdm +import srsly + +from itertools import chain +from pathlib import Path +from typing import Optional, List, Iterable, cast, Union + +from wasabi import msg + +from ._util import app, Arg, Opt, setup_gpu, import_code, walk_directory + +from ..tokens import Doc, DocBin +from ..vocab import Vocab +from ..util import ensure_path, load_model + + +path_help = """Location of the documents to predict on. +Can be a single file in .spacy format or a .jsonl file. +Files with other extensions are treated as single plain text documents. +If a directory is provided it is traversed recursively to grab +all files to be processed. +The files can be a mixture of .spacy, .jsonl and text files. +If .jsonl is provided the specified field is going +to be grabbed ("text" by default).""" + +out_help = "Path to save the resulting .spacy file" +code_help = ( + "Path to Python file with additional " "code (registered functions) to be imported" +) +gold_help = "Use gold preprocessing provided in the .spacy files" +force_msg = ( + "The provided output file already exists. " + "To force overwriting the output file, set the --force or -F flag." +) + + +DocOrStrStream = Union[Iterable[str], Iterable[Doc]] + + +def _stream_docbin(path: Path, vocab: Vocab) -> Iterable[Doc]: + """ + Stream Doc objects from DocBin. + """ + docbin = DocBin().from_disk(path) + for doc in docbin.get_docs(vocab): + yield doc + + +def _stream_jsonl(path: Path, field: str) -> Iterable[str]: + """ + Stream "text" field from JSONL. If the field "text" is + not found it raises error. + """ + for entry in srsly.read_jsonl(path): + if field not in entry: + msg.fail(f"{path} does not contain the required '{field}' field.", exits=1) + else: + yield entry[field] + + +def _stream_texts(paths: Iterable[Path]) -> Iterable[str]: + """ + Yields strings from text files in paths. + """ + for path in paths: + with open(path, "r") as fin: + text = fin.read() + yield text + + +@app.command("apply") +def apply_cli( + # fmt: off + model: str = Arg(..., help="Model name or path"), + data_path: Path = Arg(..., help=path_help, exists=True), + output_file: Path = Arg(..., help=out_help, dir_okay=False), + code_path: Optional[Path] = Opt(None, "--code", "-c", help=code_help), + text_key: str = Opt("text", "--text-key", "-tk", help="Key containing text string for JSONL"), + force_overwrite: bool = Opt(False, "--force", "-F", help="Force overwriting the output file"), + use_gpu: int = Opt(-1, "--gpu-id", "-g", help="GPU ID or -1 for CPU."), + batch_size: int = Opt(1, "--batch-size", "-b", help="Batch size."), + n_process: int = Opt(1, "--n-process", "-n", help="number of processors to use.") +): + """ + Apply a trained pipeline to documents to get predictions. + Expects a loadable spaCy pipeline and path to the data, which + can be a directory or a file. + The data files can be provided in multiple formats: + 1. .spacy files + 2. .jsonl files with a specified "field" to read the text from. + 3. Files with any other extension are assumed to be containing + a single document. + DOCS: https://spacy.io/api/cli#apply + """ + data_path = ensure_path(data_path) + output_file = ensure_path(output_file) + code_path = ensure_path(code_path) + if output_file.exists() and not force_overwrite: + msg.fail(force_msg, exits=1) + if not data_path.exists(): + msg.fail(f"Couldn't find data path: {data_path}", exits=1) + import_code(code_path) + setup_gpu(use_gpu) + apply(data_path, output_file, model, text_key, batch_size, n_process) + + +def apply( + data_path: Path, + output_file: Path, + model: str, + json_field: str, + batch_size: int, + n_process: int, +): + docbin = DocBin(store_user_data=True) + paths = walk_directory(data_path) + if len(paths) == 0: + docbin.to_disk(output_file) + msg.warn( + "Did not find data to process," + f" {data_path} seems to be an empty directory." + ) + return + nlp = load_model(model) + msg.good(f"Loaded model {model}") + vocab = nlp.vocab + streams: List[DocOrStrStream] = [] + text_files = [] + for path in paths: + if path.suffix == ".spacy": + streams.append(_stream_docbin(path, vocab)) + elif path.suffix == ".jsonl": + streams.append(_stream_jsonl(path, json_field)) + else: + text_files.append(path) + if len(text_files) > 0: + streams.append(_stream_texts(text_files)) + datagen = cast(DocOrStrStream, chain(*streams)) + for doc in tqdm.tqdm(nlp.pipe(datagen, batch_size=batch_size, n_process=n_process)): + docbin.add(doc) + if output_file.suffix == "": + output_file = output_file.with_suffix(".spacy") + docbin.to_disk(output_file) diff --git a/spacy/cli/benchmark_speed.py b/spacy/cli/benchmark_speed.py new file mode 100644 index 000000000..4eb20a5fa --- /dev/null +++ b/spacy/cli/benchmark_speed.py @@ -0,0 +1,174 @@ +from typing import Iterable, List, Optional +import random +from itertools import islice +import numpy +from pathlib import Path +import time +from tqdm import tqdm +import typer +from wasabi import msg + +from .. import util +from ..language import Language +from ..tokens import Doc +from ..training import Corpus +from ._util import Arg, Opt, benchmark_cli, setup_gpu + + +@benchmark_cli.command( + "speed", + context_settings={"allow_extra_args": True, "ignore_unknown_options": True}, +) +def benchmark_speed_cli( + # fmt: off + ctx: typer.Context, + model: str = Arg(..., help="Model name or path"), + data_path: Path = Arg(..., help="Location of binary evaluation data in .spacy format", exists=True), + batch_size: Optional[int] = Opt(None, "--batch-size", "-b", min=1, help="Override the pipeline batch size"), + no_shuffle: bool = Opt(False, "--no-shuffle", help="Do not shuffle benchmark data"), + use_gpu: int = Opt(-1, "--gpu-id", "-g", help="GPU ID or -1 for CPU"), + n_batches: int = Opt(50, "--batches", help="Minimum number of batches to benchmark", min=30,), + warmup_epochs: int = Opt(3, "--warmup", "-w", min=0, help="Number of iterations over the data for warmup"), + # fmt: on +): + """ + Benchmark a pipeline. Expects a loadable spaCy pipeline and benchmark + data in the binary .spacy format. + """ + setup_gpu(use_gpu=use_gpu, silent=False) + + nlp = util.load_model(model) + batch_size = batch_size if batch_size is not None else nlp.batch_size + corpus = Corpus(data_path) + docs = [eg.predicted for eg in corpus(nlp)] + + if len(docs) == 0: + msg.fail("Cannot benchmark speed using an empty corpus.", exits=1) + + print(f"Warming up for {warmup_epochs} epochs...") + warmup(nlp, docs, warmup_epochs, batch_size) + + print() + print(f"Benchmarking {n_batches} batches...") + wps = benchmark(nlp, docs, n_batches, batch_size, not no_shuffle) + + print() + print_outliers(wps) + print_mean_with_ci(wps) + + +# Lowercased, behaves as a context manager function. +class time_context: + """Register the running time of a context.""" + + def __enter__(self): + self.start = time.perf_counter() + return self + + def __exit__(self, type, value, traceback): + self.elapsed = time.perf_counter() - self.start + + +class Quartiles: + """Calculate the q1, q2, q3 quartiles and the inter-quartile range (iqr) + of a sample.""" + + q1: float + q2: float + q3: float + iqr: float + + def __init__(self, sample: numpy.ndarray) -> None: + self.q1 = numpy.quantile(sample, 0.25) + self.q2 = numpy.quantile(sample, 0.5) + self.q3 = numpy.quantile(sample, 0.75) + self.iqr = self.q3 - self.q1 + + +def annotate( + nlp: Language, docs: List[Doc], batch_size: Optional[int] +) -> numpy.ndarray: + docs = nlp.pipe(tqdm(docs, unit="doc"), batch_size=batch_size) + wps = [] + while True: + with time_context() as elapsed: + batch_docs = list( + islice(docs, batch_size if batch_size else nlp.batch_size) + ) + if len(batch_docs) == 0: + break + n_tokens = count_tokens(batch_docs) + wps.append(n_tokens / elapsed.elapsed) + + return numpy.array(wps) + + +def benchmark( + nlp: Language, + docs: List[Doc], + n_batches: int, + batch_size: int, + shuffle: bool, +) -> numpy.ndarray: + if shuffle: + bench_docs = [ + nlp.make_doc(random.choice(docs).text) + for _ in range(n_batches * batch_size) + ] + else: + bench_docs = [ + nlp.make_doc(docs[i % len(docs)].text) + for i in range(n_batches * batch_size) + ] + + return annotate(nlp, bench_docs, batch_size) + + +def bootstrap(x, statistic=numpy.mean, iterations=10000) -> numpy.ndarray: + """Apply a statistic to repeated random samples of an array.""" + return numpy.fromiter( + ( + statistic(numpy.random.choice(x, len(x), replace=True)) + for _ in range(iterations) + ), + numpy.float64, + ) + + +def count_tokens(docs: Iterable[Doc]) -> int: + return sum(len(doc) for doc in docs) + + +def print_mean_with_ci(sample: numpy.ndarray): + mean = numpy.mean(sample) + bootstrap_means = bootstrap(sample) + bootstrap_means.sort() + + # 95% confidence interval + low = bootstrap_means[int(len(bootstrap_means) * 0.025)] + high = bootstrap_means[int(len(bootstrap_means) * 0.975)] + + print(f"Mean: {mean:.1f} words/s (95% CI: {low-mean:.1f} +{high-mean:.1f})") + + +def print_outliers(sample: numpy.ndarray): + quartiles = Quartiles(sample) + + n_outliers = numpy.sum( + (sample < (quartiles.q1 - 1.5 * quartiles.iqr)) + | (sample > (quartiles.q3 + 1.5 * quartiles.iqr)) + ) + n_extreme_outliers = numpy.sum( + (sample < (quartiles.q1 - 3.0 * quartiles.iqr)) + | (sample > (quartiles.q3 + 3.0 * quartiles.iqr)) + ) + print( + f"Outliers: {(100 * n_outliers) / len(sample):.1f}%, extreme outliers: {(100 * n_extreme_outliers) / len(sample)}%" + ) + + +def warmup( + nlp: Language, docs: List[Doc], warmup_epochs: int, batch_size: Optional[int] +) -> numpy.ndarray: + docs = warmup_epochs * docs + return annotate(nlp, docs, batch_size) diff --git a/spacy/cli/convert.py b/spacy/cli/convert.py index 04eb7078f..68d454b3e 100644 --- a/spacy/cli/convert.py +++ b/spacy/cli/convert.py @@ -1,4 +1,4 @@ -from typing import Callable, Iterable, Mapping, Optional, Any, List, Union +from typing import Callable, Iterable, Mapping, Optional, Any, Union from enum import Enum from pathlib import Path from wasabi import Printer @@ -7,7 +7,7 @@ import re import sys import itertools -from ._util import app, Arg, Opt +from ._util import app, Arg, Opt, walk_directory from ..training import docs_to_json from ..tokens import Doc, DocBin from ..training.converters import iob_to_docs, conll_ner_to_docs, json_to_docs @@ -28,6 +28,8 @@ CONVERTERS: Mapping[str, Callable[..., Iterable[Doc]]] = { "json": json_to_docs, } +AUTO = "auto" + # File types that can be written to stdout FILE_TYPES_STDOUT = ("json",) @@ -49,7 +51,7 @@ def convert_cli( model: Optional[str] = Opt(None, "--model", "--base", "-b", help="Trained spaCy pipeline for sentence segmentation to use as base (for --seg-sents)"), morphology: bool = Opt(False, "--morphology", "-m", help="Enable appending morphology to tags"), merge_subtokens: bool = Opt(False, "--merge-subtokens", "-T", help="Merge CoNLL-U subtokens"), - converter: str = Opt("auto", "--converter", "-c", help=f"Converter: {tuple(CONVERTERS.keys())}"), + converter: str = Opt(AUTO, "--converter", "-c", help=f"Converter: {tuple(CONVERTERS.keys())}"), ner_map: Optional[Path] = Opt(None, "--ner-map", "-nm", help="NER tag mapping (as JSON-encoded dict of entity types)", exists=True), lang: Optional[str] = Opt(None, "--lang", "-l", help="Language (if tokenizer required)"), concatenate: bool = Opt(None, "--concatenate", "-C", help="Concatenate output to a single file"), @@ -70,8 +72,8 @@ def convert_cli( output_dir: Union[str, Path] = "-" if output_dir == Path("-") else output_dir silent = output_dir == "-" msg = Printer(no_print=silent) - verify_cli_args(msg, input_path, output_dir, file_type.value, converter, ner_map) converter = _get_converter(msg, converter, input_path) + verify_cli_args(msg, input_path, output_dir, file_type.value, converter, ner_map) convert( input_path, output_dir, @@ -100,7 +102,7 @@ def convert( model: Optional[str] = None, morphology: bool = False, merge_subtokens: bool = False, - converter: str = "auto", + converter: str, ner_map: Optional[Path] = None, lang: Optional[str] = None, concatenate: bool = False, @@ -189,33 +191,6 @@ def autodetect_ner_format(input_data: str) -> Optional[str]: return None -def walk_directory(path: Path, converter: str) -> List[Path]: - if not path.is_dir(): - return [path] - paths = [path] - locs = [] - seen = set() - for path in paths: - if str(path) in seen: - continue - seen.add(str(path)) - if path.parts[-1].startswith("."): - continue - elif path.is_dir(): - paths.extend(path.iterdir()) - elif converter == "json" and not path.parts[-1].endswith("json"): - continue - elif converter == "conll" and not path.parts[-1].endswith("conll"): - continue - elif converter == "iob" and not path.parts[-1].endswith("iob"): - continue - else: - locs.append(path) - # It's good to sort these, in case the ordering messes up cache. - locs.sort() - return locs - - def verify_cli_args( msg: Printer, input_path: Path, @@ -239,18 +214,22 @@ def verify_cli_args( input_locs = walk_directory(input_path, converter) if len(input_locs) == 0: msg.fail("No input files in directory", input_path, exits=1) - file_types = list(set([loc.suffix[1:] for loc in input_locs])) - if converter == "auto" and len(file_types) >= 2: - file_types_str = ",".join(file_types) - msg.fail("All input files must be same type", file_types_str, exits=1) - if converter != "auto" and converter not in CONVERTERS: + if converter not in CONVERTERS: msg.fail(f"Can't find converter for {converter}", exits=1) def _get_converter(msg, converter, input_path: Path): if input_path.is_dir(): - input_path = walk_directory(input_path, converter)[0] - if converter == "auto": + if converter == AUTO: + input_locs = walk_directory(input_path, suffix=None) + file_types = list(set([loc.suffix[1:] for loc in input_locs])) + if len(file_types) >= 2: + file_types_str = ",".join(file_types) + msg.fail("All input files must be same type", file_types_str, exits=1) + input_path = input_locs[0] + else: + input_path = walk_directory(input_path, suffix=converter)[0] + if converter == AUTO: converter = input_path.suffix[1:] if converter == "ner" or converter == "iob": with input_path.open(encoding="utf8") as file_: diff --git a/spacy/cli/debug_data.py b/spacy/cli/debug_data.py index a85324e87..97b4db285 100644 --- a/spacy/cli/debug_data.py +++ b/spacy/cli/debug_data.py @@ -7,6 +7,7 @@ import srsly from wasabi import Printer, MESSAGES, msg import typer import math +import numpy from ._util import app, Arg, Opt, show_validation_error, parse_config_overrides from ._util import import_code, debug_cli, _format_number @@ -17,6 +18,7 @@ from ..pipeline import TrainablePipe from ..pipeline._parser_internals import nonproj from ..pipeline._parser_internals.nonproj import DELIMITER from ..pipeline import Morphologizer, SpanCategorizer +from ..pipeline._edit_tree_internals.edit_trees import EditTrees from ..morphology import Morphology from ..language import Language from ..util import registry, resolve_dot_names @@ -520,9 +522,13 @@ def debug_data( if "tagger" in factory_names: msg.divider("Part-of-speech Tagging") - label_list = [label for label in gold_train_data["tags"]] - model_labels = _get_labels_from_model(nlp, "tagger") + label_list, counts = zip(*gold_train_data["tags"].items()) msg.info(f"{len(label_list)} label(s) in train data") + p = numpy.array(counts) + p = p / p.sum() + norm_entropy = (-p * numpy.log2(p)).sum() / numpy.log2(len(label_list)) + msg.info(f"{norm_entropy} is the normalised label entropy") + model_labels = _get_labels_from_model(nlp, "tagger") labels = set(label_list) missing_labels = model_labels - labels if missing_labels: @@ -671,6 +677,59 @@ def debug_data( f"Found {gold_train_data['n_cycles']} projectivized train sentence(s) with cycles" ) + if "trainable_lemmatizer" in factory_names: + msg.divider("Trainable Lemmatizer") + trees_train: Set[str] = gold_train_data["lemmatizer_trees"] + trees_dev: Set[str] = gold_dev_data["lemmatizer_trees"] + # This is necessary context when someone is attempting to interpret whether the + # number of trees exclusively in the dev set is meaningful. + msg.info(f"{len(trees_train)} lemmatizer trees generated from training data") + msg.info(f"{len(trees_dev)} lemmatizer trees generated from dev data") + dev_not_train = trees_dev - trees_train + + if len(dev_not_train) != 0: + pct = len(dev_not_train) / len(trees_dev) + msg.info( + f"{len(dev_not_train)} lemmatizer trees ({pct*100:.1f}% of dev trees)" + " were found exclusively in the dev data." + ) + else: + # Would we ever expect this case? It seems like it would be pretty rare, + # and we might actually want a warning? + msg.info("All trees in dev data present in training data.") + + if gold_train_data["n_low_cardinality_lemmas"] > 0: + n = gold_train_data["n_low_cardinality_lemmas"] + msg.warn(f"{n} training docs with 0 or 1 unique lemmas.") + + if gold_dev_data["n_low_cardinality_lemmas"] > 0: + n = gold_dev_data["n_low_cardinality_lemmas"] + msg.warn(f"{n} dev docs with 0 or 1 unique lemmas.") + + if gold_train_data["no_lemma_annotations"] > 0: + n = gold_train_data["no_lemma_annotations"] + msg.warn(f"{n} training docs with no lemma annotations.") + else: + msg.good("All training docs have lemma annotations.") + + if gold_dev_data["no_lemma_annotations"] > 0: + n = gold_dev_data["no_lemma_annotations"] + msg.warn(f"{n} dev docs with no lemma annotations.") + else: + msg.good("All dev docs have lemma annotations.") + + if gold_train_data["partial_lemma_annotations"] > 0: + n = gold_train_data["partial_lemma_annotations"] + msg.info(f"{n} training docs with partial lemma annotations.") + else: + msg.good("All training docs have complete lemma annotations.") + + if gold_dev_data["partial_lemma_annotations"] > 0: + n = gold_dev_data["partial_lemma_annotations"] + msg.info(f"{n} dev docs with partial lemma annotations.") + else: + msg.good("All dev docs have complete lemma annotations.") + msg.divider("Summary") good_counts = msg.counts[MESSAGES.GOOD] warn_counts = msg.counts[MESSAGES.WARN] @@ -732,7 +791,13 @@ def _compile_gold( "n_cats_multilabel": 0, "n_cats_bad_values": 0, "texts": set(), + "lemmatizer_trees": set(), + "no_lemma_annotations": 0, + "partial_lemma_annotations": 0, + "n_low_cardinality_lemmas": 0, } + if "trainable_lemmatizer" in factory_names: + trees = EditTrees(nlp.vocab.strings) for eg in examples: gold = eg.reference doc = eg.predicted @@ -862,6 +927,25 @@ def _compile_gold( data["n_nonproj"] += 1 if nonproj.contains_cycle(aligned_heads): data["n_cycles"] += 1 + if "trainable_lemmatizer" in factory_names: + # from EditTreeLemmatizer._labels_from_data + if all(token.lemma == 0 for token in gold): + data["no_lemma_annotations"] += 1 + continue + if any(token.lemma == 0 for token in gold): + data["partial_lemma_annotations"] += 1 + lemma_set = set() + for token in gold: + if token.lemma != 0: + lemma_set.add(token.lemma) + tree_id = trees.add(token.text, token.lemma_) + tree_str = trees.tree_to_str(tree_id) + data["lemmatizer_trees"].add(tree_str) + # We want to identify cases where lemmas aren't assigned + # or are all assigned the same value, as this would indicate + # an issue since we're expecting a large set of lemmas + if len(lemma_set) < 2 and len(gold) > 1: + data["n_low_cardinality_lemmas"] += 1 return data diff --git a/spacy/cli/evaluate.py b/spacy/cli/evaluate.py index 0d08d2c5e..8f3d6b859 100644 --- a/spacy/cli/evaluate.py +++ b/spacy/cli/evaluate.py @@ -7,12 +7,15 @@ from thinc.api import fix_random_seed from ..training import Corpus from ..tokens import Doc -from ._util import app, Arg, Opt, setup_gpu, import_code +from ._util import app, Arg, Opt, setup_gpu, import_code, benchmark_cli from ..scorer import Scorer from .. import util from .. import displacy +@benchmark_cli.command( + "accuracy", +) @app.command("evaluate") def evaluate_cli( # fmt: off @@ -36,7 +39,7 @@ def evaluate_cli( dependency parses in a HTML file, set as output directory as the displacy_path argument. - DOCS: https://spacy.io/api/cli#evaluate + DOCS: https://spacy.io/api/cli#benchmark-accuracy """ import_code(code_path) evaluate( diff --git a/spacy/cli/find_threshold.py b/spacy/cli/find_threshold.py index efa664832..6d591053d 100644 --- a/spacy/cli/find_threshold.py +++ b/spacy/cli/find_threshold.py @@ -35,7 +35,7 @@ def find_threshold_cli( code_path: Optional[Path] = Opt(None, "--code", "-c", help="Path to Python file with additional code (registered functions) to be imported"), use_gpu: int = Opt(_DEFAULTS["use_gpu"], "--gpu-id", "-g", help="GPU ID or -1 for CPU"), gold_preproc: bool = Opt(_DEFAULTS["gold_preproc"], "--gold-preproc", "-G", help="Use gold preprocessing"), - verbose: bool = Opt(False, "--silent", "-V", "-VV", help="Display more information for debugging purposes"), + verbose: bool = Opt(False, "--verbose", "-V", "-VV", help="Display more information for debugging purposes"), # fmt: on ): """ diff --git a/spacy/cli/info.py b/spacy/cli/info.py index 974bc0f4e..d82bf3fbc 100644 --- a/spacy/cli/info.py +++ b/spacy/cli/info.py @@ -1,6 +1,5 @@ from typing import Optional, Dict, Any, Union, List import platform -import pkg_resources import json from pathlib import Path from wasabi import Printer, MarkdownRenderer @@ -10,6 +9,7 @@ from ._util import app, Arg, Opt, string_to_list from .download import get_model_filename, get_latest_version from .. import util from .. import about +from ..compat import importlib_metadata @app.command("info") @@ -137,15 +137,14 @@ def info_installed_model_url(model: str) -> Optional[str]: dist-info available. """ try: - dist = pkg_resources.get_distribution(model) - data = json.loads(dist.get_metadata("direct_url.json")) - return data["url"] - except pkg_resources.DistributionNotFound: - # no such package - return None + dist = importlib_metadata.distribution(model) + text = dist.read_text("direct_url.json") + if isinstance(text, str): + data = json.loads(text) + return data["url"] except Exception: - # something else, like no file or invalid JSON - return None + pass + return None def info_model_url(model: str) -> Dict[str, Any]: diff --git a/spacy/cli/package.py b/spacy/cli/package.py index 324c5d1bb..6351f28eb 100644 --- a/spacy/cli/package.py +++ b/spacy/cli/package.py @@ -252,7 +252,7 @@ def get_third_party_dependencies( raise regerr from None module_name = func_info.get("module") # type: ignore[attr-defined] if module_name: # the code is part of a module, not a --code file - modules.add(func_info["module"].split(".")[0]) # type: ignore[index] + modules.add(func_info["module"].split(".")[0]) # type: ignore[union-attr] dependencies = [] for module_name in modules: if module_name in distributions: diff --git a/spacy/cli/pretrain.py b/spacy/cli/pretrain.py index 381d589cf..45042e605 100644 --- a/spacy/cli/pretrain.py +++ b/spacy/cli/pretrain.py @@ -23,6 +23,7 @@ def pretrain_cli( resume_path: Optional[Path] = Opt(None, "--resume-path", "-r", help="Path to pretrained weights from which to resume pretraining"), epoch_resume: Optional[int] = Opt(None, "--epoch-resume", "-er", help="The epoch to resume counting from when using --resume-path. Prevents unintended overwriting of existing weight files."), use_gpu: int = Opt(-1, "--gpu-id", "-g", help="GPU ID or -1 for CPU"), + skip_last: bool = Opt(False, "--skip-last", "-L", help="Skip saving model-last.bin"), # fmt: on ): """ @@ -74,6 +75,7 @@ def pretrain_cli( epoch_resume=epoch_resume, use_gpu=use_gpu, silent=False, + skip_last=skip_last, ) msg.good("Successfully finished pretrain") diff --git a/spacy/cli/project/pull.py b/spacy/cli/project/pull.py index 6e3cde88c..8894baa50 100644 --- a/spacy/cli/project/pull.py +++ b/spacy/cli/project/pull.py @@ -39,14 +39,17 @@ def project_pull(project_dir: Path, remote: str, *, verbose: bool = False): # in the list. while commands: for i, cmd in enumerate(list(commands)): - logger.debug(f"CMD: {cmd['name']}.") + logger.debug("CMD: %s.", cmd["name"]) deps = [project_dir / dep for dep in cmd.get("deps", [])] if all(dep.exists() for dep in deps): cmd_hash = get_command_hash("", "", deps, cmd["script"]) for output_path in cmd.get("outputs", []): url = storage.pull(output_path, command_hash=cmd_hash) logger.debug( - f"URL: {url} for {output_path} with command hash {cmd_hash}" + "URL: %s for %s with command hash %s", + url, + output_path, + cmd_hash, ) yield url, output_path @@ -58,7 +61,7 @@ def project_pull(project_dir: Path, remote: str, *, verbose: bool = False): commands.pop(i) break else: - logger.debug(f"Dependency missing. Skipping {cmd['name']} outputs.") + logger.debug("Dependency missing. Skipping %s outputs.", cmd["name"]) else: # If we didn't break the for loop, break the while loop. break diff --git a/spacy/cli/project/push.py b/spacy/cli/project/push.py index bc779e9cd..a8178de21 100644 --- a/spacy/cli/project/push.py +++ b/spacy/cli/project/push.py @@ -37,15 +37,15 @@ def project_push(project_dir: Path, remote: str): remote = config["remotes"][remote] storage = RemoteStorage(project_dir, remote) for cmd in config.get("commands", []): - logger.debug(f"CMD: cmd['name']") + logger.debug("CMD: %s", cmd["name"]) deps = [project_dir / dep for dep in cmd.get("deps", [])] if any(not dep.exists() for dep in deps): - logger.debug(f"Dependency missing. Skipping {cmd['name']} outputs") + logger.debug("Dependency missing. Skipping %s outputs", cmd["name"]) continue cmd_hash = get_command_hash( "", "", [project_dir / dep for dep in cmd.get("deps", [])], cmd["script"] ) - logger.debug(f"CMD_HASH: {cmd_hash}") + logger.debug("CMD_HASH: %s", cmd_hash) for output_path in cmd.get("outputs", []): output_loc = project_dir / output_path if output_loc.exists() and _is_not_empty_dir(output_loc): @@ -55,7 +55,7 @@ def project_push(project_dir: Path, remote: str): content_hash=get_content_hash(output_loc), ) logger.debug( - f"URL: {url} for output {output_path} with cmd_hash {cmd_hash}" + "URL: %s for output %s with cmd_hash %s", url, output_path, cmd_hash ) yield output_path, url diff --git a/spacy/cli/project/run.py b/spacy/cli/project/run.py index 6dd174902..0f4858a99 100644 --- a/spacy/cli/project/run.py +++ b/spacy/cli/project/run.py @@ -2,7 +2,6 @@ from typing import Optional, List, Dict, Sequence, Any, Iterable, Tuple import os.path from pathlib import Path -import pkg_resources from wasabi import msg from wasabi.util import locale_escape import sys @@ -331,6 +330,7 @@ def _check_requirements(requirements: List[str]) -> Tuple[bool, bool]: RETURNS (Tuple[bool, bool]): Whether (1) any packages couldn't be imported, (2) any packages with version conflicts exist. """ + import pkg_resources failed_pkgs_msgs: List[str] = [] conflicting_pkgs_msgs: List[str] = [] diff --git a/spacy/cli/templates/quickstart_training.jinja b/spacy/cli/templates/quickstart_training.jinja index b961ac892..9481e53be 100644 --- a/spacy/cli/templates/quickstart_training.jinja +++ b/spacy/cli/templates/quickstart_training.jinja @@ -3,7 +3,7 @@ the docs and the init config command. It encodes various best practices and can help generate the best possible configuration, given a user's requirements. #} {%- set use_transformer = hardware != "cpu" and transformer_data -%} {%- set transformer = transformer_data[optimize] if use_transformer else {} -%} -{%- set listener_components = ["tagger", "morphologizer", "parser", "ner", "textcat", "textcat_multilabel", "entity_linker", "spancat", "trainable_lemmatizer"] -%} +{%- set listener_components = ["tagger", "morphologizer", "parser", "ner", "textcat", "textcat_multilabel", "entity_linker", "spancat", "spancat_singlelabel", "trainable_lemmatizer"] -%} [paths] train = null dev = null @@ -24,8 +24,11 @@ gpu_allocator = null lang = "{{ lang }}" {%- set has_textcat = ("textcat" in components or "textcat_multilabel" in components) -%} {%- set with_accuracy = optimize == "accuracy" -%} -{%- set has_accurate_textcat = has_textcat and with_accuracy -%} -{%- if ("tagger" in components or "morphologizer" in components or "parser" in components or "ner" in components or "spancat" in components or "trainable_lemmatizer" in components or "entity_linker" in components or has_accurate_textcat) -%} +{# The BOW textcat doesn't need a source of features, so it can omit the +tok2vec/transformer. #} +{%- set with_accuracy_or_transformer = (use_transformer or with_accuracy) -%} +{%- set textcat_needs_features = has_textcat and with_accuracy_or_transformer -%} +{%- if ("tagger" in components or "morphologizer" in components or "parser" in components or "ner" in components or "spancat" in components or "spancat_singlelabel" in components or "trainable_lemmatizer" in components or "entity_linker" in components or textcat_needs_features) -%} {%- set full_pipeline = ["transformer" if use_transformer else "tok2vec"] + components -%} {%- else -%} {%- set full_pipeline = components -%} @@ -156,6 +159,36 @@ grad_factor = 1.0 sizes = [1,2,3] {% endif -%} +{% if "spancat_singlelabel" in components %} +[components.spancat_singlelabel] +factory = "spancat_singlelabel" +negative_weight = 1.0 +allow_overlap = true +scorer = {"@scorers":"spacy.spancat_scorer.v1"} +spans_key = "sc" + +[components.spancat_singlelabel.model] +@architectures = "spacy.SpanCategorizer.v1" + +[components.spancat_singlelabel.model.reducer] +@layers = "spacy.mean_max_reducer.v1" +hidden_size = 128 + +[components.spancat_singlelabel.model.scorer] +@layers = "Softmax.v2" + +[components.spancat_singlelabel.model.tok2vec] +@architectures = "spacy-transformers.TransformerListener.v1" +grad_factor = 1.0 + +[components.spancat_singlelabel.model.tok2vec.pooling] +@layers = "reduce_mean.v1" + +[components.spancat_singlelabel.suggester] +@misc = "spacy.ngram_suggester.v1" +sizes = [1,2,3] +{% endif %} + {% if "trainable_lemmatizer" in components -%} [components.trainable_lemmatizer] factory = "trainable_lemmatizer" @@ -221,10 +254,16 @@ no_output_layer = false {% else -%} [components.textcat.model] -@architectures = "spacy.TextCatBOW.v2" +@architectures = "spacy.TextCatCNN.v2" exclusive_classes = true -ngram_size = 1 -no_output_layer = false +nO = null + +[components.textcat.model.tok2vec] +@architectures = "spacy-transformers.TransformerListener.v1" +grad_factor = 1.0 + +[components.textcat.model.tok2vec.pooling] +@layers = "reduce_mean.v1" {%- endif %} {%- endif %} @@ -252,10 +291,16 @@ no_output_layer = false {% else -%} [components.textcat_multilabel.model] -@architectures = "spacy.TextCatBOW.v2" +@architectures = "spacy.TextCatCNN.v2" exclusive_classes = false -ngram_size = 1 -no_output_layer = false +nO = null + +[components.textcat_multilabel.model.tok2vec] +@architectures = "spacy-transformers.TransformerListener.v1" +grad_factor = 1.0 + +[components.textcat_multilabel.model.tok2vec.pooling] +@layers = "reduce_mean.v1" {%- endif %} {%- endif %} @@ -286,6 +331,7 @@ maxout_pieces = 3 {% if "morphologizer" in components %} [components.morphologizer] factory = "morphologizer" +label_smoothing = 0.05 [components.morphologizer.model] @architectures = "spacy.Tagger.v2" @@ -299,6 +345,7 @@ width = ${components.tok2vec.model.encode.width} {% if "tagger" in components %} [components.tagger] factory = "tagger" +label_smoothing = 0.05 [components.tagger.model] @architectures = "spacy.Tagger.v2" @@ -374,6 +421,33 @@ width = ${components.tok2vec.model.encode.width} sizes = [1,2,3] {% endif %} +{% if "spancat_singlelabel" in components %} +[components.spancat_singlelabel] +factory = "spancat_singlelabel" +negative_weight = 1.0 +allow_overlap = true +scorer = {"@scorers":"spacy.spancat_scorer.v1"} +spans_key = "sc" + +[components.spancat_singlelabel.model] +@architectures = "spacy.SpanCategorizer.v1" + +[components.spancat_singlelabel.model.reducer] +@layers = "spacy.mean_max_reducer.v1" +hidden_size = 128 + +[components.spancat_singlelabel.model.scorer] +@layers = "Softmax.v2" + +[components.spancat_singlelabel.model.tok2vec] +@architectures = "spacy.Tok2VecListener.v1" +width = ${components.tok2vec.model.encode.width} + +[components.spancat_singlelabel.suggester] +@misc = "spacy.ngram_suggester.v1" +sizes = [1,2,3] +{% endif %} + {% if "trainable_lemmatizer" in components -%} [components.trainable_lemmatizer] factory = "trainable_lemmatizer" diff --git a/spacy/displacy/__init__.py b/spacy/displacy/__init__.py index bc32001d7..f42dad0c9 100644 --- a/spacy/displacy/__init__.py +++ b/spacy/displacy/__init__.py @@ -11,6 +11,7 @@ from .render import DependencyRenderer, EntityRenderer, SpanRenderer from ..tokens import Doc, Span from ..errors import Errors, Warnings from ..util import is_in_jupyter +from ..util import find_available_port _html = {} @@ -36,7 +37,7 @@ def render( jupyter (bool): Override Jupyter auto-detection. options (dict): Visualiser-specific options, e.g. colors. manual (bool): Don't parse `Doc` and instead expect a dict/list of dicts. - RETURNS (str): Rendered HTML markup. + RETURNS (str): Rendered SVG or HTML markup. DOCS: https://spacy.io/api/top-level#displacy.render USAGE: https://spacy.io/usage/visualizers @@ -82,6 +83,7 @@ def serve( manual: bool = False, port: int = 5000, host: str = "0.0.0.0", + auto_select_port: bool = False, ) -> None: """Serve displaCy visualisation. @@ -93,12 +95,15 @@ def serve( manual (bool): Don't parse `Doc` and instead expect a dict/list of dicts. port (int): Port to serve visualisation. host (str): Host to serve visualisation. + auto_select_port (bool): Automatically select a port if the specified port is in use. DOCS: https://spacy.io/api/top-level#displacy.serve USAGE: https://spacy.io/usage/visualizers """ from wsgiref import simple_server + port = find_available_port(port, host, auto_select_port) + if is_in_jupyter(): warnings.warn(Warnings.W011) render(docs, style=style, page=page, minify=minify, options=options, manual=manual) @@ -120,13 +125,17 @@ def app(environ, start_response): return [res] -def parse_deps(orig_doc: Doc, options: Dict[str, Any] = {}) -> Dict[str, Any]: +def parse_deps( + orig_doc: Union[Doc, Span], options: Dict[str, Any] = {} +) -> Dict[str, Any]: """Generate dependency parse in {'words': [], 'arcs': []} format. - orig_doc (Doc): Document to parse. + orig_doc (Union[Doc, Span]): Document to parse. options (Dict[str, Any]): Dependency parse specific visualisation options. RETURNS (dict): Generated dependency parse keyed by words and arcs. """ + if isinstance(orig_doc, Span): + orig_doc = orig_doc.as_doc() doc = Doc(orig_doc.vocab).from_bytes( orig_doc.to_bytes(exclude=["user_data", "user_hooks"]) ) diff --git a/spacy/displacy/render.py b/spacy/displacy/render.py index 50dc3466c..f74222dc2 100644 --- a/spacy/displacy/render.py +++ b/spacy/displacy/render.py @@ -94,7 +94,7 @@ class SpanRenderer: parsed (list): Dependency parses to render. page (bool): Render parses wrapped as full HTML page. minify (bool): Minify HTML markup. - RETURNS (str): Rendered HTML markup. + RETURNS (str): Rendered SVG or HTML markup. """ rendered = [] for i, p in enumerate(parsed): @@ -510,7 +510,7 @@ class EntityRenderer: parsed (list): Dependency parses to render. page (bool): Render parses wrapped as full HTML page. minify (bool): Minify HTML markup. - RETURNS (str): Rendered HTML markup. + RETURNS (str): Rendered SVG or HTML markup. """ rendered = [] for i, p in enumerate(parsed): diff --git a/spacy/errors.py b/spacy/errors.py index 9c95fefa9..3dce0ef2c 100644 --- a/spacy/errors.py +++ b/spacy/errors.py @@ -214,6 +214,7 @@ class Warnings(metaclass=ErrorsWithCodes): "is a Cython extension type.") W123 = ("Argument `enable` with value {enable} does not contain all values specified in the config option " "`enabled` ({enabled}). Be aware that this might affect other components in your pipeline.") + W124 = ("{host}:{port} is already in use, using the nearest available port {serve_port} as an alternative.") class Errors(metaclass=ErrorsWithCodes): @@ -443,8 +444,7 @@ class Errors(metaclass=ErrorsWithCodes): E133 = ("The sum of prior probabilities for alias '{alias}' should not " "exceed 1, but found {sum}.") E134 = ("Entity '{entity}' is not defined in the Knowledge Base.") - E139 = ("Knowledge base for component '{name}' is empty. Use the methods " - "`kb.add_entity` and `kb.add_alias` to add entries.") + E139 = ("Knowledge base for component '{name}' is empty.") E140 = ("The list of entities, prior probabilities and entity vectors " "should be of equal length.") E141 = ("Entity vectors should be of length {required} instead of the " @@ -549,6 +549,8 @@ class Errors(metaclass=ErrorsWithCodes): "during training, make sure to include it in 'annotating components'") # New errors added in v3.x + E850 = ("The PretrainVectors objective currently only supports default or " + "floret vectors, not {mode} vectors.") E851 = ("The 'textcat' component labels should only have values of 0 or 1, " "but found value of '{val}'.") E852 = ("The tar file pulled from the remote attempted an unsafe path " @@ -961,6 +963,12 @@ class Errors(metaclass=ErrorsWithCodes): E1045 = ("{cls_name} is an abstract class and cannot be instantiated. If you are looking for spaCy's default " "knowledge base, use `InMemoryLookupKB`.") E1047 = ("`find_threshold()` only supports components with a `scorer` attribute.") + E1048 = ("Got '{unexpected}' as console progress bar type, but expected one of the following: {expected}") + E1049 = ("No available port found for displaCy on host {host}. Please specify an available port " + "with `displacy.serve(doc, port=port)`") + E1050 = ("Port {port} is already in use. Please specify an available port with `displacy.serve(doc, port=port)` " + "or use `auto_select_port=True` to pick an available port automatically.") + E1051 = ("'allow_overlap' can only be False when max_positive is 1, but found 'max_positive': {max_positive}.") # Deprecated model shortcuts, only used in errors and warnings diff --git a/spacy/kb/kb_in_memory.pyx b/spacy/kb/kb_in_memory.pyx index a87ddf0f6..3a03ed53e 100644 --- a/spacy/kb/kb_in_memory.pyx +++ b/spacy/kb/kb_in_memory.pyx @@ -25,7 +25,7 @@ cdef class InMemoryLookupKB(KnowledgeBase): """An `InMemoryLookupKB` instance stores unique identifiers for entities and their textual aliases, to support entity linking of named entities to real-world concepts. - DOCS: https://spacy.io/api/kb_in_memory + DOCS: https://spacy.io/api/inmemorylookupkb """ def __init__(self, Vocab vocab, entity_vector_length): @@ -46,6 +46,9 @@ cdef class InMemoryLookupKB(KnowledgeBase): self._alias_index = PreshMap(nr_aliases + 1) self._aliases_table = alias_vec(nr_aliases + 1) + def is_empty(self): + return len(self) == 0 + @classmethod def generate_from_disk( cls, path: Union[str, Path], exclude: Iterable[str] = SimpleFrozenList() diff --git a/spacy/lang/nl/stop_words.py b/spacy/lang/nl/stop_words.py index a2c6198e7..cd4fdefdf 100644 --- a/spacy/lang/nl/stop_words.py +++ b/spacy/lang/nl/stop_words.py @@ -15,7 +15,7 @@ STOP_WORDS = set( """ -aan af al alle alles allebei alleen allen als altijd ander anders andere anderen aangaangde aangezien achter achterna +aan af al alle alles allebei alleen allen als altijd ander anders andere anderen aangaande aangezien achter achterna afgelopen aldus alhoewel anderzijds ben bij bijna bijvoorbeeld behalve beide beiden beneden bent bepaald beter betere betreffende binnen binnenin boven diff --git a/spacy/lang/sr/__init__.py b/spacy/lang/sr/__init__.py index fd0c8c832..b99ce96ec 100644 --- a/spacy/lang/sr/__init__.py +++ b/spacy/lang/sr/__init__.py @@ -1,11 +1,14 @@ from .stop_words import STOP_WORDS from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS from .lex_attrs import LEX_ATTRS +from .punctuation import TOKENIZER_INFIXES, TOKENIZER_SUFFIXES from ...language import Language, BaseDefaults class SerbianDefaults(BaseDefaults): tokenizer_exceptions = TOKENIZER_EXCEPTIONS + infixes = TOKENIZER_INFIXES + suffixes = TOKENIZER_SUFFIXES lex_attr_getters = LEX_ATTRS stop_words = STOP_WORDS diff --git a/spacy/lang/sr/punctuation.py b/spacy/lang/sr/punctuation.py new file mode 100644 index 000000000..793a20ec2 --- /dev/null +++ b/spacy/lang/sr/punctuation.py @@ -0,0 +1,36 @@ +from ..char_classes import LIST_ELLIPSES, LIST_ICONS, LIST_PUNCT, LIST_QUOTES +from ..char_classes import CURRENCY, UNITS, PUNCT +from ..char_classes import CONCAT_QUOTES, ALPHA, ALPHA_LOWER, ALPHA_UPPER + + +_infixes = ( + LIST_ELLIPSES + + LIST_ICONS + + [ + r"(?<=[0-9])[+\-\*^](?=[0-9-])", + r"(?<=[{al}{q}])\.(?=[{au}{q}])".format( + al=ALPHA_LOWER, au=ALPHA_UPPER, q=CONCAT_QUOTES + ), + r"(?<=[{a}]),(?=[{a}])".format(a=ALPHA), + r"(?<=[{a}0-9])[:<>=/](?=[{a}])".format(a=ALPHA), + ] +) + +_suffixes = ( + LIST_PUNCT + + LIST_ELLIPSES + + LIST_QUOTES + + LIST_ICONS + + [ + r"(?<=[0-9])\+", + r"(?<=°[FfCcKk])\.", + r"(?<=[0-9])(?:{c})".format(c=CURRENCY), + r"(?<=[0-9])(?:{u})".format(u=UNITS), + r"(?<=[{a}{e}{p}(?:{q})])\.".format( + a=ALPHA, e=r"%²\-\+", q=CONCAT_QUOTES, p=PUNCT + ), + ] +) + +TOKENIZER_INFIXES = _infixes +TOKENIZER_SUFFIXES = _suffixes diff --git a/spacy/lang/sv/__init__.py b/spacy/lang/sv/__init__.py index 6963e8b79..28e5085a8 100644 --- a/spacy/lang/sv/__init__.py +++ b/spacy/lang/sv/__init__.py @@ -6,10 +6,7 @@ from .lex_attrs import LEX_ATTRS from .syntax_iterators import SYNTAX_ITERATORS from ...language import Language, BaseDefaults from ...pipeline import Lemmatizer - - -# Punctuation stolen from Danish -from ..da.punctuation import TOKENIZER_INFIXES, TOKENIZER_SUFFIXES +from .punctuation import TOKENIZER_INFIXES, TOKENIZER_SUFFIXES class SwedishDefaults(BaseDefaults): diff --git a/spacy/lang/sv/punctuation.py b/spacy/lang/sv/punctuation.py new file mode 100644 index 000000000..67f1bcdc4 --- /dev/null +++ b/spacy/lang/sv/punctuation.py @@ -0,0 +1,33 @@ +from ..char_classes import LIST_ELLIPSES, LIST_ICONS +from ..char_classes import CONCAT_QUOTES, ALPHA, ALPHA_LOWER, ALPHA_UPPER +from ..punctuation import TOKENIZER_SUFFIXES + + +_quotes = CONCAT_QUOTES.replace("'", "") + +_infixes = ( + LIST_ELLIPSES + + LIST_ICONS + + [ + r"(?<=[{al}])\.(?=[{au}])".format(al=ALPHA_LOWER, au=ALPHA_UPPER), + r"(?<=[{a}])[,!?](?=[{a}])".format(a=ALPHA), + r"(?<=[{a}])[<>=](?=[{a}])".format(a=ALPHA), + r"(?<=[{a}]):(?=[{a}])".format(a=ALPHA_UPPER), + r"(?<=[{a}]),(?=[{a}])".format(a=ALPHA), + r"(?<=[{a}])([{q}\)\]\(\[])(?=[{a}])".format(a=ALPHA, q=_quotes), + r"(?<=[{a}])--(?=[{a}])".format(a=ALPHA), + r"(?<=[{a}0-9])[<>=/](?=[{a}])".format(a=ALPHA), + r"(?<=[{a}0-9]):(?=[{a}])".format(a=ALPHA_UPPER), + ] +) + +_suffixes = [ + suffix + for suffix in TOKENIZER_SUFFIXES + if suffix not in ["'s", "'S", "’s", "’S", r"\'"] +] +_suffixes += [r"(?<=[^sSxXzZ])\'"] + + +TOKENIZER_INFIXES = _infixes +TOKENIZER_SUFFIXES = _suffixes diff --git a/spacy/language.py b/spacy/language.py index e0abfd5e7..9fdcf6328 100644 --- a/spacy/language.py +++ b/spacy/language.py @@ -104,7 +104,7 @@ def create_tokenizer() -> Callable[["Language"], Tokenizer]: @registry.misc("spacy.LookupsDataLoader.v1") def load_lookups_data(lang, tables): - util.logger.debug(f"Loading lookups from spacy-lookups-data: {tables}") + util.logger.debug("Loading lookups from spacy-lookups-data: %s", tables) lookups = load_lookups(lang=lang, tables=tables) return lookups @@ -1969,7 +1969,7 @@ class Language: pipe = self.get_pipe(pipe_name) pipe_cfg = self._pipe_configs[pipe_name] if listeners: - util.logger.debug(f"Replacing listeners of component '{pipe_name}'") + util.logger.debug("Replacing listeners of component '%s'", pipe_name) if len(list(listeners)) != len(pipe_listeners): # The number of listeners defined in the component model doesn't # match the listeners to replace, so we won't be able to update diff --git a/spacy/lexeme.pyi b/spacy/lexeme.pyi index 4fcaa82cf..9b7a6156a 100644 --- a/spacy/lexeme.pyi +++ b/spacy/lexeme.pyi @@ -25,7 +25,8 @@ class Lexeme: def orth_(self) -> str: ... @property def text(self) -> str: ... - lower: str + orth: int + lower: int norm: int shape: int prefix: int diff --git a/spacy/lexeme.pyx b/spacy/lexeme.pyx index 6c66effde..e70feaf9a 100644 --- a/spacy/lexeme.pyx +++ b/spacy/lexeme.pyx @@ -199,7 +199,7 @@ cdef class Lexeme: return self.orth_ property lower: - """RETURNS (str): Lowercase form of the lexeme.""" + """RETURNS (uint64): Lowercase form of the lexeme.""" def __get__(self): return self.c.lower diff --git a/spacy/matcher/dependencymatcher.pyx b/spacy/matcher/dependencymatcher.pyx index 74c2d002f..48fb3eb2a 100644 --- a/spacy/matcher/dependencymatcher.pyx +++ b/spacy/matcher/dependencymatcher.pyx @@ -82,8 +82,12 @@ cdef class DependencyMatcher: "$-": self._imm_left_sib, "$++": self._right_sib, "$--": self._left_sib, + ">+": self._imm_right_child, + ">-": self._imm_left_child, ">++": self._right_child, ">--": self._left_child, + "<+": self._imm_right_parent, + "<-": self._imm_left_parent, "<++": self._right_parent, "<--": self._left_parent, } @@ -427,11 +431,33 @@ cdef class DependencyMatcher: def _left_sib(self, doc, node): return [doc[child.i] for child in doc[node].head.children if child.i < node] + def _imm_right_child(self, doc, node): + for child in doc[node].rights: + if child.i == node + 1: + return [doc[child.i]] + return [] + + def _imm_left_child(self, doc, node): + for child in doc[node].lefts: + if child.i == node - 1: + return [doc[child.i]] + return [] + def _right_child(self, doc, node): - return [doc[child.i] for child in doc[node].children if child.i > node] + return [child for child in doc[node].rights] def _left_child(self, doc, node): - return [doc[child.i] for child in doc[node].children if child.i < node] + return [child for child in doc[node].lefts] + + def _imm_right_parent(self, doc, node): + if doc[node].head.i == node + 1: + return [doc[node].head] + return [] + + def _imm_left_parent(self, doc, node): + if doc[node].head.i == node - 1: + return [doc[node].head] + return [] def _right_parent(self, doc, node): if doc[node].head.i > node: diff --git a/spacy/matcher/levenshtein.pyx b/spacy/matcher/levenshtein.pyx index 8463d913d..e823ce99d 100644 --- a/spacy/matcher/levenshtein.pyx +++ b/spacy/matcher/levenshtein.pyx @@ -4,6 +4,8 @@ from libc.stdint cimport int64_t from typing import Optional +from ..util import registry + cdef extern from "polyleven.c": int64_t polyleven(PyObject *o1, PyObject *o2, int64_t k) @@ -13,3 +15,18 @@ cpdef int64_t levenshtein(a: str, b: str, k: Optional[int] = None): if k is None: k = -1 return polyleven(a, b, k) + + +cpdef bint levenshtein_compare(input_text: str, pattern_text: str, fuzzy: int = -1): + if fuzzy >= 0: + max_edits = fuzzy + else: + # allow at least two edits (to allow at least one transposition) and up + # to 30% of the pattern string length + max_edits = max(2, round(0.3 * len(pattern_text))) + return levenshtein(input_text, pattern_text, max_edits) <= max_edits + + +@registry.misc("spacy.levenshtein_compare.v1") +def make_levenshtein_compare(): + return levenshtein_compare diff --git a/spacy/matcher/matcher.pxd b/spacy/matcher/matcher.pxd index 455f978cc..51854d562 100644 --- a/spacy/matcher/matcher.pxd +++ b/spacy/matcher/matcher.pxd @@ -77,3 +77,4 @@ cdef class Matcher: cdef public object _extensions cdef public object _extra_predicates cdef public object _seen_attrs + cdef public object _fuzzy_compare diff --git a/spacy/matcher/matcher.pyi b/spacy/matcher/matcher.pyi index 390629ff8..48922865b 100644 --- a/spacy/matcher/matcher.pyi +++ b/spacy/matcher/matcher.pyi @@ -5,7 +5,12 @@ from ..vocab import Vocab from ..tokens import Doc, Span class Matcher: - def __init__(self, vocab: Vocab, validate: bool = ...) -> None: ... + def __init__( + self, + vocab: Vocab, + validate: bool = ..., + fuzzy_compare: Callable[[str, str, int], bool] = ..., + ) -> None: ... def __reduce__(self) -> Any: ... def __len__(self) -> int: ... def __contains__(self, key: str) -> bool: ... diff --git a/spacy/matcher/matcher.pyx b/spacy/matcher/matcher.pyx index c4a057ca0..b886bd2ec 100644 --- a/spacy/matcher/matcher.pyx +++ b/spacy/matcher/matcher.pyx @@ -1,4 +1,4 @@ -# cython: infer_types=True, profile=True +# cython: binding=True, infer_types=True, profile=True from typing import List, Iterable from libcpp.vector cimport vector @@ -20,10 +20,12 @@ from ..tokens.token cimport Token from ..tokens.morphanalysis cimport MorphAnalysis from ..attrs cimport ID, attr_id_t, NULL_ATTR, ORTH, POS, TAG, DEP, LEMMA, MORPH, ENT_IOB +from .levenshtein import levenshtein_compare from ..schemas import validate_token_pattern from ..errors import Errors, MatchPatternError, Warnings from ..strings import get_string_id from ..attrs import IDS +from ..util import registry DEF PADDING = 5 @@ -36,11 +38,13 @@ cdef class Matcher: USAGE: https://spacy.io/usage/rule-based-matching """ - def __init__(self, vocab, validate=True): + def __init__(self, vocab, validate=True, *, fuzzy_compare=levenshtein_compare): """Create the Matcher. vocab (Vocab): The vocabulary object, which must be shared with the - documents the matcher will operate on. + validate (bool): Validate all patterns added to this matcher. + fuzzy_compare (Callable[[str, str, int], bool]): The comparison method + for the FUZZY operators. """ self._extra_predicates = [] self._patterns = {} @@ -51,9 +55,10 @@ cdef class Matcher: self.vocab = vocab self.mem = Pool() self.validate = validate + self._fuzzy_compare = fuzzy_compare def __reduce__(self): - data = (self.vocab, self._patterns, self._callbacks) + data = (self.vocab, self._patterns, self._callbacks, self.validate, self._fuzzy_compare) return (unpickle_matcher, data, None, None) def __len__(self): @@ -128,7 +133,7 @@ cdef class Matcher: for pattern in patterns: try: specs = _preprocess_pattern(pattern, self.vocab, - self._extensions, self._extra_predicates) + self._extensions, self._extra_predicates, self._fuzzy_compare) self.patterns.push_back(init_pattern(self.mem, key, specs)) for spec in specs: for attr, _ in spec[1]: @@ -326,8 +331,8 @@ cdef class Matcher: return key -def unpickle_matcher(vocab, patterns, callbacks): - matcher = Matcher(vocab) +def unpickle_matcher(vocab, patterns, callbacks, validate, fuzzy_compare): + matcher = Matcher(vocab, validate=validate, fuzzy_compare=fuzzy_compare) for key, pattern in patterns.items(): callback = callbacks.get(key, None) matcher.add(key, pattern, on_match=callback) @@ -754,7 +759,7 @@ cdef attr_t get_ent_id(const TokenPatternC* pattern) nogil: return id_attr.value -def _preprocess_pattern(token_specs, vocab, extensions_table, extra_predicates): +def _preprocess_pattern(token_specs, vocab, extensions_table, extra_predicates, fuzzy_compare): """This function interprets the pattern, converting the various bits of syntactic sugar before we compile it into a struct with init_pattern. @@ -781,7 +786,7 @@ def _preprocess_pattern(token_specs, vocab, extensions_table, extra_predicates): ops = _get_operators(spec) attr_values = _get_attr_values(spec, string_store) extensions = _get_extensions(spec, string_store, extensions_table) - predicates = _get_extra_predicates(spec, extra_predicates, vocab) + predicates = _get_extra_predicates(spec, extra_predicates, vocab, fuzzy_compare) for op in ops: tokens.append((op, list(attr_values), list(extensions), list(predicates), token_idx)) return tokens @@ -823,19 +828,53 @@ def _get_attr_values(spec, string_store): return attr_values +def _predicate_cache_key(attr, predicate, value, *, regex=False, fuzzy=None): + # tuple order affects performance + return (attr, regex, fuzzy, predicate, srsly.json_dumps(value, sort_keys=True)) + + # These predicate helper classes are used to match the REGEX, IN, >= etc # extensions to the matcher introduced in #3173. +class _FuzzyPredicate: + operators = ("FUZZY", "FUZZY1", "FUZZY2", "FUZZY3", "FUZZY4", "FUZZY5", + "FUZZY6", "FUZZY7", "FUZZY8", "FUZZY9") + + def __init__(self, i, attr, value, predicate, is_extension=False, vocab=None, + regex=False, fuzzy=None, fuzzy_compare=None): + self.i = i + self.attr = attr + self.value = value + self.predicate = predicate + self.is_extension = is_extension + if self.predicate not in self.operators: + raise ValueError(Errors.E126.format(good=self.operators, bad=self.predicate)) + fuzz = self.predicate[len("FUZZY"):] # number after prefix + self.fuzzy = int(fuzz) if fuzz else -1 + self.fuzzy_compare = fuzzy_compare + self.key = _predicate_cache_key(self.attr, self.predicate, value, fuzzy=self.fuzzy) + + def __call__(self, Token token): + if self.is_extension: + value = token._.get(self.attr) + else: + value = token.vocab.strings[get_token_attr_for_matcher(token.c, self.attr)] + if self.value == value: + return True + return self.fuzzy_compare(value, self.value, self.fuzzy) + + class _RegexPredicate: operators = ("REGEX",) - def __init__(self, i, attr, value, predicate, is_extension=False, vocab=None): + def __init__(self, i, attr, value, predicate, is_extension=False, vocab=None, + regex=False, fuzzy=None, fuzzy_compare=None): self.i = i self.attr = attr self.value = re.compile(value) self.predicate = predicate self.is_extension = is_extension - self.key = (attr, self.predicate, srsly.json_dumps(value, sort_keys=True)) + self.key = _predicate_cache_key(self.attr, self.predicate, value) if self.predicate not in self.operators: raise ValueError(Errors.E126.format(good=self.operators, bad=self.predicate)) @@ -850,18 +889,28 @@ class _RegexPredicate: class _SetPredicate: operators = ("IN", "NOT_IN", "IS_SUBSET", "IS_SUPERSET", "INTERSECTS") - def __init__(self, i, attr, value, predicate, is_extension=False, vocab=None): + def __init__(self, i, attr, value, predicate, is_extension=False, vocab=None, + regex=False, fuzzy=None, fuzzy_compare=None): self.i = i self.attr = attr self.vocab = vocab + self.regex = regex + self.fuzzy = fuzzy + self.fuzzy_compare = fuzzy_compare if self.attr == MORPH: # normalize morph strings self.value = set(self.vocab.morphology.add(v) for v in value) else: - self.value = set(get_string_id(v) for v in value) + if self.regex: + self.value = set(re.compile(v) for v in value) + elif self.fuzzy is not None: + # add to string store + self.value = set(self.vocab.strings.add(v) for v in value) + else: + self.value = set(get_string_id(v) for v in value) self.predicate = predicate self.is_extension = is_extension - self.key = (attr, self.predicate, srsly.json_dumps(value, sort_keys=True)) + self.key = _predicate_cache_key(self.attr, self.predicate, value, regex=self.regex, fuzzy=self.fuzzy) if self.predicate not in self.operators: raise ValueError(Errors.E126.format(good=self.operators, bad=self.predicate)) @@ -889,9 +938,29 @@ class _SetPredicate: return False if self.predicate == "IN": - return value in self.value + if self.regex: + value = self.vocab.strings[value] + return any(bool(v.search(value)) for v in self.value) + elif self.fuzzy is not None: + value = self.vocab.strings[value] + return any(self.fuzzy_compare(value, self.vocab.strings[v], self.fuzzy) + for v in self.value) + elif value in self.value: + return True + else: + return False elif self.predicate == "NOT_IN": - return value not in self.value + if self.regex: + value = self.vocab.strings[value] + return not any(bool(v.search(value)) for v in self.value) + elif self.fuzzy is not None: + value = self.vocab.strings[value] + return not any(self.fuzzy_compare(value, self.vocab.strings[v], self.fuzzy) + for v in self.value) + elif value in self.value: + return False + else: + return True elif self.predicate == "IS_SUBSET": return value <= self.value elif self.predicate == "IS_SUPERSET": @@ -906,13 +975,14 @@ class _SetPredicate: class _ComparisonPredicate: operators = ("==", "!=", ">=", "<=", ">", "<") - def __init__(self, i, attr, value, predicate, is_extension=False, vocab=None): + def __init__(self, i, attr, value, predicate, is_extension=False, vocab=None, + regex=False, fuzzy=None, fuzzy_compare=None): self.i = i self.attr = attr self.value = value self.predicate = predicate self.is_extension = is_extension - self.key = (attr, self.predicate, srsly.json_dumps(value, sort_keys=True)) + self.key = _predicate_cache_key(self.attr, self.predicate, value) if self.predicate not in self.operators: raise ValueError(Errors.E126.format(good=self.operators, bad=self.predicate)) @@ -935,7 +1005,7 @@ class _ComparisonPredicate: return value < self.value -def _get_extra_predicates(spec, extra_predicates, vocab): +def _get_extra_predicates(spec, extra_predicates, vocab, fuzzy_compare): predicate_types = { "REGEX": _RegexPredicate, "IN": _SetPredicate, @@ -949,6 +1019,16 @@ def _get_extra_predicates(spec, extra_predicates, vocab): "<=": _ComparisonPredicate, ">": _ComparisonPredicate, "<": _ComparisonPredicate, + "FUZZY": _FuzzyPredicate, + "FUZZY1": _FuzzyPredicate, + "FUZZY2": _FuzzyPredicate, + "FUZZY3": _FuzzyPredicate, + "FUZZY4": _FuzzyPredicate, + "FUZZY5": _FuzzyPredicate, + "FUZZY6": _FuzzyPredicate, + "FUZZY7": _FuzzyPredicate, + "FUZZY8": _FuzzyPredicate, + "FUZZY9": _FuzzyPredicate, } seen_predicates = {pred.key: pred.i for pred in extra_predicates} output = [] @@ -966,22 +1046,47 @@ def _get_extra_predicates(spec, extra_predicates, vocab): attr = "ORTH" attr = IDS.get(attr.upper()) if isinstance(value, dict): - processed = False - value_with_upper_keys = {k.upper(): v for k, v in value.items()} - for type_, cls in predicate_types.items(): - if type_ in value_with_upper_keys: - predicate = cls(len(extra_predicates), attr, value_with_upper_keys[type_], type_, vocab=vocab) - # Don't create a redundant predicates. - # This helps with efficiency, as we're caching the results. - if predicate.key in seen_predicates: - output.append(seen_predicates[predicate.key]) - else: - extra_predicates.append(predicate) - output.append(predicate.i) - seen_predicates[predicate.key] = predicate.i - processed = True - if not processed: - warnings.warn(Warnings.W035.format(pattern=value)) + output.extend(_get_extra_predicates_dict(attr, value, vocab, predicate_types, + extra_predicates, seen_predicates, fuzzy_compare=fuzzy_compare)) + return output + + +def _get_extra_predicates_dict(attr, value_dict, vocab, predicate_types, + extra_predicates, seen_predicates, regex=False, fuzzy=None, fuzzy_compare=None): + output = [] + for type_, value in value_dict.items(): + type_ = type_.upper() + cls = predicate_types.get(type_) + if cls is None: + warnings.warn(Warnings.W035.format(pattern=value_dict)) + # ignore unrecognized predicate type + continue + elif cls == _RegexPredicate: + if isinstance(value, dict): + # add predicates inside regex operator + output.extend(_get_extra_predicates_dict(attr, value, vocab, predicate_types, + extra_predicates, seen_predicates, + regex=True)) + continue + elif cls == _FuzzyPredicate: + if isinstance(value, dict): + # add predicates inside fuzzy operator + fuzz = type_[len("FUZZY"):] # number after prefix + fuzzy_val = int(fuzz) if fuzz else -1 + output.extend(_get_extra_predicates_dict(attr, value, vocab, predicate_types, + extra_predicates, seen_predicates, + fuzzy=fuzzy_val, fuzzy_compare=fuzzy_compare)) + continue + predicate = cls(len(extra_predicates), attr, value, type_, vocab=vocab, + regex=regex, fuzzy=fuzzy, fuzzy_compare=fuzzy_compare) + # Don't create redundant predicates. + # This helps with efficiency, as we're caching the results. + if predicate.key in seen_predicates: + output.append(seen_predicates[predicate.key]) + else: + extra_predicates.append(predicate) + output.append(predicate.i) + seen_predicates[predicate.key] = predicate.i return output @@ -992,7 +1097,7 @@ def _get_extension_extra_predicates(spec, extra_predicates, predicate_types, if isinstance(value, dict): for type_, cls in predicate_types.items(): if type_ in value: - key = (attr, type_, srsly.json_dumps(value[type_], sort_keys=True)) + key = _predicate_cache_key(attr, type_, value[type_]) if key in seen_predicates: output.append(seen_predicates[key]) else: diff --git a/spacy/ml/models/entity_linker.py b/spacy/ml/models/entity_linker.py index b5f455cdc..90bb0a6d8 100644 --- a/spacy/ml/models/entity_linker.py +++ b/spacy/ml/models/entity_linker.py @@ -89,6 +89,14 @@ def load_kb( return kb_from_file +@registry.misc("spacy.EmptyKB.v2") +def empty_kb_for_config() -> Callable[[Vocab, int], KnowledgeBase]: + def empty_kb_factory(vocab: Vocab, entity_vector_length: int): + return InMemoryLookupKB(vocab=vocab, entity_vector_length=entity_vector_length) + + return empty_kb_factory + + @registry.misc("spacy.EmptyKB.v1") def empty_kb( entity_vector_length: int, diff --git a/spacy/ml/models/multi_task.py b/spacy/ml/models/multi_task.py index a7d67c6dd..7eb13b608 100644 --- a/spacy/ml/models/multi_task.py +++ b/spacy/ml/models/multi_task.py @@ -1,5 +1,5 @@ from typing import Any, Optional, Iterable, Tuple, List, Callable, TYPE_CHECKING, cast -from thinc.types import Floats2d +from thinc.types import Floats2d, Ints1d from thinc.api import chain, Maxout, LayerNorm, Softmax, Linear, zero_init, Model from thinc.api import MultiSoftmax, list2array from thinc.api import to_categorical, CosineDistance, L2Distance @@ -7,7 +7,8 @@ from thinc.loss import Loss from ...util import registry, OOV_RANK from ...errors import Errors -from ...attrs import ID +from ...attrs import ID, ORTH +from ...vectors import Mode as VectorsMode import numpy from functools import partial @@ -67,14 +68,23 @@ def get_vectors_loss(ops, docs, prediction, distance): """Compute a loss based on a distance between the documents' vectors and the prediction. """ - # The simplest way to implement this would be to vstack the - # token.vector values, but that's a bit inefficient, especially on GPU. - # Instead we fetch the index into the vectors table for each of our tokens, - # and look them up all at once. This prevents data copying. - ids = ops.flatten([doc.to_array(ID).ravel() for doc in docs]) - target = docs[0].vocab.vectors.data[ids] - target[ids == OOV_RANK] = 0 - d_target, loss = distance(prediction, target) + vocab = docs[0].vocab + if vocab.vectors.mode == VectorsMode.default: + # The simplest way to implement this would be to vstack the + # token.vector values, but that's a bit inefficient, especially on GPU. + # Instead we fetch the index into the vectors table for each of our + # tokens, and look them up all at once. This prevents data copying. + ids = ops.flatten([doc.to_array(ID).ravel() for doc in docs]) + target = docs[0].vocab.vectors.data[ids] + target[ids == OOV_RANK] = 0 + d_target, loss = distance(prediction, target) + elif vocab.vectors.mode == VectorsMode.floret: + keys = ops.flatten([cast(Ints1d, doc.to_array(ORTH)) for doc in docs]) + target = vocab.vectors.get_batch(keys) + target = ops.as_contig(target) + d_target, loss = distance(prediction, target) + else: + raise ValueError(Errors.E850.format(mode=vocab.vectors.mode)) return loss, d_target diff --git a/spacy/pipeline/edit_tree_lemmatizer.py b/spacy/pipeline/edit_tree_lemmatizer.py index a56c9975e..332badd8c 100644 --- a/spacy/pipeline/edit_tree_lemmatizer.py +++ b/spacy/pipeline/edit_tree_lemmatizer.py @@ -5,8 +5,8 @@ from itertools import islice import numpy as np import srsly -from thinc.api import Config, Model, SequenceCategoricalCrossentropy -from thinc.types import Floats2d, Ints1d, Ints2d +from thinc.api import Config, Model, SequenceCategoricalCrossentropy, NumpyOps +from thinc.types import Floats2d, Ints2d from ._edit_tree_internals.edit_trees import EditTrees from ._edit_tree_internals.schemas import validate_edit_tree @@ -20,6 +20,10 @@ from ..vocab import Vocab from .. import util +# The cutoff value of *top_k* above which an alternative method is used to process guesses. +TOP_K_GUARDRAIL = 20 + + default_model_config = """ [model] @architectures = "spacy.Tagger.v2" @@ -115,6 +119,7 @@ class EditTreeLemmatizer(TrainablePipe): self.cfg: Dict[str, Any] = {"labels": []} self.scorer = scorer + self.numpy_ops = NumpyOps() def get_loss( self, examples: Iterable[Example], scores: List[Floats2d] @@ -128,7 +133,7 @@ class EditTreeLemmatizer(TrainablePipe): for (predicted, gold_lemma) in zip( eg.predicted, eg.get_aligned("LEMMA", as_string=True) ): - if gold_lemma is None: + if gold_lemma is None or gold_lemma == "": label = -1 else: tree_id = self.trees.add(predicted.text, gold_lemma) @@ -144,6 +149,18 @@ class EditTreeLemmatizer(TrainablePipe): return float(loss), d_scores def predict(self, docs: Iterable[Doc]) -> List[Ints2d]: + if self.top_k == 1: + scores2guesses = self._scores2guesses_top_k_equals_1 + elif self.top_k <= TOP_K_GUARDRAIL: + scores2guesses = self._scores2guesses_top_k_greater_1 + else: + scores2guesses = self._scores2guesses_top_k_guardrail + # The behaviour of *_scores2guesses_top_k_greater_1()* is efficient for values + # of *top_k>1* that are likely to be useful when the edit tree lemmatizer is used + # for its principal purpose of lemmatizing tokens. However, the code could also + # be used for other purposes, and with very large values of *top_k* the method + # becomes inefficient. In such cases, *_scores2guesses_top_k_guardrail()* is used + # instead. n_docs = len(list(docs)) if not any(len(doc) for doc in docs): # Handle cases where there are no tokens in any docs. @@ -153,20 +170,52 @@ class EditTreeLemmatizer(TrainablePipe): return guesses scores = self.model.predict(docs) assert len(scores) == n_docs - guesses = self._scores2guesses(docs, scores) + guesses = scores2guesses(docs, scores) assert len(guesses) == n_docs return guesses - def _scores2guesses(self, docs, scores): + def _scores2guesses_top_k_equals_1(self, docs, scores): guesses = [] for doc, doc_scores in zip(docs, scores): - if self.top_k == 1: - doc_guesses = doc_scores.argmax(axis=1).reshape(-1, 1) - else: - doc_guesses = np.argsort(doc_scores)[..., : -self.top_k - 1 : -1] + doc_guesses = doc_scores.argmax(axis=1) + doc_guesses = self.numpy_ops.asarray(doc_guesses) - if not isinstance(doc_guesses, np.ndarray): - doc_guesses = doc_guesses.get() + doc_compat_guesses = [] + for i, token in enumerate(doc): + tree_id = self.cfg["labels"][doc_guesses[i]] + if self.trees.apply(tree_id, token.text) is not None: + doc_compat_guesses.append(tree_id) + else: + doc_compat_guesses.append(-1) + guesses.append(np.array(doc_compat_guesses)) + + return guesses + + def _scores2guesses_top_k_greater_1(self, docs, scores): + guesses = [] + top_k = min(self.top_k, len(self.labels)) + for doc, doc_scores in zip(docs, scores): + doc_scores = self.numpy_ops.asarray(doc_scores) + doc_compat_guesses = [] + for i, token in enumerate(doc): + for _ in range(top_k): + candidate = int(doc_scores[i].argmax()) + candidate_tree_id = self.cfg["labels"][candidate] + if self.trees.apply(candidate_tree_id, token.text) is not None: + doc_compat_guesses.append(candidate_tree_id) + break + doc_scores[i, candidate] = np.finfo(np.float32).min + else: + doc_compat_guesses.append(-1) + guesses.append(np.array(doc_compat_guesses)) + + return guesses + + def _scores2guesses_top_k_guardrail(self, docs, scores): + guesses = [] + for doc, doc_scores in zip(docs, scores): + doc_guesses = np.argsort(doc_scores)[..., : -self.top_k - 1 : -1] + doc_guesses = self.numpy_ops.asarray(doc_guesses) doc_compat_guesses = [] for token, candidates in zip(doc, doc_guesses): diff --git a/spacy/pipeline/entity_linker.py b/spacy/pipeline/entity_linker.py index 074168aef..2cbf45320 100644 --- a/spacy/pipeline/entity_linker.py +++ b/spacy/pipeline/entity_linker.py @@ -265,7 +265,7 @@ class EntityLinker(TrainablePipe): # Raise an error if the knowledge base is not initialized. if self.kb is None: raise ValueError(Errors.E1018.format(name=self.name)) - if len(self.kb) == 0: + if hasattr(self.kb, "is_empty") and self.kb.is_empty(): raise ValueError(Errors.E139.format(name=self.name)) def initialize( diff --git a/spacy/pipeline/entityruler.py b/spacy/pipeline/entityruler.py index 8154a077d..6a3755533 100644 --- a/spacy/pipeline/entityruler.py +++ b/spacy/pipeline/entityruler.py @@ -11,6 +11,7 @@ from ..errors import Errors, Warnings from ..util import ensure_path, to_disk, from_disk, SimpleFrozenList, registry from ..tokens import Doc, Span from ..matcher import Matcher, PhraseMatcher +from ..matcher.levenshtein import levenshtein_compare from ..scorer import get_ner_prf @@ -23,6 +24,7 @@ PatternType = Dict[str, Union[str, List[Dict[str, Any]]]] assigns=["doc.ents", "token.ent_type", "token.ent_iob"], default_config={ "phrase_matcher_attr": None, + "matcher_fuzzy_compare": {"@misc": "spacy.levenshtein_compare.v1"}, "validate": False, "overwrite_ents": False, "ent_id_sep": DEFAULT_ENT_ID_SEP, @@ -39,6 +41,7 @@ def make_entity_ruler( nlp: Language, name: str, phrase_matcher_attr: Optional[Union[int, str]], + matcher_fuzzy_compare: Callable, validate: bool, overwrite_ents: bool, ent_id_sep: str, @@ -48,6 +51,7 @@ def make_entity_ruler( nlp, name, phrase_matcher_attr=phrase_matcher_attr, + matcher_fuzzy_compare=matcher_fuzzy_compare, validate=validate, overwrite_ents=overwrite_ents, ent_id_sep=ent_id_sep, @@ -81,6 +85,7 @@ class EntityRuler(Pipe): name: str = "entity_ruler", *, phrase_matcher_attr: Optional[Union[int, str]] = None, + matcher_fuzzy_compare: Callable = levenshtein_compare, validate: bool = False, overwrite_ents: bool = False, ent_id_sep: str = DEFAULT_ENT_ID_SEP, @@ -99,7 +104,10 @@ class EntityRuler(Pipe): added. Used to disable the current entity ruler while creating phrase patterns with the nlp object. phrase_matcher_attr (int / str): Token attribute to match on, passed - to the internal PhraseMatcher as `attr` + to the internal PhraseMatcher as `attr`. + matcher_fuzzy_compare (Callable): The fuzzy comparison method for the + internal Matcher. Defaults to + spacy.matcher.levenshtein.levenshtein_compare. validate (bool): Whether patterns should be validated, passed to Matcher and PhraseMatcher as `validate` patterns (iterable): Optional patterns to load in. @@ -117,7 +125,10 @@ class EntityRuler(Pipe): self.token_patterns = defaultdict(list) # type: ignore self.phrase_patterns = defaultdict(list) # type: ignore self._validate = validate - self.matcher = Matcher(nlp.vocab, validate=validate) + self.matcher_fuzzy_compare = matcher_fuzzy_compare + self.matcher = Matcher( + nlp.vocab, validate=validate, fuzzy_compare=self.matcher_fuzzy_compare + ) self.phrase_matcher_attr = phrase_matcher_attr self.phrase_matcher = PhraseMatcher( nlp.vocab, attr=self.phrase_matcher_attr, validate=validate @@ -337,7 +348,11 @@ class EntityRuler(Pipe): self.token_patterns = defaultdict(list) self.phrase_patterns = defaultdict(list) self._ent_ids = defaultdict(tuple) - self.matcher = Matcher(self.nlp.vocab, validate=self._validate) + self.matcher = Matcher( + self.nlp.vocab, + validate=self._validate, + fuzzy_compare=self.matcher_fuzzy_compare, + ) self.phrase_matcher = PhraseMatcher( self.nlp.vocab, attr=self.phrase_matcher_attr, validate=self._validate ) @@ -431,7 +446,8 @@ class EntityRuler(Pipe): self.overwrite = cfg.get("overwrite", False) self.phrase_matcher_attr = cfg.get("phrase_matcher_attr", None) self.phrase_matcher = PhraseMatcher( - self.nlp.vocab, attr=self.phrase_matcher_attr + self.nlp.vocab, + attr=self.phrase_matcher_attr, ) self.ent_id_sep = cfg.get("ent_id_sep", DEFAULT_ENT_ID_SEP) else: diff --git a/spacy/pipeline/morphologizer.pyx b/spacy/pipeline/morphologizer.pyx index 24f98508f..be8f82212 100644 --- a/spacy/pipeline/morphologizer.pyx +++ b/spacy/pipeline/morphologizer.pyx @@ -52,7 +52,8 @@ DEFAULT_MORPH_MODEL = Config().from_str(default_model_config)["model"] @Language.factory( "morphologizer", assigns=["token.morph", "token.pos"], - default_config={"model": DEFAULT_MORPH_MODEL, "overwrite": True, "extend": False, "scorer": {"@scorers": "spacy.morphologizer_scorer.v1"}}, + default_config={"model": DEFAULT_MORPH_MODEL, "overwrite": True, "extend": False, + "scorer": {"@scorers": "spacy.morphologizer_scorer.v1"}, "label_smoothing": 0.0}, default_score_weights={"pos_acc": 0.5, "morph_acc": 0.5, "morph_per_feat": None}, ) def make_morphologizer( @@ -61,9 +62,10 @@ def make_morphologizer( name: str, overwrite: bool, extend: bool, + label_smoothing: float, scorer: Optional[Callable], ): - return Morphologizer(nlp.vocab, model, name, overwrite=overwrite, extend=extend, scorer=scorer) + return Morphologizer(nlp.vocab, model, name, overwrite=overwrite, extend=extend, label_smoothing=label_smoothing, scorer=scorer) def morphologizer_score(examples, **kwargs): @@ -94,6 +96,7 @@ class Morphologizer(Tagger): *, overwrite: bool = BACKWARD_OVERWRITE, extend: bool = BACKWARD_EXTEND, + label_smoothing: float = 0.0, scorer: Optional[Callable] = morphologizer_score, ): """Initialize a morphologizer. @@ -121,6 +124,7 @@ class Morphologizer(Tagger): "labels_pos": {}, "overwrite": overwrite, "extend": extend, + "label_smoothing": label_smoothing, } self.cfg = dict(sorted(cfg.items())) self.scorer = scorer @@ -270,7 +274,8 @@ class Morphologizer(Tagger): DOCS: https://spacy.io/api/morphologizer#get_loss """ validate_examples(examples, "Morphologizer.get_loss") - loss_func = SequenceCategoricalCrossentropy(names=self.labels, normalize=False) + loss_func = SequenceCategoricalCrossentropy(names=self.labels, normalize=False, + label_smoothing=self.cfg["label_smoothing"]) truths = [] for eg in examples: eg_truths = [] diff --git a/spacy/pipeline/span_ruler.py b/spacy/pipeline/span_ruler.py index 807a4ffe5..b0669c0ef 100644 --- a/spacy/pipeline/span_ruler.py +++ b/spacy/pipeline/span_ruler.py @@ -13,6 +13,7 @@ from ..util import ensure_path, SimpleFrozenList, registry from ..tokens import Doc, Span from ..scorer import Scorer from ..matcher import Matcher, PhraseMatcher +from ..matcher.levenshtein import levenshtein_compare from .. import util PatternType = Dict[str, Union[str, List[Dict[str, Any]]]] @@ -28,6 +29,7 @@ DEFAULT_SPANS_KEY = "ruler" "overwrite_ents": False, "scorer": {"@scorers": "spacy.entity_ruler_scorer.v1"}, "ent_id_sep": "__unused__", + "matcher_fuzzy_compare": {"@misc": "spacy.levenshtein_compare.v1"}, }, default_score_weights={ "ents_f": 1.0, @@ -40,6 +42,7 @@ def make_entity_ruler( nlp: Language, name: str, phrase_matcher_attr: Optional[Union[int, str]], + matcher_fuzzy_compare: Callable, validate: bool, overwrite_ents: bool, scorer: Optional[Callable], @@ -57,6 +60,7 @@ def make_entity_ruler( annotate_ents=True, ents_filter=ents_filter, phrase_matcher_attr=phrase_matcher_attr, + matcher_fuzzy_compare=matcher_fuzzy_compare, validate=validate, overwrite=False, scorer=scorer, @@ -72,6 +76,7 @@ def make_entity_ruler( "annotate_ents": False, "ents_filter": {"@misc": "spacy.first_longest_spans_filter.v1"}, "phrase_matcher_attr": None, + "matcher_fuzzy_compare": {"@misc": "spacy.levenshtein_compare.v1"}, "validate": False, "overwrite": True, "scorer": { @@ -94,6 +99,7 @@ def make_span_ruler( annotate_ents: bool, ents_filter: Callable[[Iterable[Span], Iterable[Span]], Iterable[Span]], phrase_matcher_attr: Optional[Union[int, str]], + matcher_fuzzy_compare: Callable, validate: bool, overwrite: bool, scorer: Optional[Callable], @@ -106,6 +112,7 @@ def make_span_ruler( annotate_ents=annotate_ents, ents_filter=ents_filter, phrase_matcher_attr=phrase_matcher_attr, + matcher_fuzzy_compare=matcher_fuzzy_compare, validate=validate, overwrite=overwrite, scorer=scorer, @@ -170,7 +177,7 @@ def prioritize_existing_ents_filter( @registry.misc("spacy.prioritize_existing_ents_filter.v1") -def make_preverse_existing_ents_filter(): +def make_preserve_existing_ents_filter(): return prioritize_existing_ents_filter @@ -216,6 +223,7 @@ class SpanRuler(Pipe): [Iterable[Span], Iterable[Span]], Iterable[Span] ] = util.filter_chain_spans, phrase_matcher_attr: Optional[Union[int, str]] = None, + matcher_fuzzy_compare: Callable = levenshtein_compare, validate: bool = False, overwrite: bool = False, scorer: Optional[Callable] = partial( @@ -246,6 +254,9 @@ class SpanRuler(Pipe): phrase_matcher_attr (Optional[Union[int, str]]): Token attribute to match on, passed to the internal PhraseMatcher as `attr`. Defaults to `None`. + matcher_fuzzy_compare (Callable): The fuzzy comparison method for the + internal Matcher. Defaults to + spacy.matcher.levenshtein.levenshtein_compare. validate (bool): Whether patterns should be validated, passed to Matcher and PhraseMatcher as `validate`. overwrite (bool): Whether to remove any existing spans under this spans @@ -266,6 +277,7 @@ class SpanRuler(Pipe): self.spans_filter = spans_filter self.ents_filter = ents_filter self.scorer = scorer + self.matcher_fuzzy_compare = matcher_fuzzy_compare self._match_label_id_map: Dict[int, Dict[str, str]] = {} self.clear() @@ -451,7 +463,11 @@ class SpanRuler(Pipe): DOCS: https://spacy.io/api/spanruler#clear """ self._patterns: List[PatternType] = [] - self.matcher: Matcher = Matcher(self.nlp.vocab, validate=self.validate) + self.matcher: Matcher = Matcher( + self.nlp.vocab, + validate=self.validate, + fuzzy_compare=self.matcher_fuzzy_compare, + ) self.phrase_matcher: PhraseMatcher = PhraseMatcher( self.nlp.vocab, attr=self.phrase_matcher_attr, diff --git a/spacy/pipeline/spancat.py b/spacy/pipeline/spancat.py index a3388e81a..5a087e42a 100644 --- a/spacy/pipeline/spancat.py +++ b/spacy/pipeline/spancat.py @@ -1,4 +1,6 @@ -from typing import List, Dict, Callable, Tuple, Optional, Iterable, Any +from typing import List, Dict, Callable, Tuple, Optional, Iterable, Any, cast, Union +from dataclasses import dataclass +from functools import partial from thinc.api import Config, Model, get_current_ops, set_dropout_rate, Ops from thinc.api import Optimizer from thinc.types import Ragged, Ints2d, Floats2d @@ -43,7 +45,36 @@ maxout_pieces = 3 depth = 4 """ +spancat_singlelabel_default_config = """ +[model] +@architectures = "spacy.SpanCategorizer.v1" +scorer = {"@layers": "Softmax.v2"} + +[model.reducer] +@layers = spacy.mean_max_reducer.v1 +hidden_size = 128 + +[model.tok2vec] +@architectures = "spacy.Tok2Vec.v2" +[model.tok2vec.embed] +@architectures = "spacy.MultiHashEmbed.v1" +width = 96 +rows = [5000, 1000, 2500, 1000] +attrs = ["NORM", "PREFIX", "SUFFIX", "SHAPE"] +include_static_vectors = false + +[model.tok2vec.encode] +@architectures = "spacy.MaxoutWindowEncoder.v2" +width = ${model.tok2vec.embed.width} +window_size = 1 +maxout_pieces = 3 +depth = 4 +""" + DEFAULT_SPANCAT_MODEL = Config().from_str(spancat_default_config)["model"] +DEFAULT_SPANCAT_SINGLELABEL_MODEL = Config().from_str( + spancat_singlelabel_default_config +)["model"] @runtime_checkable @@ -52,39 +83,42 @@ class Suggester(Protocol): ... +def ngram_suggester( + docs: Iterable[Doc], sizes: List[int], *, ops: Optional[Ops] = None +) -> Ragged: + if ops is None: + ops = get_current_ops() + spans = [] + lengths = [] + for doc in docs: + starts = ops.xp.arange(len(doc), dtype="i") + starts = starts.reshape((-1, 1)) + length = 0 + for size in sizes: + if size <= len(doc): + starts_size = starts[: len(doc) - (size - 1)] + spans.append(ops.xp.hstack((starts_size, starts_size + size))) + length += spans[-1].shape[0] + if spans: + assert spans[-1].ndim == 2, spans[-1].shape + lengths.append(length) + lengths_array = ops.asarray1i(lengths) + if len(spans) > 0: + output = Ragged(ops.xp.vstack(spans), lengths_array) + else: + output = Ragged(ops.xp.zeros((0, 0), dtype="i"), lengths_array) + + assert output.dataXd.ndim == 2 + return output + + @registry.misc("spacy.ngram_suggester.v1") def build_ngram_suggester(sizes: List[int]) -> Suggester: """Suggest all spans of the given lengths. Spans are returned as a ragged array of integers. The array has two columns, indicating the start and end position.""" - def ngram_suggester(docs: Iterable[Doc], *, ops: Optional[Ops] = None) -> Ragged: - if ops is None: - ops = get_current_ops() - spans = [] - lengths = [] - for doc in docs: - starts = ops.xp.arange(len(doc), dtype="i") - starts = starts.reshape((-1, 1)) - length = 0 - for size in sizes: - if size <= len(doc): - starts_size = starts[: len(doc) - (size - 1)] - spans.append(ops.xp.hstack((starts_size, starts_size + size))) - length += spans[-1].shape[0] - if spans: - assert spans[-1].ndim == 2, spans[-1].shape - lengths.append(length) - lengths_array = ops.asarray1i(lengths) - if len(spans) > 0: - output = Ragged(ops.xp.vstack(spans), lengths_array) - else: - output = Ragged(ops.xp.zeros((0, 0), dtype="i"), lengths_array) - - assert output.dataXd.ndim == 2 - return output - - return ngram_suggester + return partial(ngram_suggester, sizes=sizes) @registry.misc("spacy.ngram_range_suggester.v1") @@ -119,10 +153,14 @@ def make_spancat( threshold: float, max_positive: Optional[int], ) -> "SpanCategorizer": - """Create a SpanCategorizer component. The span categorizer consists of two + """Create a SpanCategorizer component and configure it for multi-label + classification to be able to assign multiple labels for each span. + The span categorizer consists of two parts: a suggester function that proposes candidate spans, and a labeller model that predicts one or more labels for each span. + name (str): The component instance name, used to add entries to the + losses during training. suggester (Callable[[Iterable[Doc], Optional[Ops]], Ragged]): A function that suggests spans. Spans are returned as a ragged array with two integer columns, for the start and end positions. @@ -144,12 +182,80 @@ def make_spancat( """ return SpanCategorizer( nlp.vocab, - suggester=suggester, model=model, - spans_key=spans_key, - threshold=threshold, - max_positive=max_positive, + suggester=suggester, name=name, + spans_key=spans_key, + negative_weight=None, + allow_overlap=True, + max_positive=max_positive, + threshold=threshold, + scorer=scorer, + add_negative_label=False, + ) + + +@Language.factory( + "spancat_singlelabel", + assigns=["doc.spans"], + default_config={ + "spans_key": "sc", + "model": DEFAULT_SPANCAT_SINGLELABEL_MODEL, + "negative_weight": 1.0, + "suggester": {"@misc": "spacy.ngram_suggester.v1", "sizes": [1, 2, 3]}, + "scorer": {"@scorers": "spacy.spancat_scorer.v1"}, + "allow_overlap": True, + }, + default_score_weights={"spans_sc_f": 1.0, "spans_sc_p": 0.0, "spans_sc_r": 0.0}, +) +def make_spancat_singlelabel( + nlp: Language, + name: str, + suggester: Suggester, + model: Model[Tuple[List[Doc], Ragged], Floats2d], + spans_key: str, + negative_weight: float, + allow_overlap: bool, + scorer: Optional[Callable], +) -> "SpanCategorizer": + """Create a SpanCategorizer component and configure it for multi-class + classification. With this configuration each span can get at most one + label. The span categorizer consists of two + parts: a suggester function that proposes candidate spans, and a labeller + model that predicts one or more labels for each span. + + name (str): The component instance name, used to add entries to the + losses during training. + suggester (Callable[[Iterable[Doc], Optional[Ops]], Ragged]): A function that suggests spans. + Spans are returned as a ragged array with two integer columns, for the + start and end positions. + model (Model[Tuple[List[Doc], Ragged], Floats2d]): A model instance that + is given a list of documents and (start, end) indices representing + candidate span offsets. The model predicts a probability for each category + for each span. + spans_key (str): Key of the doc.spans dict to save the spans under. During + initialization and training, the component will look for spans on the + reference document under the same key. + scorer (Optional[Callable]): The scoring method. Defaults to + Scorer.score_spans for the Doc.spans[spans_key] with overlapping + spans allowed. + negative_weight (float): Multiplier for the loss terms. + Can be used to downweight the negative samples if there are too many. + allow_overlap (bool): If True the data is assumed to contain overlapping spans. + Otherwise it produces non-overlapping spans greedily prioritizing + higher assigned label scores. + """ + return SpanCategorizer( + nlp.vocab, + model=model, + suggester=suggester, + name=name, + spans_key=spans_key, + negative_weight=negative_weight, + allow_overlap=allow_overlap, + max_positive=1, + add_negative_label=True, + threshold=None, scorer=scorer, ) @@ -172,6 +278,27 @@ def make_spancat_scorer(): return spancat_score +@dataclass +class _Intervals: + """ + Helper class to avoid storing overlapping spans. + """ + + def __init__(self): + self.ranges = set() + + def add(self, i, j): + for e in range(i, j): + self.ranges.add(e) + + def __contains__(self, rang): + i, j = rang + for e in range(i, j): + if e in self.ranges: + return True + return False + + class SpanCategorizer(TrainablePipe): """Pipeline component to label spans of text. @@ -185,25 +312,43 @@ class SpanCategorizer(TrainablePipe): suggester: Suggester, name: str = "spancat", *, + add_negative_label: bool = False, spans_key: str = "spans", - threshold: float = 0.5, + negative_weight: Optional[float] = 1.0, + allow_overlap: Optional[bool] = True, max_positive: Optional[int] = None, + threshold: Optional[float] = 0.5, scorer: Optional[Callable] = spancat_score, ) -> None: - """Initialize the span categorizer. + """Initialize the multi-label or multi-class span categorizer. + vocab (Vocab): The shared vocabulary. model (thinc.api.Model): The Thinc Model powering the pipeline component. + For multi-class classification (single label per span) we recommend + using a Softmax classifier as a the final layer, while for multi-label + classification (multiple possible labels per span) we recommend Logistic. + suggester (Callable[[Iterable[Doc], Optional[Ops]], Ragged]): A function that suggests spans. + Spans are returned as a ragged array with two integer columns, for the + start and end positions. name (str): The component instance name, used to add entries to the losses during training. spans_key (str): Key of the Doc.spans dict to save the spans under. During initialization and training, the component will look for spans on the reference document under the same key. Defaults to `"spans"`. - threshold (float): Minimum probability to consider a prediction - positive. Spans with a positive prediction will be saved on the Doc. - Defaults to 0.5. + add_negative_label (bool): Learn to predict a special 'negative_label' + when a Span is not annotated. + threshold (Optional[float]): Minimum probability to consider a prediction + positive. Defaults to 0.5. Spans with a positive prediction will be saved + on the Doc. max_positive (Optional[int]): Maximum number of labels to consider positive per span. Defaults to None, indicating no limit. + negative_weight (float): Multiplier for the loss terms. + Can be used to downweight the negative samples if there are too many + when add_negative_label is True. Otherwise its unused. + allow_overlap (bool): If True the data is assumed to contain overlapping spans. + Otherwise it produces non-overlapping spans greedily prioritizing + higher assigned label scores. Only used when max_positive is 1. scorer (Optional[Callable]): The scoring method. Defaults to Scorer.score_spans for the Doc.spans[spans_key] with overlapping spans allowed. @@ -215,12 +360,17 @@ class SpanCategorizer(TrainablePipe): "spans_key": spans_key, "threshold": threshold, "max_positive": max_positive, + "negative_weight": negative_weight, + "allow_overlap": allow_overlap, } self.vocab = vocab self.suggester = suggester self.model = model self.name = name self.scorer = scorer + self.add_negative_label = add_negative_label + if not allow_overlap and max_positive is not None and max_positive > 1: + raise ValueError(Errors.E1051.format(max_positive=max_positive)) @property def key(self) -> str: @@ -230,6 +380,21 @@ class SpanCategorizer(TrainablePipe): """ return str(self.cfg["spans_key"]) + def _allow_extra_label(self) -> None: + """Raise an error if the component can not add any more labels.""" + nO = None + if self.model.has_dim("nO"): + nO = self.model.get_dim("nO") + elif self.model.has_ref("output_layer") and self.model.get_ref( + "output_layer" + ).has_dim("nO"): + nO = self.model.get_ref("output_layer").get_dim("nO") + if nO is not None and nO == self._n_labels: + if not self.is_resizable: + raise ValueError( + Errors.E922.format(name=self.name, nO=self.model.get_dim("nO")) + ) + def add_label(self, label: str) -> int: """Add a new label to the pipe. @@ -263,6 +428,27 @@ class SpanCategorizer(TrainablePipe): """ return list(self.labels) + @property + def _label_map(self) -> Dict[str, int]: + """RETURNS (Dict[str, int]): The label map.""" + return {label: i for i, label in enumerate(self.labels)} + + @property + def _n_labels(self) -> int: + """RETURNS (int): Number of labels.""" + if self.add_negative_label: + return len(self.labels) + 1 + else: + return len(self.labels) + + @property + def _negative_label_i(self) -> Union[int, None]: + """RETURNS (Union[int, None]): Index of the negative label.""" + if self.add_negative_label: + return len(self.label_data) + else: + return None + def predict(self, docs: Iterable[Doc]): """Apply the pipeline's model to a batch of docs, without modifying them. @@ -304,14 +490,24 @@ class SpanCategorizer(TrainablePipe): DOCS: https://spacy.io/api/spancategorizer#set_annotations """ - labels = self.labels indices, scores = indices_scores offset = 0 for i, doc in enumerate(docs): indices_i = indices[i].dataXd - doc.spans[self.key] = self._make_span_group( - doc, indices_i, scores[offset : offset + indices.lengths[i]], labels # type: ignore[arg-type] - ) + allow_overlap = cast(bool, self.cfg["allow_overlap"]) + if self.cfg["max_positive"] == 1: + doc.spans[self.key] = self._make_span_group_singlelabel( + doc, + indices_i, + scores[offset : offset + indices.lengths[i]], + allow_overlap, + ) + else: + doc.spans[self.key] = self._make_span_group_multilabel( + doc, + indices_i, + scores[offset : offset + indices.lengths[i]], + ) offset += indices.lengths[i] def update( @@ -371,9 +567,11 @@ class SpanCategorizer(TrainablePipe): spans = Ragged( self.model.ops.to_numpy(spans.data), self.model.ops.to_numpy(spans.lengths) ) - label_map = {label: i for i, label in enumerate(self.labels)} target = numpy.zeros(scores.shape, dtype=scores.dtype) + if self.add_negative_label: + negative_spans = numpy.ones((scores.shape[0])) offset = 0 + label_map = self._label_map for i, eg in enumerate(examples): # Map (start, end) offset of spans to the row in the d_scores array, # so that we can adjust the gradient for predictions that were @@ -390,10 +588,16 @@ class SpanCategorizer(TrainablePipe): row = spans_index[key] k = label_map[gold_span.label_] target[row, k] = 1.0 + if self.add_negative_label: + # delete negative label target. + negative_spans[row] = 0.0 # The target is a flat array for all docs. Track the position # we're at within the flat array. offset += spans.lengths[i] target = self.model.ops.asarray(target, dtype="f") # type: ignore + if self.add_negative_label: + negative_samples = numpy.nonzero(negative_spans)[0] + target[negative_samples, self._negative_label_i] = 1.0 # type: ignore # The target will have the values 0 (for untrue predictions) or 1 # (for true predictions). # The scores should be in the range [0, 1]. @@ -402,6 +606,10 @@ class SpanCategorizer(TrainablePipe): # If the prediction is 0.9 and it's false, the gradient will be # 0.9 (0.9 - 0.0) d_scores = scores - target + if self.add_negative_label: + neg_weight = cast(float, self.cfg["negative_weight"]) + if neg_weight != 1.0: + d_scores[negative_samples] *= neg_weight loss = float((d_scores**2).sum()) return loss, d_scores @@ -438,7 +646,7 @@ class SpanCategorizer(TrainablePipe): if subbatch: docs = [eg.x for eg in subbatch] spans = build_ngram_suggester(sizes=[1])(docs) - Y = self.model.ops.alloc2f(spans.dataXd.shape[0], len(self.labels)) + Y = self.model.ops.alloc2f(spans.dataXd.shape[0], self._n_labels) self.model.initialize(X=(docs, spans), Y=Y) else: self.model.initialize() @@ -452,31 +660,98 @@ class SpanCategorizer(TrainablePipe): eg.reference.spans.get(self.key, []), allow_overlap=True ) - def _make_span_group( - self, doc: Doc, indices: Ints2d, scores: Floats2d, labels: List[str] + def _make_span_group_multilabel( + self, + doc: Doc, + indices: Ints2d, + scores: Floats2d, ) -> SpanGroup: + """Find the top-k labels for each span (k=max_positive).""" spans = SpanGroup(doc, name=self.key) - max_positive = self.cfg["max_positive"] + if scores.size == 0: + return spans + scores = self.model.ops.to_numpy(scores) + indices = self.model.ops.to_numpy(indices) threshold = self.cfg["threshold"] + max_positive = self.cfg["max_positive"] keeps = scores >= threshold - ranked = (scores * -1).argsort() # type: ignore if max_positive is not None: assert isinstance(max_positive, int) + if self.add_negative_label: + negative_scores = numpy.copy(scores[:, self._negative_label_i]) + scores[:, self._negative_label_i] = -numpy.inf + ranked = (scores * -1).argsort() # type: ignore + scores[:, self._negative_label_i] = negative_scores + else: + ranked = (scores * -1).argsort() # type: ignore span_filter = ranked[:, max_positive:] for i, row in enumerate(span_filter): keeps[i, row] = False - spans.attrs["scores"] = scores[keeps].flatten() - - indices = self.model.ops.to_numpy(indices) - keeps = self.model.ops.to_numpy(keeps) + attrs_scores = [] for i in range(indices.shape[0]): start = indices[i, 0] end = indices[i, 1] - for j, keep in enumerate(keeps[i]): if keep: - spans.append(Span(doc, start, end, label=labels[j])) - + if j != self._negative_label_i: + spans.append(Span(doc, start, end, label=self.labels[j])) + attrs_scores.append(scores[i, j]) + spans.attrs["scores"] = numpy.array(attrs_scores) + return spans + + def _make_span_group_singlelabel( + self, + doc: Doc, + indices: Ints2d, + scores: Floats2d, + allow_overlap: bool = True, + ) -> SpanGroup: + """Find the argmax label for each span.""" + # Handle cases when there are zero suggestions + if scores.size == 0: + return SpanGroup(doc, name=self.key) + scores = self.model.ops.to_numpy(scores) + indices = self.model.ops.to_numpy(indices) + predicted = scores.argmax(axis=1) + argmax_scores = numpy.take_along_axis( + scores, numpy.expand_dims(predicted, 1), axis=1 + ) + keeps = numpy.ones(predicted.shape, dtype=bool) + # Remove samples where the negative label is the argmax. + if self.add_negative_label: + keeps = numpy.logical_and(keeps, predicted != self._negative_label_i) + # Filter samples according to threshold. + threshold = self.cfg["threshold"] + if threshold is not None: + keeps = numpy.logical_and(keeps, (argmax_scores >= threshold).squeeze()) + # Sort spans according to argmax probability + if not allow_overlap: + # Get the probabilities + sort_idx = (argmax_scores.squeeze() * -1).argsort() + argmax_scores = argmax_scores[sort_idx] + predicted = predicted[sort_idx] + indices = indices[sort_idx] + keeps = keeps[sort_idx] + seen = _Intervals() + spans = SpanGroup(doc, name=self.key) + attrs_scores = [] + for i in range(indices.shape[0]): + if not keeps[i]: + continue + + label = predicted[i] + start = indices[i, 0] + end = indices[i, 1] + + if not allow_overlap: + if (start, end) in seen: + continue + else: + seen.add(start, end) + attrs_scores.append(argmax_scores[i]) + spans.append(Span(doc, start, end, label=self.labels[label])) + + spans.attrs["scores"] = numpy.array(attrs_scores) return spans diff --git a/spacy/pipeline/tagger.pyx b/spacy/pipeline/tagger.pyx index d6ecbf084..4d5d78035 100644 --- a/spacy/pipeline/tagger.pyx +++ b/spacy/pipeline/tagger.pyx @@ -45,7 +45,7 @@ DEFAULT_TAGGER_MODEL = Config().from_str(default_model_config)["model"] @Language.factory( "tagger", assigns=["token.tag"], - default_config={"model": DEFAULT_TAGGER_MODEL, "overwrite": False, "scorer": {"@scorers": "spacy.tagger_scorer.v1"}, "neg_prefix": "!"}, + default_config={"model": DEFAULT_TAGGER_MODEL, "overwrite": False, "scorer": {"@scorers": "spacy.tagger_scorer.v1"}, "neg_prefix": "!", "label_smoothing": 0.0}, default_score_weights={"tag_acc": 1.0}, ) def make_tagger( @@ -55,6 +55,7 @@ def make_tagger( overwrite: bool, scorer: Optional[Callable], neg_prefix: str, + label_smoothing: float, ): """Construct a part-of-speech tagger component. @@ -63,7 +64,7 @@ def make_tagger( in size, and be normalized as probabilities (all scores between 0 and 1, with the rows summing to 1). """ - return Tagger(nlp.vocab, model, name, overwrite=overwrite, scorer=scorer, neg_prefix=neg_prefix) + return Tagger(nlp.vocab, model, name, overwrite=overwrite, scorer=scorer, neg_prefix=neg_prefix, label_smoothing=label_smoothing) def tagger_score(examples, **kwargs): @@ -89,6 +90,7 @@ class Tagger(TrainablePipe): overwrite=BACKWARD_OVERWRITE, scorer=tagger_score, neg_prefix="!", + label_smoothing=0.0, ): """Initialize a part-of-speech tagger. @@ -105,7 +107,7 @@ class Tagger(TrainablePipe): self.model = model self.name = name self._rehearsal_model = None - cfg = {"labels": [], "overwrite": overwrite, "neg_prefix": neg_prefix} + cfg = {"labels": [], "overwrite": overwrite, "neg_prefix": neg_prefix, "label_smoothing": label_smoothing} self.cfg = dict(sorted(cfg.items())) self.scorer = scorer @@ -256,7 +258,7 @@ class Tagger(TrainablePipe): DOCS: https://spacy.io/api/tagger#get_loss """ validate_examples(examples, "Tagger.get_loss") - loss_func = SequenceCategoricalCrossentropy(names=self.labels, normalize=False, neg_prefix=self.cfg["neg_prefix"]) + loss_func = SequenceCategoricalCrossentropy(names=self.labels, normalize=False, neg_prefix=self.cfg["neg_prefix"], label_smoothing=self.cfg["label_smoothing"]) # Convert empty tag "" to missing value None so that both misaligned # tokens and tokens with missing annotation have the default missing # value None. diff --git a/spacy/pipeline/textcat.py b/spacy/pipeline/textcat.py index 65121114d..650a01949 100644 --- a/spacy/pipeline/textcat.py +++ b/spacy/pipeline/textcat.py @@ -74,7 +74,7 @@ subword_features = true default_config={ "threshold": 0.0, "model": DEFAULT_SINGLE_TEXTCAT_MODEL, - "scorer": {"@scorers": "spacy.textcat_scorer.v1"}, + "scorer": {"@scorers": "spacy.textcat_scorer.v2"}, }, default_score_weights={ "cats_score": 1.0, @@ -117,7 +117,7 @@ def textcat_score(examples: Iterable[Example], **kwargs) -> Dict[str, Any]: ) -@registry.scorers("spacy.textcat_scorer.v1") +@registry.scorers("spacy.textcat_scorer.v2") def make_textcat_scorer(): return textcat_score diff --git a/spacy/pipeline/textcat_multilabel.py b/spacy/pipeline/textcat_multilabel.py index 328cee723..41c0e2f63 100644 --- a/spacy/pipeline/textcat_multilabel.py +++ b/spacy/pipeline/textcat_multilabel.py @@ -74,7 +74,7 @@ subword_features = true default_config={ "threshold": 0.5, "model": DEFAULT_MULTI_TEXTCAT_MODEL, - "scorer": {"@scorers": "spacy.textcat_multilabel_scorer.v1"}, + "scorer": {"@scorers": "spacy.textcat_multilabel_scorer.v2"}, }, default_score_weights={ "cats_score": 1.0, @@ -120,7 +120,7 @@ def textcat_multilabel_score(examples: Iterable[Example], **kwargs) -> Dict[str, ) -@registry.scorers("spacy.textcat_multilabel_scorer.v1") +@registry.scorers("spacy.textcat_multilabel_scorer.v2") def make_textcat_multilabel_scorer(): return textcat_multilabel_score diff --git a/spacy/schemas.py b/spacy/schemas.py index e48fe1702..140592dcd 100644 --- a/spacy/schemas.py +++ b/spacy/schemas.py @@ -156,12 +156,40 @@ def validate_token_pattern(obj: list) -> List[str]: class TokenPatternString(BaseModel): - REGEX: Optional[StrictStr] = Field(None, alias="regex") + REGEX: Optional[Union[StrictStr, "TokenPatternString"]] = Field(None, alias="regex") IN: Optional[List[StrictStr]] = Field(None, alias="in") NOT_IN: Optional[List[StrictStr]] = Field(None, alias="not_in") IS_SUBSET: Optional[List[StrictStr]] = Field(None, alias="is_subset") IS_SUPERSET: Optional[List[StrictStr]] = Field(None, alias="is_superset") INTERSECTS: Optional[List[StrictStr]] = Field(None, alias="intersects") + FUZZY: Optional[Union[StrictStr, "TokenPatternString"]] = Field(None, alias="fuzzy") + FUZZY1: Optional[Union[StrictStr, "TokenPatternString"]] = Field( + None, alias="fuzzy1" + ) + FUZZY2: Optional[Union[StrictStr, "TokenPatternString"]] = Field( + None, alias="fuzzy2" + ) + FUZZY3: Optional[Union[StrictStr, "TokenPatternString"]] = Field( + None, alias="fuzzy3" + ) + FUZZY4: Optional[Union[StrictStr, "TokenPatternString"]] = Field( + None, alias="fuzzy4" + ) + FUZZY5: Optional[Union[StrictStr, "TokenPatternString"]] = Field( + None, alias="fuzzy5" + ) + FUZZY6: Optional[Union[StrictStr, "TokenPatternString"]] = Field( + None, alias="fuzzy6" + ) + FUZZY7: Optional[Union[StrictStr, "TokenPatternString"]] = Field( + None, alias="fuzzy7" + ) + FUZZY8: Optional[Union[StrictStr, "TokenPatternString"]] = Field( + None, alias="fuzzy8" + ) + FUZZY9: Optional[Union[StrictStr, "TokenPatternString"]] = Field( + None, alias="fuzzy9" + ) class Config: extra = "forbid" diff --git a/spacy/scorer.py b/spacy/scorer.py index 16fc303a0..de4f52be6 100644 --- a/spacy/scorer.py +++ b/spacy/scorer.py @@ -174,7 +174,7 @@ class Scorer: prf_score.score_set(pred_spans, gold_spans) if len(acc_score) > 0: return { - "token_acc": acc_score.fscore, + "token_acc": acc_score.precision, "token_p": prf_score.precision, "token_r": prf_score.recall, "token_f": prf_score.fscore, @@ -476,14 +476,12 @@ class Scorer: f_per_type = {label: PRFScore() for label in labels} auc_per_type = {label: ROCAUCScore() for label in labels} labels = set(labels) - if labels: - for eg in examples: - labels.update(eg.predicted.cats.keys()) - labels.update(eg.reference.cats.keys()) for example in examples: # Through this loop, None in the gold_cats indicates missing label. pred_cats = getter(example.predicted, attr) + pred_cats = {k: v for k, v in pred_cats.items() if k in labels} gold_cats = getter(example.reference, attr) + gold_cats = {k: v for k, v in gold_cats.items() if k in labels} for label in labels: pred_score = pred_cats.get(label, 0.0) diff --git a/spacy/tests/doc/test_span.py b/spacy/tests/doc/test_span.py index 309ae6671..a5c512dc0 100644 --- a/spacy/tests/doc/test_span.py +++ b/spacy/tests/doc/test_span.py @@ -163,6 +163,18 @@ def test_char_span(doc, i_sent, i, j, text): assert span.text == text +def test_char_span_attributes(doc): + label = "LABEL" + kb_id = "KB_ID" + span_id = "SPAN_ID" + span1 = doc.char_span(20, 45, label=label, kb_id=kb_id, span_id=span_id) + span2 = doc[1:].char_span(15, 40, label=label, kb_id=kb_id, span_id=span_id) + assert span1.text == span2.text + assert span1.label_ == span2.label_ == label + assert span1.kb_id_ == span2.kb_id_ == kb_id + assert span1.id_ == span2.id_ == span_id + + def test_spans_sent_spans(doc): sents = list(doc.sents) assert sents[0].start == 0 @@ -367,6 +379,14 @@ def test_spans_by_character(doc): span1.start_char + 1, span1.end_char, label="GPE", alignment_mode="unk" ) + # Span.char_span + alignment mode "contract" + span2 = doc[0:2].char_span( + span1.start_char - 3, span1.end_char, label="GPE", alignment_mode="contract" + ) + assert span1.start_char == span2.start_char + assert span1.end_char == span2.end_char + assert span2.label_ == "GPE" + def test_span_to_array(doc): span = doc[1:-2] @@ -696,3 +716,18 @@ def test_for_partial_ent_sents(): # equal to the sentences referenced in ent.sents. for doc_sent, ent_sent in zip(doc.sents, doc.ents[0].sents): assert doc_sent == ent_sent + + +def test_for_no_ent_sents(): + """Span.sents() should set .sents correctly, even if Span in question is trailing and doesn't form a full + sentence. + """ + doc = Doc( + English().vocab, + words=["This", "is", "a", "test.", "ENTITY"], + sent_starts=[1, 0, 0, 0, 1], + ) + doc.set_ents([Span(doc, 4, 5, "WORK")]) + sents = list(doc.ents[0].sents) + assert len(sents) == 1 + assert str(sents[0]) == str(doc.ents[0].sent) == "ENTITY" diff --git a/spacy/tests/doc/test_span_group.py b/spacy/tests/doc/test_span_group.py index 8c70a83e1..818569c64 100644 --- a/spacy/tests/doc/test_span_group.py +++ b/spacy/tests/doc/test_span_group.py @@ -1,7 +1,10 @@ +from typing import List + import pytest from random import Random from spacy.matcher import Matcher -from spacy.tokens import Span, SpanGroup +from spacy.tokens import Span, SpanGroup, Doc +from spacy.util import filter_spans @pytest.fixture @@ -240,3 +243,13 @@ def test_span_group_extend(doc): def test_span_group_dealloc(span_group): with pytest.raises(AttributeError): print(span_group.doc) + + +@pytest.mark.issue(11975) +def test_span_group_typing(doc: Doc): + """Tests whether typing of `SpanGroup` as `Iterable[Span]`-like object is accepted by mypy.""" + span_group: SpanGroup = doc.spans["SPANS"] + spans: List[Span] = list(span_group) + for i, span in enumerate(span_group): + assert span == span_group[i] == spans[i] + filter_spans(span_group) diff --git a/spacy/tests/lang/sv/test_prefix_suffix_infix.py b/spacy/tests/lang/sv/test_prefix_suffix_infix.py index bbb0ff415..0aa495992 100644 --- a/spacy/tests/lang/sv/test_prefix_suffix_infix.py +++ b/spacy/tests/lang/sv/test_prefix_suffix_infix.py @@ -32,3 +32,10 @@ def test_tokenizer_splits_comma_infix(sv_tokenizer, text): def test_tokenizer_splits_ellipsis_infix(sv_tokenizer, text): tokens = sv_tokenizer(text) assert len(tokens) == 3 + + +@pytest.mark.issue(12311) +@pytest.mark.parametrize("text", ["99:e", "c:a", "EU:s", "Maj:t"]) +def test_sv_tokenizer_handles_colon(sv_tokenizer, text): + tokens = sv_tokenizer(text) + assert len(tokens) == 1 diff --git a/spacy/tests/matcher/test_dependency_matcher.py b/spacy/tests/matcher/test_dependency_matcher.py index b4e19d69d..200384320 100644 --- a/spacy/tests/matcher/test_dependency_matcher.py +++ b/spacy/tests/matcher/test_dependency_matcher.py @@ -316,16 +316,32 @@ def test_dependency_matcher_precedence_ops(en_vocab, op, num_matches): ("the", "brown", "$--", 0), ("brown", "the", "$--", 1), ("brown", "brown", "$--", 0), + ("over", "jumped", "<+", 0), + ("quick", "fox", "<+", 0), + ("the", "quick", "<+", 0), + ("brown", "fox", "<+", 1), ("quick", "fox", "<++", 1), ("quick", "over", "<++", 0), ("over", "jumped", "<++", 0), ("the", "fox", "<++", 2), + ("brown", "fox", "<-", 0), + ("fox", "over", "<-", 0), + ("the", "over", "<-", 0), + ("over", "jumped", "<-", 1), ("brown", "fox", "<--", 0), ("fox", "jumped", "<--", 0), ("fox", "over", "<--", 1), + ("fox", "brown", ">+", 0), + ("over", "fox", ">+", 0), + ("over", "the", ">+", 0), + ("jumped", "over", ">+", 1), ("jumped", "over", ">++", 1), ("fox", "lazy", ">++", 0), ("over", "the", ">++", 0), + ("jumped", "over", ">-", 0), + ("fox", "quick", ">-", 0), + ("brown", "quick", ">-", 0), + ("fox", "brown", ">-", 1), ("brown", "fox", ">--", 0), ("fox", "brown", ">--", 1), ("jumped", "fox", ">--", 1), diff --git a/spacy/tests/matcher/test_levenshtein.py b/spacy/tests/matcher/test_levenshtein.py index d30e36132..5afb7e1fc 100644 --- a/spacy/tests/matcher/test_levenshtein.py +++ b/spacy/tests/matcher/test_levenshtein.py @@ -1,5 +1,6 @@ import pytest from spacy.matcher import levenshtein +from spacy.matcher.levenshtein import levenshtein_compare # empty string plus 10 random ASCII, 10 random unicode, and 2 random long tests @@ -42,3 +43,31 @@ from spacy.matcher import levenshtein ) def test_levenshtein(dist, a, b): assert levenshtein(a, b) == dist + + +@pytest.mark.parametrize( + "a,b,fuzzy,expected", + [ + ("a", "a", 1, True), + ("a", "a", 0, True), + ("a", "a", -1, True), + ("a", "ab", 1, True), + ("a", "ab", 0, False), + ("a", "ab", -1, True), + ("ab", "ac", 1, True), + ("ab", "ac", -1, True), + ("abc", "cde", 4, True), + ("abc", "cde", -1, False), + ("abcdef", "cdefgh", 4, True), + ("abcdef", "cdefgh", 3, False), + ("abcdef", "cdefgh", -1, False), # default (2 for length 6) + ("abcdefgh", "cdefghijk", 5, True), + ("abcdefgh", "cdefghijk", 4, False), + ("abcdefgh", "cdefghijk", -1, False), # default (2) + ("abcdefgh", "cdefghijkl", 6, True), + ("abcdefgh", "cdefghijkl", 5, False), + ("abcdefgh", "cdefghijkl", -1, False), # default (2) + ], +) +def test_levenshtein_compare(a, b, fuzzy, expected): + assert levenshtein_compare(a, b, fuzzy) == expected diff --git a/spacy/tests/matcher/test_matcher_api.py b/spacy/tests/matcher/test_matcher_api.py index ac905eeb4..09ab6c7dc 100644 --- a/spacy/tests/matcher/test_matcher_api.py +++ b/spacy/tests/matcher/test_matcher_api.py @@ -118,6 +118,155 @@ def test_matcher_match_multi(matcher): ] +@pytest.mark.parametrize( + "rules,match_locs", + [ + ( + { + "GoogleNow": [[{"ORTH": {"FUZZY": "Google"}}, {"ORTH": "Now"}]], + }, + [(2, 4)], + ), + ( + { + "Java": [[{"LOWER": {"FUZZY": "java"}}]], + }, + [(5, 6)], + ), + ( + { + "JS": [[{"ORTH": {"FUZZY": "JavaScript"}}]], + "GoogleNow": [[{"ORTH": {"FUZZY": "Google"}}, {"ORTH": "Now"}]], + "Java": [[{"LOWER": {"FUZZY": "java"}}]], + }, + [(2, 4), (5, 6), (8, 9)], + ), + # only the second pattern matches (check that predicate keys used for + # caching don't collide) + ( + { + "A": [[{"ORTH": {"FUZZY": "Javascripts"}}]], + "B": [[{"ORTH": {"FUZZY5": "Javascripts"}}]], + }, + [(8, 9)], + ), + ], +) +def test_matcher_match_fuzzy(en_vocab, rules, match_locs): + words = ["They", "like", "Goggle", "Now", "and", "Jav", "but", "not", "JvvaScrpt"] + doc = Doc(en_vocab, words=words) + + matcher = Matcher(en_vocab) + for key, patterns in rules.items(): + matcher.add(key, patterns) + assert match_locs == [(start, end) for m_id, start, end in matcher(doc)] + + +@pytest.mark.parametrize("set_op", ["IN", "NOT_IN"]) +def test_matcher_match_fuzzy_set_op_longest(en_vocab, set_op): + rules = { + "GoogleNow": [[{"ORTH": {"FUZZY": {set_op: ["Google", "Now"]}}, "OP": "+"}]] + } + matcher = Matcher(en_vocab) + for key, patterns in rules.items(): + matcher.add(key, patterns, greedy="LONGEST") + + words = ["They", "like", "Goggle", "Noo"] + doc = Doc(en_vocab, words=words) + assert len(matcher(doc)) == 1 + + +def test_matcher_match_fuzzy_set_multiple(en_vocab): + rules = { + "GoogleNow": [ + [ + { + "ORTH": {"FUZZY": {"IN": ["Google", "Now"]}, "NOT_IN": ["Goggle"]}, + "OP": "+", + } + ] + ] + } + matcher = Matcher(en_vocab) + for key, patterns in rules.items(): + matcher.add(key, patterns, greedy="LONGEST") + + words = ["They", "like", "Goggle", "Noo"] + doc = Doc(matcher.vocab, words=words) + assert matcher(doc) == [ + (doc.vocab.strings["GoogleNow"], 3, 4), + ] + + +@pytest.mark.parametrize("fuzzyn", range(1, 10)) +def test_matcher_match_fuzzyn_all_insertions(en_vocab, fuzzyn): + matcher = Matcher(en_vocab) + matcher.add("GoogleNow", [[{"ORTH": {f"FUZZY{fuzzyn}": "GoogleNow"}}]]) + # words with increasing edit distance + words = ["GoogleNow" + "a" * i for i in range(0, 10)] + doc = Doc(en_vocab, words) + assert len(matcher(doc)) == fuzzyn + 1 + + +@pytest.mark.parametrize("fuzzyn", range(1, 6)) +def test_matcher_match_fuzzyn_various_edits(en_vocab, fuzzyn): + matcher = Matcher(en_vocab) + matcher.add("GoogleNow", [[{"ORTH": {f"FUZZY{fuzzyn}": "GoogleNow"}}]]) + # words with increasing edit distance of different edit types + words = [ + "GoogleNow", + "GoogleNuw", + "GoogleNuew", + "GoogleNoweee", + "GiggleNuw3", + "gouggle5New", + ] + doc = Doc(en_vocab, words) + assert len(matcher(doc)) == fuzzyn + 1 + + +@pytest.mark.parametrize("greedy", ["FIRST", "LONGEST"]) +@pytest.mark.parametrize("set_op", ["IN", "NOT_IN"]) +def test_matcher_match_fuzzyn_set_op_longest(en_vocab, greedy, set_op): + rules = { + "GoogleNow": [[{"ORTH": {"FUZZY2": {set_op: ["Google", "Now"]}}, "OP": "+"}]] + } + matcher = Matcher(en_vocab) + for key, patterns in rules.items(): + matcher.add(key, patterns, greedy=greedy) + + words = ["They", "like", "Goggle", "Noo"] + doc = Doc(matcher.vocab, words=words) + spans = matcher(doc, as_spans=True) + assert len(spans) == 1 + if set_op == "IN": + assert spans[0].text == "Goggle Noo" + else: + assert spans[0].text == "They like" + + +def test_matcher_match_fuzzyn_set_multiple(en_vocab): + rules = { + "GoogleNow": [ + [ + { + "ORTH": {"FUZZY1": {"IN": ["Google", "Now"]}, "NOT_IN": ["Goggle"]}, + "OP": "+", + } + ] + ] + } + matcher = Matcher(en_vocab) + for key, patterns in rules.items(): + matcher.add(key, patterns, greedy="LONGEST") + + words = ["They", "like", "Goggle", "Noo"] + doc = Doc(matcher.vocab, words=words) + assert matcher(doc) == [ + (doc.vocab.strings["GoogleNow"], 3, 4), + ] + + def test_matcher_empty_dict(en_vocab): """Test matcher allows empty token specs, meaning match on any token.""" matcher = Matcher(en_vocab) @@ -437,6 +586,30 @@ def test_matcher_regex(en_vocab): assert len(matches) == 0 +def test_matcher_regex_set_in(en_vocab): + matcher = Matcher(en_vocab) + pattern = [{"ORTH": {"REGEX": {"IN": [r"(?:a)", r"(?:an)"]}}}] + matcher.add("A_OR_AN", [pattern]) + doc = Doc(en_vocab, words=["an", "a", "hi"]) + matches = matcher(doc) + assert len(matches) == 2 + doc = Doc(en_vocab, words=["bye"]) + matches = matcher(doc) + assert len(matches) == 0 + + +def test_matcher_regex_set_not_in(en_vocab): + matcher = Matcher(en_vocab) + pattern = [{"ORTH": {"REGEX": {"NOT_IN": [r"(?:a)", r"(?:an)"]}}}] + matcher.add("A_OR_AN", [pattern]) + doc = Doc(en_vocab, words=["an", "a", "hi"]) + matches = matcher(doc) + assert len(matches) == 1 + doc = Doc(en_vocab, words=["bye"]) + matches = matcher(doc) + assert len(matches) == 1 + + def test_matcher_regex_shape(en_vocab): matcher = Matcher(en_vocab) pattern = [{"SHAPE": {"REGEX": r"^[^x]+$"}}] diff --git a/spacy/tests/parser/test_ner.py b/spacy/tests/parser/test_ner.py index 00889efdc..030182a63 100644 --- a/spacy/tests/parser/test_ner.py +++ b/spacy/tests/parser/test_ner.py @@ -9,6 +9,8 @@ from spacy.lang.en import English from spacy.lang.it import Italian from spacy.language import Language from spacy.lookups import Lookups +from spacy.pipeline import EntityRecognizer +from spacy.pipeline.ner import DEFAULT_NER_MODEL from spacy.pipeline._parser_internals.ner import BiluoPushDown from spacy.training import Example, iob_to_biluo, split_bilu_label from spacy.tokens import Doc, Span @@ -16,8 +18,6 @@ from spacy.vocab import Vocab import logging from ..util import make_tempdir -from ...pipeline import EntityRecognizer -from ...pipeline.ner import DEFAULT_NER_MODEL TRAIN_DATA = [ ("Who is Shaka Khan?", {"entities": [(7, 17, "PERSON")]}), diff --git a/spacy/tests/parser/test_parse.py b/spacy/tests/parser/test_parse.py index aaf31ed56..4b05c6721 100644 --- a/spacy/tests/parser/test_parse.py +++ b/spacy/tests/parser/test_parse.py @@ -8,11 +8,11 @@ from spacy.lang.en import English from spacy.tokens import Doc from spacy.training import Example from spacy.vocab import Vocab +from spacy.pipeline import DependencyParser +from spacy.pipeline.dep_parser import DEFAULT_PARSER_MODEL +from spacy.pipeline.tok2vec import DEFAULT_TOK2VEC_MODEL -from ...pipeline import DependencyParser -from ...pipeline.dep_parser import DEFAULT_PARSER_MODEL from ..util import apply_transition_sequence, make_tempdir -from ...pipeline.tok2vec import DEFAULT_TOK2VEC_MODEL TRAIN_DATA = [ ( diff --git a/spacy/tests/pipeline/test_edit_tree_lemmatizer.py b/spacy/tests/pipeline/test_edit_tree_lemmatizer.py index b12ca5dd4..128d75680 100644 --- a/spacy/tests/pipeline/test_edit_tree_lemmatizer.py +++ b/spacy/tests/pipeline/test_edit_tree_lemmatizer.py @@ -101,14 +101,15 @@ def test_initialize_from_labels(): } -def test_no_data(): +@pytest.mark.parametrize("top_k", (1, 5, 30)) +def test_no_data(top_k): # Test that the lemmatizer provides a nice error when there's no tagging data / labels TEXTCAT_DATA = [ ("I'm so happy.", {"cats": {"POSITIVE": 1.0, "NEGATIVE": 0.0}}), ("I'm so angry", {"cats": {"POSITIVE": 0.0, "NEGATIVE": 1.0}}), ] nlp = English() - nlp.add_pipe("trainable_lemmatizer") + nlp.add_pipe("trainable_lemmatizer", config={"top_k": top_k}) nlp.add_pipe("textcat") train_examples = [] @@ -119,10 +120,11 @@ def test_no_data(): nlp.initialize(get_examples=lambda: train_examples) -def test_incomplete_data(): +@pytest.mark.parametrize("top_k", (1, 5, 30)) +def test_incomplete_data(top_k): # Test that the lemmatizer works with incomplete information nlp = English() - lemmatizer = nlp.add_pipe("trainable_lemmatizer") + lemmatizer = nlp.add_pipe("trainable_lemmatizer", config={"top_k": top_k}) lemmatizer.min_tree_freq = 1 train_examples = [] for t in PARTIAL_DATA: @@ -139,10 +141,25 @@ def test_incomplete_data(): assert doc[1].lemma_ == "like" assert doc[2].lemma_ == "blue" + # Check that incomplete annotations are ignored. + scores, _ = lemmatizer.model([eg.predicted for eg in train_examples], is_train=True) + _, dX = lemmatizer.get_loss(train_examples, scores) + xp = lemmatizer.model.ops.xp -def test_overfitting_IO(): + # Missing annotations. + assert xp.count_nonzero(dX[0][0]) == 0 + assert xp.count_nonzero(dX[0][3]) == 0 + assert xp.count_nonzero(dX[1][0]) == 0 + assert xp.count_nonzero(dX[1][3]) == 0 + + # Misaligned annotations. + assert xp.count_nonzero(dX[1][1]) == 0 + + +@pytest.mark.parametrize("top_k", (1, 5, 30)) +def test_overfitting_IO(top_k): nlp = English() - lemmatizer = nlp.add_pipe("trainable_lemmatizer") + lemmatizer = nlp.add_pipe("trainable_lemmatizer", config={"top_k": top_k}) lemmatizer.min_tree_freq = 1 train_examples = [] for t in TRAIN_DATA: @@ -175,7 +192,7 @@ def test_overfitting_IO(): # Check model after a {to,from}_bytes roundtrip nlp_bytes = nlp.to_bytes() nlp3 = English() - nlp3.add_pipe("trainable_lemmatizer") + nlp3.add_pipe("trainable_lemmatizer", config={"top_k": top_k}) nlp3.from_bytes(nlp_bytes) doc3 = nlp3(test_text) assert doc3[0].lemma_ == "she" diff --git a/spacy/tests/pipeline/test_entity_linker.py b/spacy/tests/pipeline/test_entity_linker.py index d61378002..1b83bb723 100644 --- a/spacy/tests/pipeline/test_entity_linker.py +++ b/spacy/tests/pipeline/test_entity_linker.py @@ -1,9 +1,9 @@ -from typing import Callable, Iterable, Dict, Any, Iterator +from typing import Callable, Iterable, Dict, Any, Iterator, Tuple import pytest from numpy.testing import assert_equal -from spacy import registry, util +from spacy import registry, util, Language from spacy.attrs import ENT_KB_ID from spacy.compat import pickle from spacy.kb import Candidate, InMemoryLookupKB, KnowledgeBase @@ -108,18 +108,23 @@ def test_issue7065(): @pytest.mark.issue(7065) -def test_issue7065_b(): +@pytest.mark.parametrize("entity_in_first_sentence", [True, False]) +def test_sentence_crossing_ents(entity_in_first_sentence: bool): + """Tests if NEL crashes if entities cross sentence boundaries and the first associated sentence doesn't have an + entity. + entity_in_prior_sentence (bool): Whether to include an entity in the first sentence associated with the + sentence-crossing entity. + """ # Test that the NEL doesn't crash when an entity crosses a sentence boundary nlp = English() vector_length = 3 - nlp.add_pipe("sentencizer") text = "Mahler 's Symphony No. 8 was beautiful." - entities = [(0, 6, "PERSON"), (10, 24, "WORK")] - links = { - (0, 6): {"Q7304": 1.0, "Q270853": 0.0}, - (10, 24): {"Q7304": 0.0, "Q270853": 1.0}, - } - sent_starts = [1, -1, 0, 0, 0, 0, 0, 0, 0] + entities = [(10, 24, "WORK")] + links = {(10, 24): {"Q7304": 0.0, "Q270853": 1.0}} + if entity_in_first_sentence: + entities.append((0, 6, "PERSON")) + links[(0, 6)] = {"Q7304": 1.0, "Q270853": 0.0} + sent_starts = [1, -1, 0, 0, 0, 1, 0, 0, 0] doc = nlp(text) example = Example.from_dict( doc, {"entities": entities, "links": links, "sent_starts": sent_starts} @@ -145,31 +150,14 @@ def test_issue7065_b(): # Create the Entity Linker component and add it to the pipeline entity_linker = nlp.add_pipe("entity_linker", last=True) - entity_linker.set_kb(create_kb) + entity_linker.set_kb(create_kb) # type: ignore # train the NEL pipe optimizer = nlp.initialize(get_examples=lambda: train_examples) for i in range(2): - losses = {} - nlp.update(train_examples, sgd=optimizer, losses=losses) + nlp.update(train_examples, sgd=optimizer) - # Add a custom rule-based component to mimick NER - patterns = [ - {"label": "PERSON", "pattern": [{"LOWER": "mahler"}]}, - { - "label": "WORK", - "pattern": [ - {"LOWER": "symphony"}, - {"LOWER": "no"}, - {"LOWER": "."}, - {"LOWER": "8"}, - ], - }, - ] - ruler = nlp.add_pipe("entity_ruler", before="entity_linker") - ruler.add_patterns(patterns) - # test the trained model - this should not throw E148 - doc = nlp(text) - assert doc + # This shouldn't crash. + entity_linker.predict([example.reference]) # type: ignore def test_no_entities(): @@ -353,6 +341,9 @@ def test_kb_default(nlp): """Test that the default (empty) KB is loaded upon construction""" entity_linker = nlp.add_pipe("entity_linker", config={}) assert len(entity_linker.kb) == 0 + with pytest.raises(ValueError, match="E139"): + # this raises an error because the KB is empty + entity_linker.validate_kb() assert entity_linker.kb.get_size_entities() == 0 assert entity_linker.kb.get_size_aliases() == 0 # 64 is the default value from pipeline.entity_linker diff --git a/spacy/tests/pipeline/test_entity_ruler.py b/spacy/tests/pipeline/test_entity_ruler.py index 6851e2a7c..417f930cb 100644 --- a/spacy/tests/pipeline/test_entity_ruler.py +++ b/spacy/tests/pipeline/test_entity_ruler.py @@ -382,6 +382,43 @@ def test_entity_ruler_overlapping_spans(nlp, entity_ruler_factory): assert doc.ents[0].label_ == "FOOBAR" +@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS) +def test_entity_ruler_fuzzy_pipe(nlp, entity_ruler_factory): + ruler = nlp.add_pipe(entity_ruler_factory, name="entity_ruler") + patterns = [{"label": "HELLO", "pattern": [{"LOWER": {"FUZZY": "hello"}}]}] + ruler.add_patterns(patterns) + doc = nlp("helloo") + assert len(doc.ents) == 1 + assert doc.ents[0].label_ == "HELLO" + + +@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS) +def test_entity_ruler_fuzzy(nlp, entity_ruler_factory): + ruler = nlp.add_pipe(entity_ruler_factory, name="entity_ruler") + patterns = [{"label": "HELLO", "pattern": [{"LOWER": {"FUZZY": "hello"}}]}] + ruler.add_patterns(patterns) + doc = nlp("helloo") + assert len(doc.ents) == 1 + assert doc.ents[0].label_ == "HELLO" + + +@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS) +def test_entity_ruler_fuzzy_disabled(nlp, entity_ruler_factory): + @registry.misc("test_fuzzy_compare_disabled") + def make_test_fuzzy_compare_disabled(): + return lambda x, y, z: False + + ruler = nlp.add_pipe( + entity_ruler_factory, + name="entity_ruler", + config={"matcher_fuzzy_compare": {"@misc": "test_fuzzy_compare_disabled"}}, + ) + patterns = [{"label": "HELLO", "pattern": [{"LOWER": {"FUZZY": "hello"}}]}] + ruler.add_patterns(patterns) + doc = nlp("helloo") + assert len(doc.ents) == 0 + + @pytest.mark.parametrize("n_process", [1, 2]) @pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS) def test_entity_ruler_multiprocessing(nlp, n_process, entity_ruler_factory): diff --git a/spacy/tests/pipeline/test_morphologizer.py b/spacy/tests/pipeline/test_morphologizer.py index 33696bfd8..74c571ccf 100644 --- a/spacy/tests/pipeline/test_morphologizer.py +++ b/spacy/tests/pipeline/test_morphologizer.py @@ -1,5 +1,7 @@ import pytest -from numpy.testing import assert_equal +from numpy.testing import assert_equal, assert_almost_equal + +from thinc.api import get_current_ops from spacy import util from spacy.training import Example @@ -19,6 +21,8 @@ def test_label_types(): morphologizer.add_label(9) +TAGS = ["Feat=N", "Feat=V", "Feat=J"] + TRAIN_DATA = [ ( "I like green eggs", @@ -32,6 +36,30 @@ TRAIN_DATA = [ ] +def test_label_smoothing(): + nlp = Language() + morph_no_ls = nlp.add_pipe("morphologizer", "no_label_smoothing") + morph_ls = nlp.add_pipe( + "morphologizer", "label_smoothing", config=dict(label_smoothing=0.05) + ) + train_examples = [] + losses = {} + for tag in TAGS: + morph_no_ls.add_label(tag) + morph_ls.add_label(tag) + for t in TRAIN_DATA: + train_examples.append(Example.from_dict(nlp.make_doc(t[0]), t[1])) + + nlp.initialize(get_examples=lambda: train_examples) + tag_scores, bp_tag_scores = morph_ls.model.begin_update( + [eg.predicted for eg in train_examples] + ) + ops = get_current_ops() + no_ls_grads = ops.to_numpy(morph_no_ls.get_loss(train_examples, tag_scores)[1][0]) + ls_grads = ops.to_numpy(morph_ls.get_loss(train_examples, tag_scores)[1][0]) + assert_almost_equal(ls_grads / no_ls_grads, 0.94285715) + + def test_no_label(): nlp = Language() nlp.add_pipe("morphologizer") diff --git a/spacy/tests/pipeline/test_spancat.py b/spacy/tests/pipeline/test_spancat.py index e9db983d3..199ef2b2a 100644 --- a/spacy/tests/pipeline/test_spancat.py +++ b/spacy/tests/pipeline/test_spancat.py @@ -1,7 +1,7 @@ import pytest import numpy from numpy.testing import assert_array_equal, assert_almost_equal -from thinc.api import get_current_ops, Ragged +from thinc.api import get_current_ops, NumpyOps, Ragged from spacy import util from spacy.lang.en import English @@ -15,6 +15,8 @@ OPS = get_current_ops() SPAN_KEY = "labeled_spans" +SPANCAT_COMPONENTS = ["spancat", "spancat_singlelabel"] + TRAIN_DATA = [ ("Who is Shaka Khan?", {"spans": {SPAN_KEY: [(7, 17, "PERSON")]}}), ( @@ -41,38 +43,42 @@ def make_examples(nlp, data=TRAIN_DATA): return train_examples -def test_no_label(): +@pytest.mark.parametrize("name", SPANCAT_COMPONENTS) +def test_no_label(name): nlp = Language() - nlp.add_pipe("spancat", config={"spans_key": SPAN_KEY}) + nlp.add_pipe(name, config={"spans_key": SPAN_KEY}) with pytest.raises(ValueError): nlp.initialize() -def test_no_resize(): +@pytest.mark.parametrize("name", SPANCAT_COMPONENTS) +def test_no_resize(name): nlp = Language() - spancat = nlp.add_pipe("spancat", config={"spans_key": SPAN_KEY}) + spancat = nlp.add_pipe(name, config={"spans_key": SPAN_KEY}) spancat.add_label("Thing") spancat.add_label("Phrase") assert spancat.labels == ("Thing", "Phrase") nlp.initialize() - assert spancat.model.get_dim("nO") == 2 + assert spancat.model.get_dim("nO") == spancat._n_labels # this throws an error because the spancat can't be resized after initialization with pytest.raises(ValueError): spancat.add_label("Stuff") -def test_implicit_labels(): +@pytest.mark.parametrize("name", SPANCAT_COMPONENTS) +def test_implicit_labels(name): nlp = Language() - spancat = nlp.add_pipe("spancat", config={"spans_key": SPAN_KEY}) + spancat = nlp.add_pipe(name, config={"spans_key": SPAN_KEY}) assert len(spancat.labels) == 0 train_examples = make_examples(nlp) nlp.initialize(get_examples=lambda: train_examples) assert spancat.labels == ("PERSON", "LOC") -def test_explicit_labels(): +@pytest.mark.parametrize("name", SPANCAT_COMPONENTS) +def test_explicit_labels(name): nlp = Language() - spancat = nlp.add_pipe("spancat", config={"spans_key": SPAN_KEY}) + spancat = nlp.add_pipe(name, config={"spans_key": SPAN_KEY}) assert len(spancat.labels) == 0 spancat.add_label("PERSON") spancat.add_label("LOC") @@ -102,13 +108,13 @@ def test_doc_gc(): # XXX This fails with length 0 sometimes assert len(spangroup) > 0 with pytest.raises(RuntimeError): - span = spangroup[0] + spangroup[0] @pytest.mark.parametrize( "max_positive,nr_results", [(None, 4), (1, 2), (2, 3), (3, 4), (4, 4)] ) -def test_make_spangroup(max_positive, nr_results): +def test_make_spangroup_multilabel(max_positive, nr_results): fix_random_seed(0) nlp = Language() spancat = nlp.add_pipe( @@ -120,10 +126,12 @@ def test_make_spangroup(max_positive, nr_results): indices = ngram_suggester([doc])[0].dataXd assert_array_equal(OPS.to_numpy(indices), numpy.asarray([[0, 1], [1, 2], [0, 2]])) labels = ["Thing", "City", "Person", "GreatCity"] + for label in labels: + spancat.add_label(label) scores = numpy.asarray( [[0.2, 0.4, 0.3, 0.1], [0.1, 0.6, 0.2, 0.4], [0.8, 0.7, 0.3, 0.9]], dtype="f" ) - spangroup = spancat._make_span_group(doc, indices, scores, labels) + spangroup = spancat._make_span_group_multilabel(doc, indices, scores) assert len(spangroup) == nr_results # first span is always the second token "London" @@ -154,6 +162,130 @@ def test_make_spangroup(max_positive, nr_results): assert_almost_equal(0.9, spangroup.attrs["scores"][-1], 5) +@pytest.mark.parametrize( + "threshold,allow_overlap,nr_results", + [(0.05, True, 3), (0.05, False, 1), (0.5, True, 2), (0.5, False, 1)], +) +def test_make_spangroup_singlelabel(threshold, allow_overlap, nr_results): + fix_random_seed(0) + nlp = Language() + spancat = nlp.add_pipe( + "spancat", + config={ + "spans_key": SPAN_KEY, + "threshold": threshold, + "max_positive": 1, + }, + ) + doc = nlp.make_doc("Greater London") + ngram_suggester = registry.misc.get("spacy.ngram_suggester.v1")(sizes=[1, 2]) + indices = ngram_suggester([doc])[0].dataXd + assert_array_equal(OPS.to_numpy(indices), numpy.asarray([[0, 1], [1, 2], [0, 2]])) + labels = ["Thing", "City", "Person", "GreatCity"] + for label in labels: + spancat.add_label(label) + scores = numpy.asarray( + [[0.2, 0.4, 0.3, 0.1], [0.1, 0.6, 0.2, 0.4], [0.8, 0.7, 0.3, 0.9]], dtype="f" + ) + spangroup = spancat._make_span_group_singlelabel( + doc, indices, scores, allow_overlap + ) + if threshold > 0.4: + if allow_overlap: + assert spangroup[0].text == "London" + assert spangroup[0].label_ == "City" + assert_almost_equal(0.6, spangroup.attrs["scores"][0], 5) + assert spangroup[1].text == "Greater London" + assert spangroup[1].label_ == "GreatCity" + assert spangroup.attrs["scores"][1] == 0.9 + assert_almost_equal(0.9, spangroup.attrs["scores"][1], 5) + else: + assert spangroup[0].text == "Greater London" + assert spangroup[0].label_ == "GreatCity" + assert spangroup.attrs["scores"][0] == 0.9 + else: + if allow_overlap: + assert spangroup[0].text == "Greater" + assert spangroup[0].label_ == "City" + assert spangroup[1].text == "London" + assert spangroup[1].label_ == "City" + assert spangroup[2].text == "Greater London" + assert spangroup[2].label_ == "GreatCity" + else: + assert spangroup[0].text == "Greater London" + + +def test_make_spangroup_negative_label(): + fix_random_seed(0) + nlp_single = Language() + nlp_multi = Language() + spancat_single = nlp_single.add_pipe( + "spancat", + config={ + "spans_key": SPAN_KEY, + "threshold": 0.1, + "max_positive": 1, + }, + ) + spancat_multi = nlp_multi.add_pipe( + "spancat", + config={ + "spans_key": SPAN_KEY, + "threshold": 0.1, + "max_positive": 2, + }, + ) + spancat_single.add_negative_label = True + spancat_multi.add_negative_label = True + doc = nlp_single.make_doc("Greater London") + labels = ["Thing", "City", "Person", "GreatCity"] + for label in labels: + spancat_multi.add_label(label) + spancat_single.add_label(label) + ngram_suggester = registry.misc.get("spacy.ngram_suggester.v1")(sizes=[1, 2]) + indices = ngram_suggester([doc])[0].dataXd + assert_array_equal(OPS.to_numpy(indices), numpy.asarray([[0, 1], [1, 2], [0, 2]])) + scores = numpy.asarray( + [ + [0.2, 0.4, 0.3, 0.1, 0.1], + [0.1, 0.6, 0.2, 0.4, 0.9], + [0.8, 0.7, 0.3, 0.9, 0.1], + ], + dtype="f", + ) + spangroup_multi = spancat_multi._make_span_group_multilabel(doc, indices, scores) + spangroup_single = spancat_single._make_span_group_singlelabel(doc, indices, scores) + assert len(spangroup_single) == 2 + assert spangroup_single[0].text == "Greater" + assert spangroup_single[0].label_ == "City" + assert_almost_equal(0.4, spangroup_single.attrs["scores"][0], 5) + assert spangroup_single[1].text == "Greater London" + assert spangroup_single[1].label_ == "GreatCity" + assert spangroup_single.attrs["scores"][1] == 0.9 + assert_almost_equal(0.9, spangroup_single.attrs["scores"][1], 5) + + assert len(spangroup_multi) == 6 + assert spangroup_multi[0].text == "Greater" + assert spangroup_multi[0].label_ == "City" + assert_almost_equal(0.4, spangroup_multi.attrs["scores"][0], 5) + assert spangroup_multi[1].text == "Greater" + assert spangroup_multi[1].label_ == "Person" + assert_almost_equal(0.3, spangroup_multi.attrs["scores"][1], 5) + assert spangroup_multi[2].text == "London" + assert spangroup_multi[2].label_ == "City" + assert_almost_equal(0.6, spangroup_multi.attrs["scores"][2], 5) + assert spangroup_multi[3].text == "London" + assert spangroup_multi[3].label_ == "GreatCity" + assert_almost_equal(0.4, spangroup_multi.attrs["scores"][3], 5) + assert spangroup_multi[4].text == "Greater London" + assert spangroup_multi[4].label_ == "Thing" + assert spangroup_multi[4].text == "Greater London" + assert_almost_equal(0.8, spangroup_multi.attrs["scores"][4], 5) + assert spangroup_multi[5].text == "Greater London" + assert spangroup_multi[5].label_ == "GreatCity" + assert_almost_equal(0.9, spangroup_multi.attrs["scores"][5], 5) + + def test_ngram_suggester(en_tokenizer): # test different n-gram lengths for size in [1, 2, 3]: @@ -371,9 +503,9 @@ def test_overfitting_IO_overlapping(): assert set([span.label_ for span in spans2]) == {"LOC", "DOUBLE_LOC"} -def test_zero_suggestions(): +@pytest.mark.parametrize("name", SPANCAT_COMPONENTS) +def test_zero_suggestions(name): # Test with a suggester that can return 0 suggestions - @registry.misc("test_mixed_zero_suggester") def make_mixed_zero_suggester(): def mixed_zero_suggester(docs, *, ops=None): @@ -400,7 +532,7 @@ def test_zero_suggestions(): fix_random_seed(0) nlp = English() spancat = nlp.add_pipe( - "spancat", + name, config={ "suggester": {"@misc": "test_mixed_zero_suggester"}, "spans_key": SPAN_KEY, @@ -408,7 +540,7 @@ def test_zero_suggestions(): ) train_examples = make_examples(nlp) optimizer = nlp.initialize(get_examples=lambda: train_examples) - assert spancat.model.get_dim("nO") == 2 + assert spancat.model.get_dim("nO") == spancat._n_labels assert set(spancat.labels) == {"LOC", "PERSON"} nlp.update(train_examples, sgd=optimizer) @@ -424,9 +556,10 @@ def test_zero_suggestions(): list(nlp.pipe(["", "one", "three three three"])) -def test_set_candidates(): +@pytest.mark.parametrize("name", SPANCAT_COMPONENTS) +def test_set_candidates(name): nlp = Language() - spancat = nlp.add_pipe("spancat", config={"spans_key": SPAN_KEY}) + spancat = nlp.add_pipe(name, config={"spans_key": SPAN_KEY}) train_examples = make_examples(nlp) nlp.initialize(get_examples=lambda: train_examples) texts = [ @@ -444,3 +577,21 @@ def test_set_candidates(): assert len(docs[0].spans["candidates"]) == 9 assert docs[0].spans["candidates"][0].text == "Just" assert docs[0].spans["candidates"][4].text == "Just a" + + +@pytest.mark.parametrize("name", SPANCAT_COMPONENTS) +@pytest.mark.parametrize("n_process", [1, 2]) +def test_spancat_multiprocessing(name, n_process): + if isinstance(get_current_ops, NumpyOps) or n_process < 2: + nlp = Language() + spancat = nlp.add_pipe(name, config={"spans_key": SPAN_KEY}) + train_examples = make_examples(nlp) + nlp.initialize(get_examples=lambda: train_examples) + texts = [ + "Just a sentence.", + "I like London and Berlin", + "I like Berlin", + "I eat ham.", + ] + docs = list(nlp.pipe(texts, n_process=n_process)) + assert len(docs) == len(texts) diff --git a/spacy/tests/pipeline/test_tagger.py b/spacy/tests/pipeline/test_tagger.py index 96e75851e..746f32ee3 100644 --- a/spacy/tests/pipeline/test_tagger.py +++ b/spacy/tests/pipeline/test_tagger.py @@ -1,12 +1,12 @@ import pytest -from numpy.testing import assert_equal +from numpy.testing import assert_equal, assert_almost_equal from spacy.attrs import TAG from spacy import util from spacy.training import Example from spacy.lang.en import English from spacy.language import Language -from thinc.api import compounding +from thinc.api import compounding, get_current_ops from ..util import make_tempdir @@ -67,6 +67,30 @@ PARTIAL_DATA = [ ] +def test_label_smoothing(): + nlp = Language() + tagger_no_ls = nlp.add_pipe("tagger", "no_label_smoothing") + tagger_ls = nlp.add_pipe( + "tagger", "label_smoothing", config=dict(label_smoothing=0.05) + ) + train_examples = [] + losses = {} + for tag in TAGS: + tagger_no_ls.add_label(tag) + tagger_ls.add_label(tag) + for t in TRAIN_DATA: + train_examples.append(Example.from_dict(nlp.make_doc(t[0]), t[1])) + + nlp.initialize(get_examples=lambda: train_examples) + tag_scores, bp_tag_scores = tagger_ls.model.begin_update( + [eg.predicted for eg in train_examples] + ) + ops = get_current_ops() + no_ls_grads = ops.to_numpy(tagger_no_ls.get_loss(train_examples, tag_scores)[1][0]) + ls_grads = ops.to_numpy(tagger_ls.get_loss(train_examples, tag_scores)[1][0]) + assert_almost_equal(ls_grads / no_ls_grads, 0.925) + + def test_no_label(): nlp = Language() nlp.add_pipe("tagger") diff --git a/spacy/tests/pipeline/test_textcat.py b/spacy/tests/pipeline/test_textcat.py index 155ce99a2..d042f3445 100644 --- a/spacy/tests/pipeline/test_textcat.py +++ b/spacy/tests/pipeline/test_textcat.py @@ -895,3 +895,26 @@ def test_textcat_multi_threshold(): scores = nlp.evaluate(train_examples, scorer_cfg={"threshold": 0}) assert scores["cats_f_per_type"]["POSITIVE"]["r"] == 1.0 + + +@pytest.mark.parametrize( + "component_name,scorer", + [ + ("textcat", "spacy.textcat_scorer.v1"), + ("textcat_multilabel", "spacy.textcat_multilabel_scorer.v1"), + ], +) +def test_textcat_legacy_scorers(component_name, scorer): + """Check that legacy scorers are registered and produce the expected score + keys.""" + nlp = English() + nlp.add_pipe(component_name, config={"scorer": {"@scorers": scorer}}) + + train_examples = [] + for text, annotations in TRAIN_DATA_SINGLE_LABEL: + train_examples.append(Example.from_dict(nlp.make_doc(text), annotations)) + nlp.initialize(get_examples=lambda: train_examples) + + # score the model (it's not actually trained but that doesn't matter) + scores = nlp.evaluate(train_examples) + assert 0 <= scores["cats_score"] <= 1 diff --git a/spacy/tests/serialize/test_serialize_doc.py b/spacy/tests/serialize/test_serialize_doc.py index 15bf67bfd..eea13445e 100644 --- a/spacy/tests/serialize/test_serialize_doc.py +++ b/spacy/tests/serialize/test_serialize_doc.py @@ -213,6 +213,13 @@ def test_serialize_doc_exclude(en_vocab): def test_serialize_doc_span_groups(en_vocab): doc = Doc(en_vocab, words=["hello", "world", "!"]) - doc.spans["content"] = [doc[0:2]] + span = doc[0:2] + span.label_ = "test_serialize_doc_span_groups_label" + span.id_ = "test_serialize_doc_span_groups_id" + span.kb_id_ = "test_serialize_doc_span_groups_kb_id" + doc.spans["content"] = [span] new_doc = Doc(en_vocab).from_bytes(doc.to_bytes()) assert len(new_doc.spans["content"]) == 1 + assert new_doc.spans["content"][0].label_ == "test_serialize_doc_span_groups_label" + assert new_doc.spans["content"][0].id_ == "test_serialize_doc_span_groups_id" + assert new_doc.spans["content"][0].kb_id_ == "test_serialize_doc_span_groups_kb_id" diff --git a/spacy/tests/serialize/test_serialize_docbin.py b/spacy/tests/serialize/test_serialize_docbin.py index 9f8e5e06b..6f7b1001c 100644 --- a/spacy/tests/serialize/test_serialize_docbin.py +++ b/spacy/tests/serialize/test_serialize_docbin.py @@ -49,7 +49,11 @@ def test_serialize_doc_bin(): nlp = English() for doc in nlp.pipe(texts): doc.cats = cats - doc.spans["start"] = [doc[0:2]] + span = doc[0:2] + span.label_ = "UNUSUAL_SPAN_LABEL" + span.id_ = "UNUSUAL_SPAN_ID" + span.kb_id_ = "UNUSUAL_SPAN_KB_ID" + doc.spans["start"] = [span] doc[0].norm_ = "UNUSUAL_TOKEN_NORM" doc[0].ent_id_ = "UNUSUAL_TOKEN_ENT_ID" doc_bin.add(doc) @@ -63,6 +67,9 @@ def test_serialize_doc_bin(): assert doc.text == texts[i] assert doc.cats == cats assert len(doc.spans) == 1 + assert doc.spans["start"][0].label_ == "UNUSUAL_SPAN_LABEL" + assert doc.spans["start"][0].id_ == "UNUSUAL_SPAN_ID" + assert doc.spans["start"][0].kb_id_ == "UNUSUAL_SPAN_KB_ID" assert doc[0].norm_ == "UNUSUAL_TOKEN_NORM" assert doc[0].ent_id_ == "UNUSUAL_TOKEN_ENT_ID" diff --git a/spacy/tests/serialize/test_serialize_kb.py b/spacy/tests/serialize/test_serialize_kb.py index 8d3653ab1..f9d2e226b 100644 --- a/spacy/tests/serialize/test_serialize_kb.py +++ b/spacy/tests/serialize/test_serialize_kb.py @@ -1,7 +1,10 @@ -from typing import Callable +from pathlib import Path +from typing import Callable, Iterable, Any, Dict -from spacy import util -from spacy.util import ensure_path, registry, load_model_from_config +import srsly + +from spacy import util, Errors +from spacy.util import ensure_path, registry, load_model_from_config, SimpleFrozenList from spacy.kb.kb_in_memory import InMemoryLookupKB from spacy.vocab import Vocab from thinc.api import Config @@ -91,7 +94,10 @@ def test_serialize_subclassed_kb(): [components.entity_linker] factory = "entity_linker" - + + [components.entity_linker.generate_empty_kb] + @misc = "kb_test.CustomEmptyKB.v1" + [initialize] [initialize.components] @@ -99,7 +105,7 @@ def test_serialize_subclassed_kb(): [initialize.components.entity_linker] [initialize.components.entity_linker.kb_loader] - @misc = "spacy.CustomKB.v1" + @misc = "kb_test.CustomKB.v1" entity_vector_length = 342 custom_field = 666 """ @@ -109,10 +115,57 @@ def test_serialize_subclassed_kb(): super().__init__(vocab, entity_vector_length) self.custom_field = custom_field - @registry.misc("spacy.CustomKB.v1") + def to_disk(self, path, exclude: Iterable[str] = SimpleFrozenList()): + """We overwrite InMemoryLookupKB.to_disk() to ensure that self.custom_field is stored as well.""" + path = ensure_path(path) + if not path.exists(): + path.mkdir(parents=True) + if not path.is_dir(): + raise ValueError(Errors.E928.format(loc=path)) + + def serialize_custom_fields(file_path: Path) -> None: + srsly.write_json(file_path, {"custom_field": self.custom_field}) + + serialize = { + "contents": lambda p: self.write_contents(p), + "strings.json": lambda p: self.vocab.strings.to_disk(p), + "custom_fields": lambda p: serialize_custom_fields(p), + } + util.to_disk(path, serialize, exclude) + + def from_disk(self, path, exclude: Iterable[str] = SimpleFrozenList()): + """We overwrite InMemoryLookupKB.from_disk() to ensure that self.custom_field is loaded as well.""" + path = ensure_path(path) + if not path.exists(): + raise ValueError(Errors.E929.format(loc=path)) + if not path.is_dir(): + raise ValueError(Errors.E928.format(loc=path)) + + def deserialize_custom_fields(file_path: Path) -> None: + self.custom_field = srsly.read_json(file_path)["custom_field"] + + deserialize: Dict[str, Callable[[Any], Any]] = { + "contents": lambda p: self.read_contents(p), + "strings.json": lambda p: self.vocab.strings.from_disk(p), + "custom_fields": lambda p: deserialize_custom_fields(p), + } + util.from_disk(path, deserialize, exclude) + + @registry.misc("kb_test.CustomEmptyKB.v1") + def empty_custom_kb() -> Callable[[Vocab, int], SubInMemoryLookupKB]: + def empty_kb_factory(vocab: Vocab, entity_vector_length: int): + return SubInMemoryLookupKB( + vocab=vocab, + entity_vector_length=entity_vector_length, + custom_field=0, + ) + + return empty_kb_factory + + @registry.misc("kb_test.CustomKB.v1") def custom_kb( entity_vector_length: int, custom_field: int - ) -> Callable[[Vocab], InMemoryLookupKB]: + ) -> Callable[[Vocab], SubInMemoryLookupKB]: def custom_kb_factory(vocab): kb = SubInMemoryLookupKB( vocab=vocab, @@ -139,6 +192,6 @@ def test_serialize_subclassed_kb(): nlp2 = util.load_model_from_path(tmp_dir) entity_linker2 = nlp2.get_pipe("entity_linker") # After IO, the KB is the standard one - assert type(entity_linker2.kb) == InMemoryLookupKB + assert type(entity_linker2.kb) == SubInMemoryLookupKB assert entity_linker2.kb.entity_vector_length == 342 - assert not hasattr(entity_linker2.kb, "custom_field") + assert entity_linker2.kb.custom_field == 666 diff --git a/spacy/tests/test_cli.py b/spacy/tests/test_cli.py index 42af08749..1fdf059b3 100644 --- a/spacy/tests/test_cli.py +++ b/spacy/tests/test_cli.py @@ -2,9 +2,10 @@ import os import math from collections import Counter from typing import Tuple, List, Dict, Any -import pkg_resources import time +from pathlib import Path +import spacy import numpy import pytest import srsly @@ -14,7 +15,7 @@ from thinc.api import Config, ConfigValidationError from spacy import about from spacy.cli import info -from spacy.cli._util import is_subpath_of, load_project_config +from spacy.cli._util import is_subpath_of, load_project_config, walk_directory from spacy.cli._util import parse_config_overrides, string_to_list from spacy.cli._util import substitute_project_variables from spacy.cli._util import validate_project_commands @@ -27,11 +28,13 @@ from spacy.cli.debug_data import _print_span_characteristics from spacy.cli.debug_data import _get_spans_length_freq_dist from spacy.cli.download import get_compatibility, get_version from spacy.cli.init_config import RECOMMENDATIONS, init_config, fill_config +from spacy.cli.init_pipeline import _init_labels from spacy.cli.package import get_third_party_dependencies from spacy.cli.package import _is_permitted_package_name from spacy.cli.project.remote_storage import RemoteStorage from spacy.cli.project.run import _check_requirements from spacy.cli.validate import get_model_pkgs +from spacy.cli.apply import apply from spacy.cli.find_threshold import find_threshold from spacy.lang.en import English from spacy.lang.nl import Dutch @@ -44,7 +47,6 @@ from spacy.training.converters import conll_ner_to_docs, conllu_to_docs from spacy.training.converters import iob_to_docs from spacy.util import ENV_VARS, get_minor_version, load_model_from_config, load_config -from ..cli.init_pipeline import _init_labels from .util import make_tempdir @@ -550,7 +552,14 @@ def test_parse_cli_overrides(): @pytest.mark.parametrize("lang", ["en", "nl"]) @pytest.mark.parametrize( - "pipeline", [["tagger", "parser", "ner"], [], ["ner", "textcat", "sentencizer"]] + "pipeline", + [ + ["tagger", "parser", "ner"], + [], + ["ner", "textcat", "sentencizer"], + ["morphologizer", "spancat", "entity_linker"], + ["spancat_singlelabel", "textcat_multilabel"], + ], ) @pytest.mark.parametrize("optimize", ["efficiency", "accuracy"]) @pytest.mark.parametrize("pretraining", [True, False]) @@ -615,7 +624,6 @@ def test_string_to_list_intify(value): assert string_to_list(value, intify=True) == [1, 2, 3] -@pytest.mark.skip(reason="Temporarily skip for dev version") def test_download_compatibility(): spec = SpecifierSet("==" + about.__version__) spec.prereleases = False @@ -626,7 +634,6 @@ def test_download_compatibility(): assert get_minor_version(about.__version__) == get_minor_version(version) -@pytest.mark.skip(reason="Temporarily skip for dev version") def test_validate_compatibility_table(): spec = SpecifierSet("==" + about.__version__) spec.prereleases = False @@ -885,6 +892,82 @@ def test_span_length_freq_dist_output_must_be_correct(): assert list(span_freqs.keys()) == [3, 1, 4, 5, 2] +def test_applycli_empty_dir(): + with make_tempdir() as data_path: + output = data_path / "test.spacy" + apply(data_path, output, "blank:en", "text", 1, 1) + + +def test_applycli_docbin(): + with make_tempdir() as data_path: + output = data_path / "testout.spacy" + nlp = spacy.blank("en") + doc = nlp("testing apply cli.") + # test empty DocBin case + docbin = DocBin() + docbin.to_disk(data_path / "testin.spacy") + apply(data_path, output, "blank:en", "text", 1, 1) + docbin.add(doc) + docbin.to_disk(data_path / "testin.spacy") + apply(data_path, output, "blank:en", "text", 1, 1) + + +def test_applycli_jsonl(): + with make_tempdir() as data_path: + output = data_path / "testout.spacy" + data = [{"field": "Testing apply cli.", "key": 234}] + data2 = [{"field": "234"}] + srsly.write_jsonl(data_path / "test.jsonl", data) + apply(data_path, output, "blank:en", "field", 1, 1) + srsly.write_jsonl(data_path / "test2.jsonl", data2) + apply(data_path, output, "blank:en", "field", 1, 1) + + +def test_applycli_txt(): + with make_tempdir() as data_path: + output = data_path / "testout.spacy" + with open(data_path / "test.foo", "w") as ftest: + ftest.write("Testing apply cli.") + apply(data_path, output, "blank:en", "text", 1, 1) + + +def test_applycli_mixed(): + with make_tempdir() as data_path: + output = data_path / "testout.spacy" + text = "Testing apply cli" + nlp = spacy.blank("en") + doc = nlp(text) + jsonl_data = [{"text": text}] + srsly.write_jsonl(data_path / "test.jsonl", jsonl_data) + docbin = DocBin() + docbin.add(doc) + docbin.to_disk(data_path / "testin.spacy") + with open(data_path / "test.txt", "w") as ftest: + ftest.write(text) + apply(data_path, output, "blank:en", "text", 1, 1) + # Check whether it worked + result = list(DocBin().from_disk(output).get_docs(nlp.vocab)) + assert len(result) == 3 + for doc in result: + assert doc.text == text + + +def test_applycli_user_data(): + Doc.set_extension("ext", default=0) + val = ("ext", 0) + with make_tempdir() as data_path: + output = data_path / "testout.spacy" + nlp = spacy.blank("en") + doc = nlp("testing apply cli.") + doc._.ext = val + docbin = DocBin(store_user_data=True) + docbin.add(doc) + docbin.to_disk(data_path / "testin.spacy") + apply(data_path, output, "blank:en", "", 1, 1) + result = list(DocBin().from_disk(output).get_docs(nlp.vocab)) + assert result[0]._.ext == val + + def test_local_remote_storage(): with make_tempdir() as d: filename = "a.txt" @@ -940,8 +1023,6 @@ def test_local_remote_storage_pull_missing(): def test_cli_find_threshold(capsys): - thresholds = numpy.linspace(0, 1, 10) - def make_examples(nlp: Language) -> List[Example]: docs: List[Example] = [] @@ -997,7 +1078,7 @@ def test_cli_find_threshold(capsys): ) with make_tempdir() as nlp_dir: nlp.to_disk(nlp_dir) - res = find_threshold( + best_threshold, best_score, res = find_threshold( model=nlp_dir, data_path=docs_dir / "docs.spacy", pipe_name="tc_multi", @@ -1005,16 +1086,14 @@ def test_cli_find_threshold(capsys): scores_key="cats_macro_f", silent=True, ) - assert res[0] != thresholds[0] - assert thresholds[0] < res[0] < thresholds[9] - assert res[1] == 1.0 - assert res[2][1.0] == 0.0 + assert best_score == max(res.values()) + assert res[1.0] == 0.0 # Test with spancat. nlp, _ = init_nlp((("spancat", {}),)) with make_tempdir() as nlp_dir: nlp.to_disk(nlp_dir) - res = find_threshold( + best_threshold, best_score, res = find_threshold( model=nlp_dir, data_path=docs_dir / "docs.spacy", pipe_name="spancat", @@ -1022,10 +1101,8 @@ def test_cli_find_threshold(capsys): scores_key="spans_sc_f", silent=True, ) - assert res[0] != thresholds[0] - assert thresholds[0] < res[0] < thresholds[8] - assert res[1] >= 0.6 - assert res[2][1.0] == 0.0 + assert best_score == max(res.values()) + assert res[1.0] == 0.0 # Having multiple textcat_multilabel components should work, since the name has to be specified. nlp, _ = init_nlp((("textcat_multilabel", {}),)) @@ -1055,6 +1132,7 @@ def test_cli_find_threshold(capsys): ) +@pytest.mark.filterwarnings("ignore::DeprecationWarning") @pytest.mark.parametrize( "reqs,output", [ @@ -1087,6 +1165,8 @@ def test_cli_find_threshold(capsys): ], ) def test_project_check_requirements(reqs, output): + import pkg_resources + # excessive guard against unlikely package name try: pkg_resources.require("spacyunknowndoesnotexist12345") @@ -1107,3 +1187,92 @@ def test_upload_download_local_file(): download_file(remote_file, local_file) with local_file.open(mode="r") as file_: assert file_.read() == content + + +def test_walk_directory(): + with make_tempdir() as d: + files = [ + "data1.iob", + "data2.iob", + "data3.json", + "data4.conll", + "data5.conll", + "data6.conll", + "data7.txt", + ] + + for f in files: + Path(d / f).touch() + + assert (len(walk_directory(d))) == 7 + assert (len(walk_directory(d, suffix=None))) == 7 + assert (len(walk_directory(d, suffix="json"))) == 1 + assert (len(walk_directory(d, suffix="iob"))) == 2 + assert (len(walk_directory(d, suffix="conll"))) == 3 + assert (len(walk_directory(d, suffix="pdf"))) == 0 + + +def test_debug_data_trainable_lemmatizer_basic(): + examples = [ + ("She likes green eggs", {"lemmas": ["she", "like", "green", "egg"]}), + ("Eat blue ham", {"lemmas": ["eat", "blue", "ham"]}), + ] + nlp = Language() + train_examples = [] + for t in examples: + train_examples.append(Example.from_dict(nlp.make_doc(t[0]), t[1])) + + data = _compile_gold(train_examples, ["trainable_lemmatizer"], nlp, True) + # ref test_edit_tree_lemmatizer::test_initialize_from_labels + # this results in 4 trees + assert len(data["lemmatizer_trees"]) == 4 + + +def test_debug_data_trainable_lemmatizer_partial(): + partial_examples = [ + # partial annotation + ("She likes green eggs", {"lemmas": ["", "like", "green", ""]}), + # misaligned partial annotation + ( + "He hates green eggs", + { + "words": ["He", "hat", "es", "green", "eggs"], + "lemmas": ["", "hat", "e", "green", ""], + }, + ), + ] + nlp = Language() + train_examples = [] + for t in partial_examples: + train_examples.append(Example.from_dict(nlp.make_doc(t[0]), t[1])) + + data = _compile_gold(train_examples, ["trainable_lemmatizer"], nlp, True) + assert data["partial_lemma_annotations"] == 2 + + +def test_debug_data_trainable_lemmatizer_low_cardinality(): + low_cardinality_examples = [ + ("She likes green eggs", {"lemmas": ["no", "no", "no", "no"]}), + ("Eat blue ham", {"lemmas": ["no", "no", "no"]}), + ] + nlp = Language() + train_examples = [] + for t in low_cardinality_examples: + train_examples.append(Example.from_dict(nlp.make_doc(t[0]), t[1])) + + data = _compile_gold(train_examples, ["trainable_lemmatizer"], nlp, True) + assert data["n_low_cardinality_lemmas"] == 2 + + +def test_debug_data_trainable_lemmatizer_not_annotated(): + unannotated_examples = [ + ("She likes green eggs", {}), + ("Eat blue ham", {}), + ] + nlp = Language() + train_examples = [] + for t in unannotated_examples: + train_examples.append(Example.from_dict(nlp.make_doc(t[0]), t[1])) + + data = _compile_gold(train_examples, ["trainable_lemmatizer"], nlp, True) + assert data["no_lemma_annotations"] == 2 diff --git a/spacy/tests/test_cli_app.py b/spacy/tests/test_cli_app.py new file mode 100644 index 000000000..9ba4f0e5c --- /dev/null +++ b/spacy/tests/test_cli_app.py @@ -0,0 +1,237 @@ +import os +from pathlib import Path +import pytest +import srsly +from typer.testing import CliRunner +from spacy.tokens import DocBin, Doc + +from spacy.cli._util import app, get_git_version +from .util import make_tempdir, normalize_whitespace + + +def has_git(): + try: + get_git_version() + return True + except RuntimeError: + return False + + +def test_convert_auto(): + with make_tempdir() as d_in, make_tempdir() as d_out: + for f in ["data1.iob", "data2.iob", "data3.iob"]: + Path(d_in / f).touch() + + # ensure that "automatic" suffix detection works + result = CliRunner().invoke(app, ["convert", str(d_in), str(d_out)]) + assert "Generated output file" in result.stdout + out_files = os.listdir(d_out) + assert len(out_files) == 3 + assert "data1.spacy" in out_files + assert "data2.spacy" in out_files + assert "data3.spacy" in out_files + + +def test_convert_auto_conflict(): + with make_tempdir() as d_in, make_tempdir() as d_out: + for f in ["data1.iob", "data2.iob", "data3.json"]: + Path(d_in / f).touch() + + # ensure that "automatic" suffix detection warns when there are different file types + result = CliRunner().invoke(app, ["convert", str(d_in), str(d_out)]) + assert "All input files must be same type" in result.stdout + out_files = os.listdir(d_out) + assert len(out_files) == 0 + + +def test_benchmark_accuracy_alias(): + # Verify that the `evaluate` alias works correctly. + result_benchmark = CliRunner().invoke(app, ["benchmark", "accuracy", "--help"]) + result_evaluate = CliRunner().invoke(app, ["evaluate", "--help"]) + assert normalize_whitespace(result_benchmark.stdout) == normalize_whitespace( + result_evaluate.stdout.replace("spacy evaluate", "spacy benchmark accuracy") + ) + + +def test_debug_data_trainable_lemmatizer_cli(en_vocab): + train_docs = [ + Doc(en_vocab, words=["I", "like", "cats"], lemmas=["I", "like", "cat"]), + Doc( + en_vocab, + words=["Dogs", "are", "great", "too"], + lemmas=["dog", "be", "great", "too"], + ), + ] + dev_docs = [ + Doc(en_vocab, words=["Cats", "are", "cute"], lemmas=["cat", "be", "cute"]), + Doc(en_vocab, words=["Pets", "are", "great"], lemmas=["pet", "be", "great"]), + ] + with make_tempdir() as d_in: + train_bin = DocBin(docs=train_docs) + train_bin.to_disk(d_in / "train.spacy") + dev_bin = DocBin(docs=dev_docs) + dev_bin.to_disk(d_in / "dev.spacy") + # `debug data` requires an input pipeline config + CliRunner().invoke( + app, + [ + "init", + "config", + f"{d_in}/config.cfg", + "--lang", + "en", + "--pipeline", + "trainable_lemmatizer", + ], + ) + result_debug_data = CliRunner().invoke( + app, + [ + "debug", + "data", + f"{d_in}/config.cfg", + "--paths.train", + f"{d_in}/train.spacy", + "--paths.dev", + f"{d_in}/dev.spacy", + ], + ) + # Instead of checking specific wording of the output, which may change, + # we'll check that this section of the debug output is present. + assert "= Trainable Lemmatizer =" in result_debug_data.stdout + + +# project tests + +SAMPLE_PROJECT = { + "title": "Sample project", + "description": "This is a project for testing", + "assets": [ + { + "dest": "assets/spacy-readme.md", + "url": "https://github.com/explosion/spaCy/raw/dec81508d28b47f09a06203c472b37f00db6c869/README.md", + "checksum": "411b2c89ccf34288fae8ed126bf652f7", + }, + { + "dest": "assets/citation.cff", + "url": "https://github.com/explosion/spaCy/raw/master/CITATION.cff", + "checksum": "c996bfd80202d480eb2e592369714e5e", + "extra": True, + }, + ], + "commands": [ + { + "name": "ok", + "help": "print ok", + "script": ["python -c \"print('okokok')\""], + }, + { + "name": "create", + "help": "make a file", + "script": ["touch abc.txt"], + "outputs": ["abc.txt"], + }, + { + "name": "clean", + "help": "remove test file", + "script": ["rm abc.txt"], + }, + ], +} + +SAMPLE_PROJECT_TEXT = srsly.yaml_dumps(SAMPLE_PROJECT) + + +@pytest.fixture +def project_dir(): + with make_tempdir() as pdir: + (pdir / "project.yml").write_text(SAMPLE_PROJECT_TEXT) + yield pdir + + +def test_project_document(project_dir): + readme_path = project_dir / "README.md" + assert not readme_path.exists(), "README already exists" + result = CliRunner().invoke( + app, ["project", "document", str(project_dir), "-o", str(readme_path)] + ) + assert result.exit_code == 0 + assert readme_path.is_file() + text = readme_path.read_text("utf-8") + assert SAMPLE_PROJECT["description"] in text + + +def test_project_assets(project_dir): + asset_dir = project_dir / "assets" + assert not asset_dir.exists(), "Assets dir is already present" + result = CliRunner().invoke(app, ["project", "assets", str(project_dir)]) + assert result.exit_code == 0 + assert (asset_dir / "spacy-readme.md").is_file(), "Assets not downloaded" + # check that extras work + result = CliRunner().invoke(app, ["project", "assets", "--extra", str(project_dir)]) + assert result.exit_code == 0 + assert (asset_dir / "citation.cff").is_file(), "Extras not downloaded" + + +def test_project_run(project_dir): + # make sure dry run works + test_file = project_dir / "abc.txt" + result = CliRunner().invoke( + app, ["project", "run", "--dry", "create", str(project_dir)] + ) + assert result.exit_code == 0 + assert not test_file.is_file() + result = CliRunner().invoke(app, ["project", "run", "create", str(project_dir)]) + assert result.exit_code == 0 + assert test_file.is_file() + result = CliRunner().invoke(app, ["project", "run", "ok", str(project_dir)]) + assert result.exit_code == 0 + assert "okokok" in result.stdout + + +@pytest.mark.skipif(not has_git(), reason="git not installed") +@pytest.mark.parametrize( + "options", + [ + "", + # "--sparse", + "--branch v3", + "--repo https://github.com/explosion/projects --branch v3", + ], +) +def test_project_clone(options): + with make_tempdir() as workspace: + out = workspace / "project" + target = "benchmarks/ner_conll03" + if not options: + options = [] + else: + options = options.split() + result = CliRunner().invoke( + app, ["project", "clone", target, *options, str(out)] + ) + assert result.exit_code == 0 + assert (out / "README.md").is_file() + + +def test_project_push_pull(project_dir): + proj = dict(SAMPLE_PROJECT) + remote = "xyz" + + with make_tempdir() as remote_dir: + proj["remotes"] = {remote: str(remote_dir)} + proj_text = srsly.yaml_dumps(proj) + (project_dir / "project.yml").write_text(proj_text) + + test_file = project_dir / "abc.txt" + result = CliRunner().invoke(app, ["project", "run", "create", str(project_dir)]) + assert result.exit_code == 0 + assert test_file.is_file() + result = CliRunner().invoke(app, ["project", "push", remote, str(project_dir)]) + assert result.exit_code == 0 + result = CliRunner().invoke(app, ["project", "run", "clean", str(project_dir)]) + assert result.exit_code == 0 + assert not test_file.exists() + result = CliRunner().invoke(app, ["project", "pull", remote, str(project_dir)]) + assert result.exit_code == 0 + assert test_file.is_file() diff --git a/spacy/tests/test_displacy.py b/spacy/tests/test_displacy.py index f298b38e0..837a92e02 100644 --- a/spacy/tests/test_displacy.py +++ b/spacy/tests/test_displacy.py @@ -275,6 +275,20 @@ def test_displacy_parse_deps(en_vocab): {"start": 2, "end": 3, "label": "det", "dir": "left"}, {"start": 1, "end": 3, "label": "attr", "dir": "right"}, ] + # Test that displacy.parse_deps converts Span to Doc + deps = displacy.parse_deps(doc[:]) + assert isinstance(deps, dict) + assert deps["words"] == [ + {"lemma": None, "text": words[0], "tag": pos[0]}, + {"lemma": None, "text": words[1], "tag": pos[1]}, + {"lemma": None, "text": words[2], "tag": pos[2]}, + {"lemma": None, "text": words[3], "tag": pos[3]}, + ] + assert deps["arcs"] == [ + {"start": 0, "end": 1, "label": "nsubj", "dir": "left"}, + {"start": 2, "end": 3, "label": "det", "dir": "left"}, + {"start": 1, "end": 3, "label": "attr", "dir": "right"}, + ] def test_displacy_invalid_arcs(): diff --git a/spacy/tests/test_language.py b/spacy/tests/test_language.py index 03a98d32f..236856dad 100644 --- a/spacy/tests/test_language.py +++ b/spacy/tests/test_language.py @@ -3,6 +3,7 @@ import logging from unittest import mock import pytest from spacy.language import Language +from spacy.scorer import Scorer from spacy.tokens import Doc, Span from spacy.vocab import Vocab from spacy.training import Example @@ -45,7 +46,7 @@ def assert_sents_error(doc): def warn_error(proc_name, proc, docs, e): logger = logging.getLogger("spacy") - logger.warning(f"Trouble with component {proc_name}.") + logger.warning("Trouble with component %s.", proc_name) @pytest.fixture @@ -126,6 +127,112 @@ def test_evaluate_no_pipe(nlp): nlp.evaluate([Example.from_dict(doc, annots)]) +def test_evaluate_textcat_multilabel(en_vocab): + """Test that evaluate works with a multilabel textcat pipe.""" + nlp = Language(en_vocab) + textcat_multilabel = nlp.add_pipe("textcat_multilabel") + for label in ("FEATURE", "REQUEST", "BUG", "QUESTION"): + textcat_multilabel.add_label(label) + nlp.initialize() + + annots = {"cats": {"FEATURE": 1.0, "QUESTION": 1.0}} + doc = nlp.make_doc("hello world") + example = Example.from_dict(doc, annots) + scores = nlp.evaluate([example]) + labels = nlp.get_pipe("textcat_multilabel").labels + for label in labels: + assert scores["cats_f_per_type"].get(label) is not None + for key in example.reference.cats.keys(): + if key not in labels: + assert scores["cats_f_per_type"].get(key) is None + + +def test_evaluate_multiple_textcat_final(en_vocab): + """Test that evaluate evaluates the final textcat component in a pipeline + with more than one textcat or textcat_multilabel.""" + nlp = Language(en_vocab) + textcat = nlp.add_pipe("textcat") + for label in ("POSITIVE", "NEGATIVE"): + textcat.add_label(label) + textcat_multilabel = nlp.add_pipe("textcat_multilabel") + for label in ("FEATURE", "REQUEST", "BUG", "QUESTION"): + textcat_multilabel.add_label(label) + nlp.initialize() + + annots = { + "cats": { + "POSITIVE": 1.0, + "NEGATIVE": 0.0, + "FEATURE": 1.0, + "QUESTION": 1.0, + "POSITIVE": 1.0, + "NEGATIVE": 0.0, + } + } + doc = nlp.make_doc("hello world") + example = Example.from_dict(doc, annots) + scores = nlp.evaluate([example]) + # get the labels from the final pipe + labels = nlp.get_pipe(nlp.pipe_names[-1]).labels + for label in labels: + assert scores["cats_f_per_type"].get(label) is not None + for key in example.reference.cats.keys(): + if key not in labels: + assert scores["cats_f_per_type"].get(key) is None + + +def test_evaluate_multiple_textcat_separate(en_vocab): + """Test that evaluate can evaluate multiple textcat components separately + with custom scorers.""" + + def custom_textcat_score(examples, **kwargs): + scores = Scorer.score_cats( + examples, + "cats", + multi_label=False, + **kwargs, + ) + return {f"custom_{k}": v for k, v in scores.items()} + + @spacy.registry.scorers("test_custom_textcat_scorer") + def make_custom_textcat_scorer(): + return custom_textcat_score + + nlp = Language(en_vocab) + textcat = nlp.add_pipe( + "textcat", + config={"scorer": {"@scorers": "test_custom_textcat_scorer"}}, + ) + for label in ("POSITIVE", "NEGATIVE"): + textcat.add_label(label) + textcat_multilabel = nlp.add_pipe("textcat_multilabel") + for label in ("FEATURE", "REQUEST", "BUG", "QUESTION"): + textcat_multilabel.add_label(label) + nlp.initialize() + + annots = { + "cats": { + "POSITIVE": 1.0, + "NEGATIVE": 0.0, + "FEATURE": 1.0, + "QUESTION": 1.0, + "POSITIVE": 1.0, + "NEGATIVE": 0.0, + } + } + doc = nlp.make_doc("hello world") + example = Example.from_dict(doc, annots) + scores = nlp.evaluate([example]) + # check custom scores for the textcat pipe + assert "custom_cats_f_per_type" in scores + labels = nlp.get_pipe("textcat").labels + assert set(scores["custom_cats_f_per_type"].keys()) == set(labels) + # check default scores for the textcat_multilabel pipe + assert "cats_f_per_type" in scores + labels = nlp.get_pipe("textcat_multilabel").labels + assert set(scores["cats_f_per_type"].keys()) == set(labels) + + def vector_modification_pipe(doc): doc.vector += 1 return doc diff --git a/spacy/tests/test_misc.py b/spacy/tests/test_misc.py index 1c9b045ac..618f17334 100644 --- a/spacy/tests/test_misc.py +++ b/spacy/tests/test_misc.py @@ -8,7 +8,7 @@ from spacy import prefer_gpu, require_gpu, require_cpu from spacy.ml._precomputable_affine import PrecomputableAffine from spacy.ml._precomputable_affine import _backprop_precomputable_affine_padding from spacy.util import dot_to_object, SimpleFrozenList, import_file -from spacy.util import to_ternary_int +from spacy.util import to_ternary_int, find_available_port from thinc.api import Config, Optimizer, ConfigValidationError from thinc.api import get_current_ops, set_current_ops, NumpyOps, CupyOps, MPSOps from thinc.compat import has_cupy_gpu, has_torch_mps_gpu @@ -434,3 +434,16 @@ def test_to_ternary_int(): assert to_ternary_int(-10) == -1 assert to_ternary_int("string") == -1 assert to_ternary_int([0, "string"]) == -1 + + +def test_find_available_port(): + host = "0.0.0.0" + port = 5000 + assert find_available_port(port, host) == port, "Port 5000 isn't free" + + from wsgiref.simple_server import make_server, demo_app + + with make_server(host, port, demo_app) as httpd: + with pytest.warns(UserWarning, match="already in use"): + found_port = find_available_port(port, host, auto_select=True) + assert found_port == port + 1, "Didn't find next port" diff --git a/spacy/tests/test_scorer.py b/spacy/tests/test_scorer.py index b903f1669..dbb47b423 100644 --- a/spacy/tests/test_scorer.py +++ b/spacy/tests/test_scorer.py @@ -110,7 +110,7 @@ def test_tokenization(sented_doc): ) example.predicted[1].is_sent_start = False scores = scorer.score([example]) - assert scores["token_acc"] == approx(0.66666666) + assert scores["token_acc"] == 0.5 assert scores["token_p"] == 0.5 assert scores["token_r"] == approx(0.33333333) assert scores["token_f"] == 0.4 diff --git a/spacy/tests/training/test_corpus.py b/spacy/tests/training/test_corpus.py new file mode 100644 index 000000000..b4f9cc13a --- /dev/null +++ b/spacy/tests/training/test_corpus.py @@ -0,0 +1,78 @@ +from typing import IO, Generator, Iterable, List, TextIO, Tuple +from contextlib import contextmanager +from pathlib import Path +import pytest +import tempfile + +from spacy.lang.en import English +from spacy.training import Example, PlainTextCorpus +from spacy.util import make_tempdir + +# Intentional newlines to check that they are skipped. +PLAIN_TEXT_DOC = """ + +This is a doc. It contains two sentences. +This is another doc. + +A third doc. + +""" + +PLAIN_TEXT_DOC_TOKENIZED = [ + [ + "This", + "is", + "a", + "doc", + ".", + "It", + "contains", + "two", + "sentences", + ".", + ], + ["This", "is", "another", "doc", "."], + ["A", "third", "doc", "."], +] + + +@pytest.mark.parametrize("min_length", [0, 5]) +@pytest.mark.parametrize("max_length", [0, 5]) +def test_plain_text_reader(min_length, max_length): + nlp = English() + with _string_to_tmp_file(PLAIN_TEXT_DOC) as file_path: + corpus = PlainTextCorpus( + file_path, min_length=min_length, max_length=max_length + ) + + check = [ + doc + for doc in PLAIN_TEXT_DOC_TOKENIZED + if len(doc) >= min_length and (max_length == 0 or len(doc) <= max_length) + ] + reference, predicted = _examples_to_tokens(corpus(nlp)) + + assert reference == check + assert predicted == check + + +@contextmanager +def _string_to_tmp_file(s: str) -> Generator[Path, None, None]: + with make_tempdir() as d: + file_path = Path(d) / "string.txt" + with open(file_path, "w", encoding="utf-8") as f: + f.write(s) + yield file_path + + +def _examples_to_tokens( + examples: Iterable[Example], +) -> Tuple[List[List[str]], List[List[str]]]: + reference = [] + predicted = [] + + for eg in examples: + reference.append([t.text for t in eg.reference]) + predicted.append([t.text for t in eg.predicted]) + + return reference, predicted diff --git a/spacy/tests/training/test_pretraining.py b/spacy/tests/training/test_pretraining.py index 9359c8485..6cfdeed20 100644 --- a/spacy/tests/training/test_pretraining.py +++ b/spacy/tests/training/test_pretraining.py @@ -2,17 +2,19 @@ from pathlib import Path import numpy as np import pytest import srsly -from spacy.vocab import Vocab -from thinc.api import Config +from thinc.api import Config, get_current_ops +from spacy import util +from spacy.lang.en import English +from spacy.training.initialize import init_nlp +from spacy.training.loop import train +from spacy.training.pretrain import pretrain +from spacy.tokens import Doc, DocBin +from spacy.language import DEFAULT_CONFIG_PRETRAIN_PATH, DEFAULT_CONFIG_PATH +from spacy.ml.models.multi_task import create_pretrain_vectors +from spacy.vectors import Vectors +from spacy.vocab import Vocab from ..util import make_tempdir -from ... import util -from ...lang.en import English -from ...training.initialize import init_nlp -from ...training.loop import train -from ...training.pretrain import pretrain -from ...tokens import Doc, DocBin -from ...language import DEFAULT_CONFIG_PRETRAIN_PATH, DEFAULT_CONFIG_PATH pretrain_string_listener = """ [nlp] @@ -163,7 +165,8 @@ def test_pretraining_default(): @pytest.mark.parametrize("objective", CHAR_OBJECTIVES) -def test_pretraining_tok2vec_characters(objective): +@pytest.mark.parametrize("skip_last", (True, False)) +def test_pretraining_tok2vec_characters(objective, skip_last): """Test that pretraining works with the character objective""" config = Config().from_str(pretrain_string_listener) config["pretraining"]["objective"] = objective @@ -176,10 +179,14 @@ def test_pretraining_tok2vec_characters(objective): filled["paths"]["raw_text"] = file_path filled = filled.interpolate() assert filled["pretraining"]["component"] == "tok2vec" - pretrain(filled, tmp_dir) + pretrain(filled, tmp_dir, skip_last=skip_last) assert Path(tmp_dir / "model0.bin").exists() assert Path(tmp_dir / "model4.bin").exists() assert not Path(tmp_dir / "model5.bin").exists() + if skip_last: + assert not Path(tmp_dir / "model-last.bin").exists() + else: + assert Path(tmp_dir / "model-last.bin").exists() @pytest.mark.parametrize("objective", VECTOR_OBJECTIVES) @@ -235,6 +242,7 @@ def test_pretraining_tagger_tok2vec(config): pretrain(filled, tmp_dir) assert Path(tmp_dir / "model0.bin").exists() assert Path(tmp_dir / "model4.bin").exists() + assert Path(tmp_dir / "model-last.bin").exists() assert not Path(tmp_dir / "model5.bin").exists() @@ -346,3 +354,26 @@ def write_vectors_model(tmp_dir): nlp = English(vocab) nlp.to_disk(nlp_path) return str(nlp_path) + + +def test_pretrain_default_vectors(): + nlp = English() + nlp.add_pipe("tok2vec") + nlp.initialize() + + # default vectors are supported + nlp.vocab.vectors = Vectors(shape=(10, 10)) + create_pretrain_vectors(1, 1, "cosine")(nlp.vocab, nlp.get_pipe("tok2vec").model) + + # floret vectors are supported + nlp.vocab.vectors = Vectors( + data=get_current_ops().xp.zeros((10, 10)), mode="floret", hash_count=1 + ) + create_pretrain_vectors(1, 1, "cosine")(nlp.vocab, nlp.get_pipe("tok2vec").model) + + # error for no vectors + with pytest.raises(ValueError, match="E875"): + nlp.vocab.vectors = Vectors() + create_pretrain_vectors(1, 1, "cosine")( + nlp.vocab, nlp.get_pipe("tok2vec").model + ) diff --git a/spacy/tests/util.py b/spacy/tests/util.py index d5f3c39ff..c2647558d 100644 --- a/spacy/tests/util.py +++ b/spacy/tests/util.py @@ -1,6 +1,7 @@ import numpy import tempfile import contextlib +import re import srsly from spacy.tokens import Doc from spacy.vocab import Vocab @@ -95,3 +96,7 @@ def assert_packed_msg_equal(b1, b2): for (k1, v1), (k2, v2) in zip(sorted(msg1.items()), sorted(msg2.items())): assert k1 == k2 assert v1 == v2 + + +def normalize_whitespace(s): + return re.sub(r"\s+", " ", s) diff --git a/spacy/tokens/_serialize.py b/spacy/tokens/_serialize.py index c4e8f26f4..73c857d1f 100644 --- a/spacy/tokens/_serialize.py +++ b/spacy/tokens/_serialize.py @@ -124,6 +124,10 @@ class DocBin: for key, group in doc.spans.items(): for span in group: self.strings.add(span.label_) + if span.kb_id in span.doc.vocab.strings: + self.strings.add(span.kb_id_) + if span.id in span.doc.vocab.strings: + self.strings.add(span.id_) def get_docs(self, vocab: Vocab) -> Iterator[Doc]: """Recover Doc objects from the annotations, using the given vocab. diff --git a/spacy/tokens/doc.pyi b/spacy/tokens/doc.pyi index 7760b4b5e..6deb83ec0 100644 --- a/spacy/tokens/doc.pyi +++ b/spacy/tokens/doc.pyi @@ -110,6 +110,7 @@ class Doc: kb_id: Union[int, str] = ..., vector: Optional[Floats1d] = ..., alignment_mode: str = ..., + span_id: Union[int, str] = ..., ) -> Span: ... def similarity(self, other: Union[Doc, Span, Token, Lexeme]) -> float: ... @property diff --git a/spacy/tokens/doc.pyx b/spacy/tokens/doc.pyx index ca190cbe0..c5869b967 100644 --- a/spacy/tokens/doc.pyx +++ b/spacy/tokens/doc.pyx @@ -530,9 +530,9 @@ cdef class Doc: doc (Doc): The parent document. start_idx (int): The index of the first character of the span. end_idx (int): The index of the first character after the span. - label (uint64 or string): A label to attach to the Span, e.g. for + label (Union[int, str]): A label to attach to the Span, e.g. for named entities. - kb_id (uint64 or string): An ID from a KB to capture the meaning of a + kb_id (Union[int, str]): An ID from a KB to capture the meaning of a named entity. vector (ndarray[ndim=1, dtype='float32']): A meaning representation of the span. @@ -541,14 +541,11 @@ cdef class Doc: with token boundaries), "contract" (span of all tokens completely within the character span), "expand" (span of all tokens at least partially covered by the character span). Defaults to "strict". + span_id (Union[int, str]): An identifier to associate with the span. RETURNS (Span): The newly constructed object. DOCS: https://spacy.io/api/doc#char_span """ - if not isinstance(label, int): - label = self.vocab.strings.add(label) - if not isinstance(kb_id, int): - kb_id = self.vocab.strings.add(kb_id) alignment_modes = ("strict", "contract", "expand") if alignment_mode not in alignment_modes: raise ValueError( @@ -1359,6 +1356,10 @@ cdef class Doc: for group in self.spans.values(): for span in group: strings.add(span.label_) + if span.kb_id in span.doc.vocab.strings: + strings.add(span.kb_id_) + if span.id in span.doc.vocab.strings: + strings.add(span.id_) # Msgpack doesn't distinguish between lists and tuples, which is # vexing for user data. As a best guess, we *know* that within # keys, we must have tuples. In values we just have to hope diff --git a/spacy/tokens/span.pyi b/spacy/tokens/span.pyi index 0a6f306a6..a92f19e20 100644 --- a/spacy/tokens/span.pyi +++ b/spacy/tokens/span.pyi @@ -95,9 +95,12 @@ class Span: self, start_idx: int, end_idx: int, - label: int = ..., - kb_id: int = ..., + label: Union[int, str] = ..., + kb_id: Union[int, str] = ..., vector: Optional[Floats1d] = ..., + id: Union[int, str] = ..., + alignment_mode: str = ..., + span_id: Union[int, str] = ..., ) -> Span: ... @property def conjuncts(self) -> Tuple[Token]: ... diff --git a/spacy/tokens/span.pyx b/spacy/tokens/span.pyx index 75ef5df5b..29b8ce703 100644 --- a/spacy/tokens/span.pyx +++ b/spacy/tokens/span.pyx @@ -362,7 +362,7 @@ cdef class Span: result = xp.dot(vector, other.vector) / (self.vector_norm * other.vector_norm) # ensure we get a scalar back (numpy does this automatically but cupy doesn't) return result.item() - + cpdef np.ndarray to_array(self, object py_attr_ids): """Given a list of M attribute IDs, export the tokens to a numpy `ndarray` of shape `(N, M)`, where `N` is the length of the document. @@ -467,7 +467,6 @@ cdef class Span: if start == self.doc.length - 1: yield Span(self.doc, start, self.doc.length) - @property def ents(self): """The named entities that fall completely within the span. Returns @@ -643,21 +642,28 @@ cdef class Span: else: return self.doc[root] - def char_span(self, int start_idx, int end_idx, label=0, kb_id=0, vector=None, id=0): + def char_span(self, int start_idx, int end_idx, label=0, kb_id=0, vector=None, id=0, alignment_mode="strict", span_id=0): """Create a `Span` object from the slice `span.text[start : end]`. start (int): The index of the first character of the span. end (int): The index of the first character after the span. - label (uint64 or string): A label to attach to the Span, e.g. for + label (Union[int, str]): A label to attach to the Span, e.g. for named entities. - kb_id (uint64 or string): An ID from a KB to capture the meaning of a named entity. + kb_id (Union[int, str]): An ID from a KB to capture the meaning of a named entity. vector (ndarray[ndim=1, dtype='float32']): A meaning representation of the span. + id (Union[int, str]): Unused. + alignment_mode (str): How character indices are aligned to token + boundaries. Options: "strict" (character indices must be aligned + with token boundaries), "contract" (span of all tokens completely + within the character span), "expand" (span of all tokens at least + partially covered by the character span). Defaults to "strict". + span_id (Union[int, str]): An identifier to associate with the span. RETURNS (Span): The newly constructed object. """ start_idx += self.c.start_char end_idx += self.c.start_char - return self.doc.char_span(start_idx, end_idx, label=label, kb_id=kb_id, vector=vector) + return self.doc.char_span(start_idx, end_idx, label=label, kb_id=kb_id, vector=vector, alignment_mode=alignment_mode, span_id=span_id) @property def conjuncts(self): diff --git a/spacy/tokens/span_group.pyi b/spacy/tokens/span_group.pyi index 21cd124ab..0b4aa83aa 100644 --- a/spacy/tokens/span_group.pyi +++ b/spacy/tokens/span_group.pyi @@ -18,6 +18,7 @@ class SpanGroup: def doc(self) -> Doc: ... @property def has_overlap(self) -> bool: ... + def __iter__(self): ... def __len__(self) -> int: ... def append(self, span: Span) -> None: ... def extend(self, spans: Iterable[Span]) -> None: ... diff --git a/spacy/tokens/span_group.pyx b/spacy/tokens/span_group.pyx index 1aa3c0bc8..608dda283 100644 --- a/spacy/tokens/span_group.pyx +++ b/spacy/tokens/span_group.pyx @@ -158,6 +158,16 @@ cdef class SpanGroup: return self._concat(other) return NotImplemented + def __iter__(self): + """ + Iterate over the spans in this SpanGroup. + YIELDS (Span): A span in this SpanGroup. + + DOCS: https://spacy.io/api/spangroup#iter + """ + for i in range(self.c.size()): + yield self[i] + def append(self, Span span): """Add a span to the group. The span must refer to the same Doc object as the span group. diff --git a/spacy/training/__init__.py b/spacy/training/__init__.py index 71d1fa775..a6f873f05 100644 --- a/spacy/training/__init__.py +++ b/spacy/training/__init__.py @@ -1,4 +1,4 @@ -from .corpus import Corpus, JsonlCorpus # noqa: F401 +from .corpus import Corpus, JsonlCorpus, PlainTextCorpus # noqa: F401 from .example import Example, validate_examples, validate_get_examples # noqa: F401 from .alignment import Alignment # noqa: F401 from .augment import dont_augment, orth_variants_augmenter # noqa: F401 diff --git a/spacy/training/callbacks.py b/spacy/training/callbacks.py index 426fddf90..7e2494f5b 100644 --- a/spacy/training/callbacks.py +++ b/spacy/training/callbacks.py @@ -11,7 +11,7 @@ def create_copy_from_base_model( ) -> Callable[[Language], Language]: def copy_from_base_model(nlp): if tokenizer: - logger.info(f"Copying tokenizer from: {tokenizer}") + logger.info("Copying tokenizer from: %s", tokenizer) base_nlp = load_model(tokenizer) if nlp.config["nlp"]["tokenizer"] == base_nlp.config["nlp"]["tokenizer"]: nlp.tokenizer.from_bytes(base_nlp.tokenizer.to_bytes(exclude=["vocab"])) @@ -23,7 +23,7 @@ def create_copy_from_base_model( ) ) if vocab: - logger.info(f"Copying vocab from: {vocab}") + logger.info("Copying vocab from: %s", vocab) # only reload if the vocab is from a different model if tokenizer != vocab: base_nlp = load_model(vocab) diff --git a/spacy/training/corpus.py b/spacy/training/corpus.py index b9f929fcd..086ad831c 100644 --- a/spacy/training/corpus.py +++ b/spacy/training/corpus.py @@ -29,7 +29,7 @@ def create_docbin_reader( ) -> Callable[["Language"], Iterable[Example]]: if path is None: raise ValueError(Errors.E913) - util.logger.debug(f"Loading corpus from path: {path}") + util.logger.debug("Loading corpus from path: %s", path) return Corpus( path, gold_preproc=gold_preproc, @@ -58,6 +58,28 @@ def read_labels(path: Path, *, require: bool = False): return srsly.read_json(path) +@util.registry.readers("spacy.PlainTextCorpus.v1") +def create_plain_text_reader( + path: Optional[Path], + min_length: int = 0, + max_length: int = 0, +) -> Callable[["Language"], Iterable[Doc]]: + """Iterate Example objects from a file or directory of plain text + UTF-8 files with one line per doc. + + path (Path): The directory or filename to read from. + min_length (int): Minimum document length (in tokens). Shorter documents + will be skipped. Defaults to 0, which indicates no limit. + max_length (int): Maximum document length (in tokens). Longer documents will + be skipped. Defaults to 0, which indicates no limit. + + DOCS: https://spacy.io/api/corpus#plaintextcorpus + """ + if path is None: + raise ValueError(Errors.E913) + return PlainTextCorpus(path, min_length=min_length, max_length=max_length) + + def walk_corpus(path: Union[str, Path], file_type) -> List[Path]: path = util.ensure_path(path) if not path.is_dir() and path.parts[-1].endswith(file_type): @@ -257,3 +279,52 @@ class JsonlCorpus: # We don't *need* an example here, but it seems nice to # make it match the Corpus signature. yield Example(doc, Doc(nlp.vocab, words=words, spaces=spaces)) + + +class PlainTextCorpus: + """Iterate Example objects from a file or directory of plain text + UTF-8 files with one line per doc. + + path (Path): The directory or filename to read from. + min_length (int): Minimum document length (in tokens). Shorter documents + will be skipped. Defaults to 0, which indicates no limit. + max_length (int): Maximum document length (in tokens). Longer documents will + be skipped. Defaults to 0, which indicates no limit. + + DOCS: https://spacy.io/api/corpus#plaintextcorpus + """ + + file_type = "txt" + + def __init__( + self, + path: Optional[Union[str, Path]], + *, + min_length: int = 0, + max_length: int = 0, + ) -> None: + self.path = util.ensure_path(path) + self.min_length = min_length + self.max_length = max_length + + def __call__(self, nlp: "Language") -> Iterator[Example]: + """Yield examples from the data. + + nlp (Language): The current nlp object. + YIELDS (Example): The example objects. + + DOCS: https://spacy.io/api/corpus#plaintextcorpus-call + """ + for loc in walk_corpus(self.path, ".txt"): + with open(loc, encoding="utf-8") as f: + for text in f: + text = text.rstrip("\r\n") + if len(text): + doc = nlp.make_doc(text) + if self.min_length >= 1 and len(doc) < self.min_length: + continue + elif self.max_length >= 1 and len(doc) > self.max_length: + continue + # We don't *need* an example here, but it seems nice to + # make it match the Corpus signature. + yield Example(doc, doc.copy()) diff --git a/spacy/training/initialize.py b/spacy/training/initialize.py index 6304e4a84..e90617852 100644 --- a/spacy/training/initialize.py +++ b/spacy/training/initialize.py @@ -62,10 +62,10 @@ def init_nlp(config: Config, *, use_gpu: int = -1) -> "Language": frozen_components = T["frozen_components"] # Sourced components that require resume_training resume_components = [p for p in sourced if p not in frozen_components] - logger.info(f"Pipeline: {nlp.pipe_names}") + logger.info("Pipeline: %s", nlp.pipe_names) if resume_components: with nlp.select_pipes(enable=resume_components): - logger.info(f"Resuming training for: {resume_components}") + logger.info("Resuming training for: %s", resume_components) nlp.resume_training(sgd=optimizer) # Make sure that listeners are defined before initializing further nlp._link_components() @@ -73,16 +73,17 @@ def init_nlp(config: Config, *, use_gpu: int = -1) -> "Language": if T["max_epochs"] == -1: sample_size = 100 logger.debug( - f"Due to streamed train corpus, using only first {sample_size} " - f"examples for initialization. If necessary, provide all labels " - f"in [initialize]. More info: https://spacy.io/api/cli#init_labels" + "Due to streamed train corpus, using only first %s examples for initialization. " + "If necessary, provide all labels in [initialize]. " + "More info: https://spacy.io/api/cli#init_labels", + sample_size, ) nlp.initialize( lambda: islice(train_corpus(nlp), sample_size), sgd=optimizer ) else: nlp.initialize(lambda: train_corpus(nlp), sgd=optimizer) - logger.info(f"Initialized pipeline components: {nlp.pipe_names}") + logger.info("Initialized pipeline components: %s", nlp.pipe_names) # Detect components with listeners that are not frozen consistently for name, proc in nlp.pipeline: for listener in getattr( @@ -109,7 +110,7 @@ def init_vocab( ) -> None: if lookups: nlp.vocab.lookups = lookups - logger.info(f"Added vocab lookups: {', '.join(lookups.tables)}") + logger.info("Added vocab lookups: %s", ", ".join(lookups.tables)) data_path = ensure_path(data) if data_path is not None: lex_attrs = srsly.read_jsonl(data_path) @@ -125,11 +126,11 @@ def init_vocab( else: oov_prob = DEFAULT_OOV_PROB nlp.vocab.cfg.update({"oov_prob": oov_prob}) - logger.info(f"Added {len(nlp.vocab)} lexical entries to the vocab") + logger.info("Added %d lexical entries to the vocab", len(nlp.vocab)) logger.info("Created vocabulary") if vectors is not None: load_vectors_into_model(nlp, vectors) - logger.info(f"Added vectors: {vectors}") + logger.info("Added vectors: %s", vectors) # warn if source model vectors are not identical sourced_vectors_hashes = nlp.meta.pop("_sourced_vectors_hashes", {}) vectors_hash = hash(nlp.vocab.vectors.to_bytes(exclude=["strings"])) @@ -191,7 +192,7 @@ def init_tok2vec( if weights_data is not None: layer = get_tok2vec_ref(nlp, P) layer.from_bytes(weights_data) - logger.info(f"Loaded pretrained weights from {init_tok2vec}") + logger.info("Loaded pretrained weights from %s", init_tok2vec) return True return False @@ -216,13 +217,13 @@ def convert_vectors( nlp.vocab.deduplicate_vectors() else: if vectors_loc: - logger.info(f"Reading vectors from {vectors_loc}") + logger.info("Reading vectors from %s", vectors_loc) vectors_data, vector_keys, floret_settings = read_vectors( vectors_loc, truncate, mode=mode, ) - logger.info(f"Loaded vectors from {vectors_loc}") + logger.info("Loaded vectors from %s", vectors_loc) else: vectors_data, vector_keys = (None, None) if vector_keys is not None and mode != VectorsMode.floret: diff --git a/spacy/training/loggers.py b/spacy/training/loggers.py index 408ea7140..7de31822e 100644 --- a/spacy/training/loggers.py +++ b/spacy/training/loggers.py @@ -26,6 +26,8 @@ def setup_table( return final_cols, final_widths, ["r" for _ in final_widths] +# We cannot rename this method as it's directly imported +# and used by external packages such as spacy-loggers. @registry.loggers("spacy.ConsoleLogger.v2") def console_logger( progress_bar: bool = False, @@ -33,7 +35,27 @@ def console_logger( output_file: Optional[Union[str, Path]] = None, ): """The ConsoleLogger.v2 prints out training logs in the console and/or saves them to a jsonl file. - progress_bar (bool): Whether the logger should print the progress bar. + progress_bar (bool): Whether the logger should print a progress bar tracking the steps till the next evaluation pass. + console_output (bool): Whether the logger should print the logs on the console. + output_file (Optional[Union[str, Path]]): The file to save the training logs to. + """ + return console_logger_v3( + progress_bar=None if progress_bar is False else "eval", + console_output=console_output, + output_file=output_file, + ) + + +@registry.loggers("spacy.ConsoleLogger.v3") +def console_logger_v3( + progress_bar: Optional[str] = None, + console_output: bool = True, + output_file: Optional[Union[str, Path]] = None, +): + """The ConsoleLogger.v3 prints out training logs in the console and/or saves them to a jsonl file. + progress_bar (Optional[str]): Type of progress bar to show in the console. Allowed values: + train - Tracks the number of steps from the beginning of training until the full training run is complete (training.max_steps is reached). + eval - Tracks the number of steps between the previous and next evaluation (training.eval_frequency is reached). console_output (bool): Whether the logger should print the logs on the console. output_file (Optional[Union[str, Path]]): The file to save the training logs to. """ @@ -70,6 +92,7 @@ def console_logger( for name, proc in nlp.pipeline if hasattr(proc, "is_trainable") and proc.is_trainable ] + max_steps = nlp.config["training"]["max_steps"] eval_frequency = nlp.config["training"]["eval_frequency"] score_weights = nlp.config["training"]["score_weights"] score_cols = [col for col, value in score_weights.items() if value is not None] @@ -84,6 +107,13 @@ def console_logger( write(msg.row(table_header, widths=table_widths, spacing=spacing)) write(msg.row(["-" * width for width in table_widths], spacing=spacing)) progress = None + expected_progress_types = ("train", "eval") + if progress_bar is not None and progress_bar not in expected_progress_types: + raise ValueError( + Errors.E1048.format( + unexpected=progress_bar, expected=expected_progress_types + ) + ) def log_step(info: Optional[Dict[str, Any]]) -> None: nonlocal progress @@ -141,11 +171,23 @@ def console_logger( ) ) if progress_bar: + if progress_bar == "train": + total = max_steps + desc = f"Last Eval Epoch: {info['epoch']}" + initial = info["step"] + else: + total = eval_frequency + desc = f"Epoch {info['epoch']+1}" + initial = 0 # Set disable=None, so that it disables on non-TTY progress = tqdm.tqdm( - total=eval_frequency, disable=None, leave=False, file=stderr + total=total, + disable=None, + leave=False, + file=stderr, + initial=initial, ) - progress.set_description(f"Epoch {info['epoch']+1}") + progress.set_description(desc) def finalize() -> None: if output_stream: diff --git a/spacy/training/loop.py b/spacy/training/loop.py index 885257772..eca40e3d9 100644 --- a/spacy/training/loop.py +++ b/spacy/training/loop.py @@ -370,6 +370,6 @@ def clean_output_dir(path: Optional[Path]) -> None: if subdir.exists(): try: shutil.rmtree(str(subdir)) - logger.debug(f"Removed existing output directory: {subdir}") + logger.debug("Removed existing output directory: %s", subdir) except Exception as e: raise IOError(Errors.E901.format(path=path)) from e diff --git a/spacy/training/pretrain.py b/spacy/training/pretrain.py index 52af84aaf..ebbc5d837 100644 --- a/spacy/training/pretrain.py +++ b/spacy/training/pretrain.py @@ -24,6 +24,7 @@ def pretrain( epoch_resume: Optional[int] = None, use_gpu: int = -1, silent: bool = True, + skip_last: bool = False, ): msg = Printer(no_print=silent) if config["training"]["seed"] is not None: @@ -60,10 +61,14 @@ def pretrain( row_settings = {"widths": (3, 10, 10, 6, 4), "aligns": ("r", "r", "r", "r", "r")} msg.row(("#", "# Words", "Total Loss", "Loss", "w/s"), **row_settings) - def _save_model(epoch, is_temp=False): + def _save_model(epoch, is_temp=False, is_last=False): is_temp_str = ".temp" if is_temp else "" with model.use_params(optimizer.averages): - with (output_dir / f"model{epoch}{is_temp_str}.bin").open("wb") as file_: + if is_last: + save_path = output_dir / f"model-last.bin" + else: + save_path = output_dir / f"model{epoch}{is_temp_str}.bin" + with (save_path).open("wb") as file_: file_.write(model.get_ref("tok2vec").to_bytes()) log = { "nr_word": tracker.nr_word, @@ -76,22 +81,26 @@ def pretrain( # TODO: I think we probably want this to look more like the # 'create_train_batches' function? - for epoch in range(epoch_resume, P["max_epochs"]): - for batch_id, batch in enumerate(batcher(corpus(nlp))): - docs = ensure_docs(batch) - loss = make_update(model, docs, optimizer, objective) - progress = tracker.update(epoch, loss, docs) - if progress: - msg.row(progress, **row_settings) - if P["n_save_every"] and (batch_id % P["n_save_every"] == 0): - _save_model(epoch, is_temp=True) + try: + for epoch in range(epoch_resume, P["max_epochs"]): + for batch_id, batch in enumerate(batcher(corpus(nlp))): + docs = ensure_docs(batch) + loss = make_update(model, docs, optimizer, objective) + progress = tracker.update(epoch, loss, docs) + if progress: + msg.row(progress, **row_settings) + if P["n_save_every"] and (batch_id % P["n_save_every"] == 0): + _save_model(epoch, is_temp=True) - if P["n_save_epoch"]: - if epoch % P["n_save_epoch"] == 0 or epoch == P["max_epochs"] - 1: + if P["n_save_epoch"]: + if epoch % P["n_save_epoch"] == 0 or epoch == P["max_epochs"] - 1: + _save_model(epoch) + else: _save_model(epoch) - else: - _save_model(epoch) - tracker.epoch_loss = 0.0 + tracker.epoch_loss = 0.0 + finally: + if not skip_last: + _save_model(P["max_epochs"], is_last=True) def ensure_docs(examples_or_docs: Iterable[Union[Doc, Example]]) -> List[Doc]: diff --git a/spacy/util.py b/spacy/util.py index 8d211a9a5..8cc89217d 100644 --- a/spacy/util.py +++ b/spacy/util.py @@ -31,6 +31,8 @@ import shlex import inspect import pkgutil import logging +import socket +import stat try: import cupy.random @@ -59,7 +61,7 @@ if TYPE_CHECKING: # fmt: off OOV_RANK = numpy.iinfo(numpy.uint64).max DEFAULT_OOV_PROB = -20 -LEXEME_NORM_LANGS = ["cs", "da", "de", "el", "en", "id", "lb", "mk", "pt", "ru", "sr", "ta", "th"] +LEXEME_NORM_LANGS = ["cs", "da", "de", "el", "en", "grc", "id", "lb", "mk", "pt", "ru", "sr", "ta", "th"] # Default order of sections in the config file. Not all sections needs to exist, # and additional sections are added at the end, in alphabetical order. @@ -143,8 +145,17 @@ class registry(thinc.registry): return func @classmethod - def find(cls, registry_name: str, func_name: str) -> Callable: - """Get info about a registered function from the registry.""" + def find( + cls, registry_name: str, func_name: str + ) -> Dict[str, Optional[Union[str, int]]]: + """Find information about a registered function, including the + module and path to the file it's defined in, the line number and the + docstring, if available. + + registry_name (str): Name of the catalogue registry. + func_name (str): Name of the registered function. + RETURNS (Dict[str, Optional[Union[str, int]]]): The function info. + """ # We're overwriting this classmethod so we're able to provide more # specific error messages and implement a fallback to spacy-legacy. if not hasattr(cls, registry_name): @@ -1040,8 +1051,15 @@ def make_tempdir() -> Generator[Path, None, None]: """ d = Path(tempfile.mkdtemp()) yield d + + # On Windows, git clones use read-only files, which cause permission errors + # when being deleted. This forcibly fixes permissions. + def force_remove(rmfunc, path, ex): + os.chmod(path, stat.S_IWRITE) + rmfunc(path) + try: - shutil.rmtree(str(d)) + shutil.rmtree(str(d), onerror=force_remove) except PermissionError as e: warnings.warn(Warnings.W091.format(dir=d, msg=e)) @@ -1736,3 +1754,50 @@ def all_equal(iterable): (or if the input is an empty sequence), False otherwise.""" g = itertools.groupby(iterable) return next(g, True) and not next(g, False) + + +def _is_port_in_use(port: int, host: str = "localhost") -> bool: + """Check if 'host:port' is in use. Return True if it is, False otherwise. + + port (int): the port to check + host (str): the host to check (default "localhost") + RETURNS (bool): Whether 'host:port' is in use. + """ + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + try: + s.bind((host, port)) + return False + except socket.error: + return True + finally: + s.close() + + +def find_available_port(start: int, host: str, auto_select: bool = False) -> int: + """Given a starting port and a host, handle finding a port. + + If `auto_select` is False, a busy port will raise an error. + + If `auto_select` is True, the next free higher port will be used. + + start (int): the port to start looking from + host (str): the host to find a port on + auto_select (bool): whether to automatically select a new port if the given port is busy (default False) + RETURNS (int): The port to use. + """ + if not _is_port_in_use(start, host): + return start + + port = start + if not auto_select: + raise ValueError(Errors.E1050.format(port=port)) + + while _is_port_in_use(port, host) and port < 65535: + port += 1 + + if port == 65535 and _is_port_in_use(port, host): + raise ValueError(Errors.E1049.format(host=host)) + + # if we get here, the port changed + warnings.warn(Warnings.W124.format(host=host, port=start, serve_port=port)) + return port diff --git a/website/.dockerignore b/website/.dockerignore new file mode 100644 index 000000000..e4a88552e --- /dev/null +++ b/website/.dockerignore @@ -0,0 +1,9 @@ +.cache/ +.next/ +public/ +node_modules +.npm +logs +*.log +npm-debug.log* +quickstart-training-generator.js diff --git a/website/.eslintrc.json b/website/.eslintrc.json new file mode 100644 index 000000000..1c2aa65d7 --- /dev/null +++ b/website/.eslintrc.json @@ -0,0 +1,3 @@ +{ + "extends": "next/core-web-vitals" +} diff --git a/website/.gitignore b/website/.gitignore new file mode 100644 index 000000000..599c0953a --- /dev/null +++ b/website/.gitignore @@ -0,0 +1,46 @@ +# See https://help.github.com/articles/ignoring-files/ for more about ignoring files. + +quickstart-training-generator.js + +# dependencies +/node_modules +/.pnp +.pnp.js + +# testing +/coverage + +# next.js +/.next/ +/out/ + +# production +/build + +# misc +.DS_Store +*.pem + +# debug +npm-debug.log* +yarn-debug.log* +yarn-error.log* +.pnpm-debug.log* + +# local env files +.env*.local + +# vercel +.vercel + +# typescript +*.tsbuildinfo +next-env.d.ts + +!.vscode/extensions.json +!public + +public/robots.txt +public/sitemap* +public/sw.js* +public/workbox* diff --git a/website/.nvmrc b/website/.nvmrc new file mode 100644 index 000000000..3c032078a --- /dev/null +++ b/website/.nvmrc @@ -0,0 +1 @@ +18 diff --git a/website/.prettierignore b/website/.prettierignore new file mode 100644 index 000000000..d0d878e40 --- /dev/null +++ b/website/.prettierignore @@ -0,0 +1 @@ +.next \ No newline at end of file diff --git a/website/.prettierrc b/website/.prettierrc index 7555c734a..03904b1c4 100644 --- a/website/.prettierrc +++ b/website/.prettierrc @@ -20,12 +20,11 @@ } }, { - "files": "*.md", + "files": ["package.json", "package-lock.json"], "options": { "tabWidth": 2, "printWidth": 80, - "proseWrap": "always", - "htmlWhitespaceSensitivity": "strict" + "proseWrap": "always" } }, { diff --git a/website/.vscode/extensions.json b/website/.vscode/extensions.json new file mode 100644 index 000000000..4b533827a --- /dev/null +++ b/website/.vscode/extensions.json @@ -0,0 +1,8 @@ +{ + "recommendations": [ + "dbaeumer.vscode-eslint", + "unifiedjs.vscode-mdx", + "esbenp.prettier-vscode", + "syler.sass-indented" + ] +} diff --git a/website/Dockerfile b/website/Dockerfile index f71733e55..9b2f6cac4 100644 --- a/website/Dockerfile +++ b/website/Dockerfile @@ -1,16 +1,14 @@ -FROM node:11.15.0 +FROM node:18 -WORKDIR /spacy-io - -RUN npm install -g gatsby-cli@2.7.4 - -COPY package.json . -COPY package-lock.json . - -RUN npm install +USER node # This is so the installed node_modules will be up one directory # from where a user mounts files, so that they don't accidentally mount # their own node_modules from a different build # https://nodejs.org/api/modules.html#modules_loading_from_node_modules_folders -WORKDIR /spacy-io/website/ +WORKDIR /home/node +COPY --chown=node package.json . +COPY --chown=node package-lock.json . +RUN npm install + +WORKDIR /home/node/website/ diff --git a/website/README.md b/website/README.md index 890a48ef9..a434efe9a 100644 --- a/website/README.md +++ b/website/README.md @@ -7,17 +7,16 @@ The styleguide for the spaCy website is available at ## Setup and installation -Before running the setup, make sure your versions of -[Node](https://nodejs.org/en/) and [npm](https://www.npmjs.com/) are up to date. -Node v10.15 or later is required. - ```bash # Clone the repository git clone https://github.com/explosion/spaCy cd spaCy/website -# Install Gatsby's command-line tool -npm install --global gatsby-cli +# Switch to the correct Node version +# +# If you don't have NVM and don't want to use it, you can manually switch to the Node version +# stated in /.nvmrc and skip this step +nvm use # Install the dependencies npm install @@ -36,52 +35,47 @@ file in the root defines the settings used in this codebase. ## Building & developing the site with Docker -Sometimes it's hard to get a local environment working due to rapid updates to -node dependencies, so it may be easier to use docker for building the docs. +While it shouldn't be necessary and is not recommended you can run this site in a Docker container. If you'd like to do this, **be sure you do _not_ include your local `node_modules` folder**, since there are some dependencies that need to be built for the image system. Rename it before using. -```bash -docker run -it \ - -v $(pwd):/spacy-io/website \ - -p 8000:8000 \ - ghcr.io/explosion/spacy-io \ - gatsby develop -H 0.0.0.0 -``` - -This will allow you to access the built website at http://0.0.0.0:8000/ in your -browser, and still edit code in your editor while having the site reflect those -changes. - -**Note**: If you're working on a Mac with an M1 processor, you might see -segfault errors from `qemu` if you use the default image. To fix this use the -`arm64` tagged image in the `docker run` command -(ghcr.io/explosion/spacy-io:arm64). - -### Building the Docker image - -If you'd like to build the image locally, you can do so like this: +First build the Docker image. This only needs to be done on the first run +or when changes are made to `Dockerfile` or the website dependencies: ```bash docker build -t spacy-io . ``` -This will take some time, so if you want to use the prebuilt image you'll save a -bit of time. +You can then build and run the website with: + +```bash +docker run -it \ + --rm \ + -v $(pwd):/home/node/website \ + -p 3000:3000 \ + spacy-io \ + npm run dev -- -H 0.0.0.0 +``` + +This will allow you to access the built website at http://0.0.0.0:3000/ in your +browser, and still edit code in your editor while having the site reflect those +changes. ## Project structure ```yaml ├── docs # the actual markdown content ├── meta # JSON-formatted site metadata +| ├── dynamicMeta.js # At build time generated meta data | ├── languages.json # supported languages and statistical models | ├── sidebars.json # sidebar navigations for different sections | ├── site.json # general site metadata | ├── type-annotations.json # Type annotations | └── universe.json # data for the spaCy universe section -├── public # compiled site +├── pages # Next router pages +├── public # static images and other assets ├── setup # Jinja setup ├── src # source | ├── components # React components @@ -96,9 +90,11 @@ bit of time. | | └── universe.js # layout templates for universe | └── widgets # non-reusable components with content, e.g. changelog ├── .eslintrc.json # ESLint config file +├── .nvmrc # NVM config file +| # (to support "nvm use" to switch to correct Node version) +| ├── .prettierrc # Prettier config file -├── gatsby-browser.js # browser-specific hooks for Gatsby -├── gatsby-config.js # Gatsby configuration -├── gatsby-node.js # Node-specific hooks for Gatsby -└── package.json # package settings and dependencies +├── next.config.mjs # Next config file +├── package.json # package settings and dependencies +└── tsconfig.json # TypeScript config file ``` diff --git a/website/UNIVERSE.md b/website/UNIVERSE.md index 770bbde13..ac4e2e684 100644 --- a/website/UNIVERSE.md +++ b/website/UNIVERSE.md @@ -2,42 +2,52 @@ # spaCy Universe -The [spaCy Universe](https://spacy.io/universe) collects the many great resources developed with or for spaCy. It -includes standalone packages, plugins, extensions, educational materials, -operational utilities and bindings for other languages. +The [spaCy Universe](https://spacy.io/universe) collects the many great +resources developed with or for spaCy. It includes standalone packages, plugins, +extensions, educational materials, operational utilities and bindings for other +languages. If you have a project that you want the spaCy community to make use of, you can suggest it by submitting a pull request to this repository. The Universe database is open-source and collected in a simple JSON file. Looking for inspiration for your own spaCy plugin or extension? Check out the -[`project ideas`](https://github.com/explosion/spaCy/discussions?discussions_q=category%3A%22New+Features+%26+Project+Ideas%22) +[`project ideas`](https://github.com/explosion/spaCy/discussions?discussions_q=category%3A%22New+Features+%26+Project+Ideas%22) discussion forum. ## Checklist ### Projects -✅ Libraries and packages should be **open-source** (with a user-friendly license) and at least somewhat **documented** (e.g. a simple `README` with usage instructions). +✅ Libraries and packages should be **open-source** (with a user-friendly +license) and at least somewhat **documented** (e.g. a simple `README` with usage +instructions). -✅ We're happy to include work in progress and prereleases, but we'd like to keep the emphasis on projects that should be useful to the community **right away**. +✅ We're happy to include work in progress and prereleases, but we'd like to +keep the emphasis on projects that should be useful to the community **right +away**. ✅ Demos and visualizers should be available via a **public URL**. ### Educational Materials -✅ Books should be **available for purchase or download** (not just pre-order). Ebooks and self-published books are fine, too, if they include enough substantial content. +✅ Books should be **available for purchase or download** (not just pre-order). +Ebooks and self-published books are fine, too, if they include enough +substantial content. -✅ The `"url"` of book entries should either point to the publisher's website or a reseller of your choice (ideally one that ships worldwide or as close as possible). +✅ The `"url"` of book entries should either point to the publisher's website or +a reseller of your choice (ideally one that ships worldwide or as close as +possible). -✅ If an online course is only available behind a paywall, it should at least have a **free excerpt** or chapter available, so users know what to expect. +✅ If an online course is only available behind a paywall, it should at least +have a **free excerpt** or chapter available, so users know what to expect. ## JSON format -To add a project, fork this repository, edit the [`universe.json`](meta/universe.json) -and add an object of the following format to the list of `"resources"`. Before -you submit your pull request, make sure to use a linter to verify that your -markup is correct. +To add a project, fork this repository, edit the +[`universe.json`](meta/universe.json) and add an object of the following format +to the list of `"resources"`. Before you submit your pull request, make sure to +use a linter to verify that your markup is correct. ```json { @@ -69,26 +79,26 @@ markup is correct. } ``` -| Field | Type | Description | -| --- | --- | --- | -| `id` | string | Unique ID of the project. | -| `title` | string | Project title. If not set, the `id` will be used as the display title. | -| `slogan` | string | A short description of the project. Displayed in the overview and under the title. | -| `description` | string | A longer description of the project. Markdown is allowed, but should be limited to basic formatting like bold, italics, code or links. | -| `github` | string | Associated GitHub repo in the format `user/repo`. Will be displayed as a link and used for release, license and star badges. | -| `pip` | string | Package name on pip. If available, the installation command will be displayed. | -| `cran` | string | For R packages: package name on CRAN. If available, the installation command will be displayed. | -| `code_example` | array | Short example that shows how to use the project. Formatted as an array with one string per line. | -| `code_language` | string | Defaults to `'python'`. Optional code language used for syntax highlighting with [Prism](http://prismjs.com/). | -| `url` | string | Optional project link to display as button. | -| `thumb` | string | Optional URL to project thumbnail to display in overview and project header. Recommended size is 100x100px. | -| `image` | string | Optional URL to project image to display with description. | -| `author` | string | Name(s) of project author(s). | -| `author_links` | object | Usernames and links to display as icons to author info. Currently supports `twitter` and `github` usernames, as well as `website` link. | -| `category` | list | One or more categories to assign to project. Must be one of the available options. | -| `tags` | list | Still experimental and not used for filtering: one or more tags to assign to project. | +| Field | Type | Description | +| --------------- | ------ | --------------------------------------------------------------------------------------------------------------------------------------- | +| `id` | string | Unique ID of the project. | +| `title` | string | Project title. If not set, the `id` will be used as the display title. | +| `slogan` | string | A short description of the project. Displayed in the overview and under the title. | +| `description` | string | A longer description of the project. Markdown is allowed, but should be limited to basic formatting like bold, italics, code or links. | +| `github` | string | Associated GitHub repo in the format `user/repo`. Will be displayed as a link and used for release, license and star badges. | +| `pip` | string | Package name on pip. If available, the installation command will be displayed. | +| `cran` | string | For R packages: package name on CRAN. If available, the installation command will be displayed. | +| `code_example` | array | Short example that shows how to use the project. Formatted as an array with one string per line. | +| `code_language` | string | Defaults to `'python'`. Optional code language used for syntax highlighting with [Prism](http://prismjs.com/). | +| `url` | string | Optional project link to display as button. | +| `thumb` | string | Optional URL to project thumbnail to display in overview and project header. Recommended size is 100x100px. | +| `image` | string | Optional URL to project image to display with description. | +| `author` | string | Name(s) of project author(s). | +| `author_links` | object | Usernames and links to display as icons to author info. Currently supports `twitter` and `github` usernames, as well as `website` link. | +| `category` | list | One or more categories to assign to project. Must be one of the available options. | +| `tags` | list | Still experimental and not used for filtering: one or more tags to assign to project. | To separate them from the projects, educational materials also specify -`"type": "education`. Books can also set a `"cover"` field containing a URL -to a cover image. If available, it's used in the overview and displayed on -the individual book page. +`"type": "education`. Books can also set a `"cover"` field containing a URL to a +cover image. If available, it's used in the overview and displayed on the +individual book page. diff --git a/website/docs/api/architectures.md b/website/docs/api/architectures.mdx similarity index 95% rename from website/docs/api/architectures.md rename to website/docs/api/architectures.mdx index 4c5447f75..268c04a07 100644 --- a/website/docs/api/architectures.md +++ b/website/docs/api/architectures.mdx @@ -26,9 +26,9 @@ part of the [training config](/usage/training#custom-functions). Also see the usage documentation on [layers and model architectures](/usage/layers-architectures). -## Tok2Vec architectures {#tok2vec-arch source="spacy/ml/models/tok2vec.py"} +## Tok2Vec architectures {id="tok2vec-arch",source="spacy/ml/models/tok2vec.py"} -### spacy.Tok2Vec.v2 {#Tok2Vec} +### spacy.Tok2Vec.v2 {id="Tok2Vec"} > #### Example config > @@ -56,7 +56,7 @@ blog post for background. | `encode` | Encode context into the embeddings, using an architecture such as a CNN, BiLSTM or transformer. For example, [MaxoutWindowEncoder](/api/architectures#MaxoutWindowEncoder). ~~Model[List[Floats2d], List[Floats2d]]~~ | | **CREATES** | The model using the architecture. ~~Model[List[Doc], List[Floats2d]]~~ | -### spacy.HashEmbedCNN.v2 {#HashEmbedCNN} +### spacy.HashEmbedCNN.v2 {id="HashEmbedCNN"} > #### Example Config > @@ -89,7 +89,7 @@ consisting of a CNN and a layer-normalized maxout activation function. | `pretrained_vectors` | Whether to also use static vectors. ~~bool~~ | | **CREATES** | The model using the architecture. ~~Model[List[Doc], List[Floats2d]]~~ | -### spacy.Tok2VecListener.v1 {#Tok2VecListener} +### spacy.Tok2VecListener.v1 {id="Tok2VecListener"} > #### Example config > @@ -139,7 +139,7 @@ the `Tok2Vec` component. | `upstream` | A string to identify the "upstream" `Tok2Vec` component to communicate with. By default, the upstream name is the wildcard string `"*"`, but you could also specify the name of the `Tok2Vec` component. You'll almost never have multiple upstream `Tok2Vec` components, so the wildcard string will almost always be fine. ~~str~~ | | **CREATES** | The model using the architecture. ~~Model[List[Doc], List[Floats2d]]~~ | -### spacy.MultiHashEmbed.v2 {#MultiHashEmbed} +### spacy.MultiHashEmbed.v2 {id="MultiHashEmbed"} > #### Example config > @@ -170,7 +170,7 @@ updated). | `include_static_vectors` | Whether to also use static word vectors. Requires a vectors table to be loaded in the [`Doc`](/api/doc) objects' vocab. ~~bool~~ | | **CREATES** | The model using the architecture. ~~Model[List[Doc], List[Floats2d]]~~ | -### spacy.CharacterEmbed.v2 {#CharacterEmbed} +### spacy.CharacterEmbed.v2 {id="CharacterEmbed"} > #### Example config > @@ -207,7 +207,7 @@ network to construct a single vector to represent the information. | `nC` | The number of UTF-8 bytes to embed per word. Recommended values are between `3` and `8`, although it may depend on the length of words in the language. ~~int~~ | | **CREATES** | The model using the architecture. ~~Model[List[Doc], List[Floats2d]]~~ | -### spacy.MaxoutWindowEncoder.v2 {#MaxoutWindowEncoder} +### spacy.MaxoutWindowEncoder.v2 {id="MaxoutWindowEncoder"} > #### Example config > @@ -231,7 +231,7 @@ and residual connections. | `depth` | The number of convolutional layers. Recommended value is `4`. ~~int~~ | | **CREATES** | The model using the architecture. ~~Model[List[Floats2d], List[Floats2d]]~~ | -### spacy.MishWindowEncoder.v2 {#MishWindowEncoder} +### spacy.MishWindowEncoder.v2 {id="MishWindowEncoder"} > #### Example config > @@ -254,7 +254,7 @@ and residual connections. | `depth` | The number of convolutional layers. Recommended value is `4`. ~~int~~ | | **CREATES** | The model using the architecture. ~~Model[List[Floats2d], List[Floats2d]]~~ | -### spacy.TorchBiLSTMEncoder.v1 {#TorchBiLSTMEncoder} +### spacy.TorchBiLSTMEncoder.v1 {id="TorchBiLSTMEncoder"} > #### Example config > @@ -276,7 +276,7 @@ Encode context using bidirectional LSTM layers. Requires | `dropout` | Creates a Dropout layer on the outputs of each LSTM layer except the last layer. Set to 0.0 to disable this functionality. ~~float~~ | | **CREATES** | The model using the architecture. ~~Model[List[Floats2d], List[Floats2d]]~~ | -### spacy.StaticVectors.v2 {#StaticVectors} +### spacy.StaticVectors.v2 {id="StaticVectors"} > #### Example config > @@ -306,7 +306,7 @@ mapped to a zero vector. See the documentation on | `key_attr` | Defaults to `"ORTH"`. ~~str~~ | | **CREATES** | The model using the architecture. ~~Model[List[Doc], Ragged]~~ | -### spacy.FeatureExtractor.v1 {#FeatureExtractor} +### spacy.FeatureExtractor.v1 {id="FeatureExtractor"} > #### Example config > @@ -324,7 +324,7 @@ of feature names to extract, which should refer to token attributes. | `columns` | The token attributes to extract. ~~List[Union[int, str]]~~ | | **CREATES** | The created feature extraction layer. ~~Model[List[Doc], List[Ints2d]]~~ | -## Transformer architectures {#transformers source="github.com/explosion/spacy-transformers/blob/master/spacy_transformers/architectures.py"} +## Transformer architectures {id="transformers",source="github.com/explosion/spacy-transformers/blob/master/spacy_transformers/architectures.py"} The following architectures are provided by the package [`spacy-transformers`](https://github.com/explosion/spacy-transformers). See the @@ -341,7 +341,7 @@ for details and system requirements. -### spacy-transformers.TransformerModel.v3 {#TransformerModel} +### spacy-transformers.TransformerModel.v3 {id="TransformerModel"} > #### Example Config > @@ -390,7 +390,7 @@ in other components, see | | | -Mixed-precision support is currently an experimental feature. + Mixed-precision support is currently an experimental feature. @@ -404,7 +404,7 @@ The other arguments are shared between all versions. -### spacy-transformers.TransformerListener.v1 {#TransformerListener} +### spacy-transformers.TransformerListener.v1 {id="TransformerListener"} > #### Example Config > @@ -434,7 +434,7 @@ a single token vector given zero or more wordpiece vectors. | `upstream` | A string to identify the "upstream" `Transformer` component to communicate with. By default, the upstream name is the wildcard string `"*"`, but you could also specify the name of the `Transformer` component. You'll almost never have multiple upstream `Transformer` components, so the wildcard string will almost always be fine. ~~str~~ | | **CREATES** | The model using the architecture. ~~Model[List[Doc], List[Floats2d]]~~ | -### spacy-transformers.Tok2VecTransformer.v3 {#Tok2VecTransformer} +### spacy-transformers.Tok2VecTransformer.v3 {id="Tok2VecTransformer"} > #### Example Config > @@ -467,7 +467,7 @@ one component. | **CREATES** | The model using the architecture. ~~Model[List[Doc], List[Floats2d]]~~ | -Mixed-precision support is currently an experimental feature. + Mixed-precision support is currently an experimental feature. @@ -481,7 +481,7 @@ The other arguments are shared between all versions. -## Pretraining architectures {#pretrain source="spacy/ml/models/multi_task.py"} +## Pretraining architectures {id="pretrain",source="spacy/ml/models/multi_task.py"} The spacy `pretrain` command lets you initialize a `Tok2Vec` layer in your pipeline with information from raw text. To this end, additional layers are @@ -494,7 +494,7 @@ BERT. For more information, see the section on [pretraining](/usage/embeddings-transformers#pretraining). -### spacy.PretrainVectors.v1 {#pretrain_vectors} +### spacy.PretrainVectors.v1 {id="pretrain_vectors"} > #### Example config > @@ -525,7 +525,7 @@ vectors. | `loss` | The loss function can be either "cosine" or "L2". We typically recommend to use "cosine". ~~~str~~ | | **CREATES** | A callable function that can create the Model, given the `vocab` of the pipeline and the `tok2vec` layer to pretrain. ~~Callable[[Vocab, Model], Model]~~ | -### spacy.PretrainCharacters.v1 {#pretrain_chars} +### spacy.PretrainCharacters.v1 {id="pretrain_chars"} > #### Example config > @@ -551,9 +551,9 @@ for a Tok2Vec layer. | `n_characters` | The window of characters - e.g. if `n_characters = 2`, the model will try to predict the first two and last two characters of the word. ~~int~~ | | **CREATES** | A callable function that can create the Model, given the `vocab` of the pipeline and the `tok2vec` layer to pretrain. ~~Callable[[Vocab, Model], Model]~~ | -## Parser & NER architectures {#parser} +## Parser & NER architectures {id="parser"} -### spacy.TransitionBasedParser.v2 {#TransitionBasedParser source="spacy/ml/models/parser.py"} +### spacy.TransitionBasedParser.v2 {id="TransitionBasedParser",source="spacy/ml/models/parser.py"} > #### Example Config > @@ -612,9 +612,9 @@ same signature, but the `use_upper` argument was `True` by default. -## Tagging architectures {#tagger source="spacy/ml/models/tagger.py"} +## Tagging architectures {id="tagger",source="spacy/ml/models/tagger.py"} -### spacy.Tagger.v2 {#Tagger} +### spacy.Tagger.v2 {id="Tagger"} > #### Example Config > @@ -648,7 +648,7 @@ The other arguments are shared between all versions. -## Text classification architectures {#textcat source="spacy/ml/models/textcat.py"} +## Text classification architectures {id="textcat",source="spacy/ml/models/textcat.py"} A text classification architecture needs to take a [`Doc`](/api/doc) as input, and produce a score for each potential label class. Textcat challenges can be @@ -672,7 +672,7 @@ single-label use-cases where `exclusive_classes = true`, while the -### spacy.TextCatEnsemble.v2 {#TextCatEnsemble} +### spacy.TextCatEnsemble.v2 {id="TextCatEnsemble"} > #### Example Config > @@ -737,7 +737,7 @@ but used an internal `tok2vec` instead of taking it as argument: -### spacy.TextCatCNN.v2 {#TextCatCNN} +### spacy.TextCatCNN.v2 {id="TextCatCNN"} > #### Example Config > @@ -777,7 +777,7 @@ after training. -### spacy.TextCatBOW.v2 {#TextCatBOW} +### spacy.TextCatBOW.v2 {id="TextCatBOW"} > #### Example Config > @@ -809,9 +809,9 @@ after training. -## Span classification architectures {#spancat source="spacy/ml/models/spancat.py"} +## Span classification architectures {id="spancat",source="spacy/ml/models/spancat.py"} -### spacy.SpanCategorizer.v1 {#SpanCategorizer} +### spacy.SpanCategorizer.v1 {id="SpanCategorizer"} > #### Example Config > @@ -848,7 +848,7 @@ single vector, and a scorer model to map the vectors to probabilities. | `scorer` | The scorer model. ~~Model[Floats2d, Floats2d]~~ | | **CREATES** | The model using the architecture. ~~Model[Tuple[List[Doc], Ragged], Floats2d]~~ | -### spacy.mean_max_reducer.v1 {#mean_max_reducer} +### spacy.mean_max_reducer.v1 {id="mean_max_reducer"} Reduce sequences by concatenating their mean and max pooled vectors, and then combine the concatenated vectors with a hidden layer. @@ -857,7 +857,7 @@ combine the concatenated vectors with a hidden layer. | ------------- | ------------------------------------- | | `hidden_size` | The size of the hidden layer. ~~int~~ | -## Entity linking architectures {#entitylinker source="spacy/ml/models/entity_linker.py"} +## Entity linking architectures {id="entitylinker",source="spacy/ml/models/entity_linker.py"} An [`EntityLinker`](/api/entitylinker) component disambiguates textual mentions (tagged as named entities) to unique identifiers, grounding the named entities @@ -870,7 +870,7 @@ into the "real world". This requires 3 main components: - A machine learning [`Model`](https://thinc.ai/docs/api-model) that picks the most plausible ID from the set of candidates. -### spacy.EntityLinker.v2 {#EntityLinker} +### spacy.EntityLinker.v2 {id="EntityLinker"} > #### Example Config > @@ -899,16 +899,22 @@ The `EntityLinker` model architecture is a Thinc `Model` with a | `nO` | Output dimension, determined by the length of the vectors encoding each entity in the KB. If the `nO` dimension is not set, the entity linking component will set it when `initialize` is called. ~~Optional[int]~~ | | **CREATES** | The model using the architecture. ~~Model[List[Doc], Floats2d]~~ | -### spacy.EmptyKB.v1 {#EmptyKB} +### spacy.EmptyKB.v1 {id="EmptyKB.v1"} A function that creates an empty `KnowledgeBase` from a [`Vocab`](/api/vocab) -instance. This is the default when a new entity linker component is created. +instance. | Name | Description | | ---------------------- | ----------------------------------------------------------------------------------- | | `entity_vector_length` | The length of the vectors encoding each entity in the KB. Defaults to `64`. ~~int~~ | -### spacy.KBFromFile.v1 {#KBFromFile} +### spacy.EmptyKB.v2 {id="EmptyKB"} + +A function that creates an empty `KnowledgeBase` from a [`Vocab`](/api/vocab) +instance. This is the default when a new entity linker component is created. It +returns a `Callable[[Vocab, int], InMemoryLookupKB]`. + +### spacy.KBFromFile.v1 {id="KBFromFile"} A function that reads an existing `KnowledgeBase` from file. @@ -916,7 +922,7 @@ A function that reads an existing `KnowledgeBase` from file. | --------- | -------------------------------------------------------- | | `kb_path` | The location of the KB that was stored to file. ~~Path~~ | -### spacy.CandidateGenerator.v1 {#CandidateGenerator} +### spacy.CandidateGenerator.v1 {id="CandidateGenerator"} A function that takes as input a [`KnowledgeBase`](/api/kb) and a [`Span`](/api/span) object denoting a named entity, and returns a list of @@ -924,7 +930,16 @@ plausible [`Candidate`](/api/kb/#candidate) objects. The default `CandidateGenerator` uses the text of a mention to find its potential aliases in the `KnowledgeBase`. Note that this function is case-dependent. -## Coreference {#coref-architectures tag="experimental"} +### spacy.CandidateBatchGenerator.v1 {id="CandidateBatchGenerator"} + +A function that takes as input a [`KnowledgeBase`](/api/kb) and an `Iterable` of +[`Span`](/api/span) objects denoting named entities, and returns a list of +plausible [`Candidate`](/api/kb/#candidate) objects per specified +[`Span`](/api/span). The default `CandidateBatchGenerator` uses the text of a +mention to find its potential aliases in the `KnowledgeBase`. Note that this +function is case-dependent. + +## Coreference {id="coref-architectures",tag="experimental"} A [`CoreferenceResolver`](/api/coref) component identifies tokens that refer to the same entity. A [`SpanResolver`](/api/span-resolver) component infers spans @@ -932,7 +947,7 @@ from single tokens. Together these components can be used to reproduce traditional coreference models. You can also omit the `SpanResolver` if working with only token-level clusters is acceptable. -### spacy-experimental.Coref.v1 {#Coref tag="experimental"} +### spacy-experimental.Coref.v1 {id="Coref",tag="experimental"} > #### Example Config > @@ -967,7 +982,7 @@ The `Coref` model architecture is a Thinc `Model`. | `antecedent_batch_size` | Internal batch size. ~~int~~ | | **CREATES** | The model using the architecture. ~~Model[List[Doc], Floats2d]~~ | -### spacy-experimental.SpanResolver.v1 {#SpanResolver tag="experimental"} +### spacy-experimental.SpanResolver.v1 {id="SpanResolver",tag="experimental"} > #### Example Config > diff --git a/website/docs/api/attributeruler.md b/website/docs/api/attributeruler.mdx similarity index 94% rename from website/docs/api/attributeruler.md rename to website/docs/api/attributeruler.mdx index 965bffbcc..c18319187 100644 --- a/website/docs/api/attributeruler.md +++ b/website/docs/api/attributeruler.mdx @@ -2,7 +2,7 @@ title: AttributeRuler tag: class source: spacy/pipeline/attributeruler.py -new: 3 +version: 3 teaser: 'Pipeline component for rule-based token attribute assignment' api_string_name: attribute_ruler api_trainable: false @@ -15,7 +15,7 @@ between attributes such as mapping fine-grained POS tags to coarse-grained POS tags. See the [usage guide](/usage/linguistic-features/#mappings-exceptions) for examples. -## Config and implementation {#config} +## Config and implementation {id="config"} The default config is defined by the pipeline component factory and describes how the component should be configured. You can override its settings via the @@ -37,7 +37,7 @@ how the component should be configured. You can override its settings via the %%GITHUB_SPACY/spacy/pipeline/attributeruler.py ``` -## AttributeRuler.\_\_init\_\_ {#init tag="method"} +## AttributeRuler.\_\_init\_\_ {id="init",tag="method"} Initialize the attribute ruler. @@ -56,7 +56,7 @@ Initialize the attribute ruler. | `validate` | Whether patterns should be validated (passed to the [`Matcher`](/api/matcher#init)). Defaults to `False`. ~~bool~~ | | `scorer` | The scoring method. Defaults to [`Scorer.score_token_attr`](/api/scorer#score_token_attr) for the attributes `"tag`", `"pos"`, `"morph"` and `"lemma"` and [`Scorer.score_token_attr_per_feat`](/api/scorer#score_token_attr_per_feat) for the attribute `"morph"`. ~~Optional[Callable]~~ | -## AttributeRuler.\_\_call\_\_ {#call tag="method"} +## AttributeRuler.\_\_call\_\_ {id="call",tag="method"} Apply the attribute ruler to a `Doc`, setting token attributes for tokens matched by the provided patterns. @@ -66,7 +66,7 @@ matched by the provided patterns. | `doc` | The document to process. ~~Doc~~ | | **RETURNS** | The processed document. ~~Doc~~ | -## AttributeRuler.add {#add tag="method"} +## AttributeRuler.add {id="add",tag="method"} Add patterns to the attribute ruler. The patterns are a list of `Matcher` patterns and the attributes are a dict of attributes to set on the matched @@ -89,7 +89,7 @@ may be negative to index from the end of the span. | `attrs` | The attributes to assign to the target token in the matched span. ~~Dict[str, Any]~~ | | `index` | The index of the token in the matched span to modify. May be negative to index from the end of the span. Defaults to `0`. ~~int~~ | -## AttributeRuler.add_patterns {#add_patterns tag="method"} +## AttributeRuler.add_patterns {id="add_patterns",tag="method"} > #### Example > @@ -116,7 +116,7 @@ keys `"patterns"`, `"attrs"` and `"index"`, which match the arguments of | ---------- | -------------------------------------------------------------------------- | | `patterns` | The patterns to add. ~~Iterable[Dict[str, Union[List[dict], dict, int]]]~~ | -## AttributeRuler.patterns {#patterns tag="property"} +## AttributeRuler.patterns {id="patterns",tag="property"} Get all patterns that have been added to the attribute ruler in the `patterns_dict` format accepted by @@ -126,7 +126,7 @@ Get all patterns that have been added to the attribute ruler in the | ----------- | -------------------------------------------------------------------------------------------- | | **RETURNS** | The patterns added to the attribute ruler. ~~List[Dict[str, Union[List[dict], dict, int]]]~~ | -## AttributeRuler.initialize {#initialize tag="method"} +## AttributeRuler.initialize {id="initialize",tag="method"} Initialize the component with data and used before training to load in rules from a file. This method is typically called by @@ -160,7 +160,7 @@ config. | `tag_map` | The tag map that maps fine-grained tags to coarse-grained tags and morphological features. Defaults to `None`. ~~Optional[Dict[str, Dict[Union[int, str], Union[int, str]]]]~~ | | `morph_rules` | The morph rules that map token text and fine-grained tags to coarse-grained tags, lemmas and morphological features. Defaults to `None`. ~~Optional[Dict[str, Dict[str, Dict[Union[int, str], Union[int, str]]]]]~~ | -## AttributeRuler.load_from_tag_map {#load_from_tag_map tag="method"} +## AttributeRuler.load_from_tag_map {id="load_from_tag_map",tag="method"} Load attribute ruler patterns from a tag map. @@ -168,7 +168,7 @@ Load attribute ruler patterns from a tag map. | --------- | ------------------------------------------------------------------------------------------------------------------------------------------------ | | `tag_map` | The tag map that maps fine-grained tags to coarse-grained tags and morphological features. ~~Dict[str, Dict[Union[int, str], Union[int, str]]]~~ | -## AttributeRuler.load_from_morph_rules {#load_from_morph_rules tag="method"} +## AttributeRuler.load_from_morph_rules {id="load_from_morph_rules",tag="method"} Load attribute ruler patterns from morph rules. @@ -176,7 +176,7 @@ Load attribute ruler patterns from morph rules. | ------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | `morph_rules` | The morph rules that map token text and fine-grained tags to coarse-grained tags, lemmas and morphological features. ~~Dict[str, Dict[str, Dict[Union[int, str], Union[int, str]]]]~~ | -## AttributeRuler.to_disk {#to_disk tag="method"} +## AttributeRuler.to_disk {id="to_disk",tag="method"} Serialize the pipe to disk. @@ -193,7 +193,7 @@ Serialize the pipe to disk. | _keyword-only_ | | | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | -## AttributeRuler.from_disk {#from_disk tag="method"} +## AttributeRuler.from_disk {id="from_disk",tag="method"} Load the pipe from disk. Modifies the object in place and returns it. @@ -211,7 +211,7 @@ Load the pipe from disk. Modifies the object in place and returns it. | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | | **RETURNS** | The modified `AttributeRuler` object. ~~AttributeRuler~~ | -## AttributeRuler.to_bytes {#to_bytes tag="method"} +## AttributeRuler.to_bytes {id="to_bytes",tag="method"} > #### Example > @@ -228,7 +228,7 @@ Serialize the pipe to a bytestring. | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | | **RETURNS** | The serialized form of the `AttributeRuler` object. ~~bytes~~ | -## AttributeRuler.from_bytes {#from_bytes tag="method"} +## AttributeRuler.from_bytes {id="from_bytes",tag="method"} Load the pipe from a bytestring. Modifies the object in place and returns it. @@ -247,7 +247,7 @@ Load the pipe from a bytestring. Modifies the object in place and returns it. | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | | **RETURNS** | The `AttributeRuler` object. ~~AttributeRuler~~ | -## Serialization fields {#serialization-fields} +## Serialization fields {id="serialization-fields"} During serialization, spaCy will export several data fields used to restore different aspects of the object. If needed, you can exclude them from diff --git a/website/docs/api/attributes.md b/website/docs/api/attributes.mdx similarity index 98% rename from website/docs/api/attributes.md rename to website/docs/api/attributes.mdx index adacd3898..3142b741d 100644 --- a/website/docs/api/attributes.md +++ b/website/docs/api/attributes.mdx @@ -41,10 +41,9 @@ from string attribute names to internal attribute IDs is stored in The corresponding [`Token` object attributes](/api/token#attributes) can be accessed using the same names in lowercase, e.g. `token.orth` or `token.length`. -For attributes that represent string values, the internal integer ID is -accessed as `Token.attr`, e.g. `token.dep`, while the string value can be -retrieved by appending `_` as in `token.dep_`. - +For attributes that represent string values, the internal integer ID is accessed +as `Token.attr`, e.g. `token.dep`, while the string value can be retrieved by +appending `_` as in `token.dep_`. | Attribute | Description | | ------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------- | diff --git a/website/docs/api/cli.md b/website/docs/api/cli.mdx similarity index 87% rename from website/docs/api/cli.md rename to website/docs/api/cli.mdx index 8823a3bd8..323ea2a92 100644 --- a/website/docs/api/cli.md +++ b/website/docs/api/cli.mdx @@ -12,6 +12,8 @@ menu: - ['train', 'train'] - ['pretrain', 'pretrain'] - ['evaluate', 'evaluate'] + - ['benchmark', 'benchmark'] + - ['apply', 'apply'] - ['find-threshold', 'find-threshold'] - ['assemble', 'assemble'] - ['package', 'package'] @@ -25,7 +27,7 @@ a list of available commands, you can type `python -m spacy --help`. You can also add the `--help` flag to any command or subcommand to see the description, available arguments and usage. -## download {#download tag="command"} +## download {id="download",tag="command"} Download [trained pipelines](/usage/models) for spaCy. The downloader finds the best-matching compatible version and uses `pip install` to download the Python @@ -43,7 +45,7 @@ pipeline name to be specified with its version (e.g. `en_core_web_sm-3.0.0`). > will also allow you to add it as a versioned package dependency to your > project. -```cli +```bash $ python -m spacy download [model] [--direct] [--sdist] [pip_args] ``` @@ -56,24 +58,24 @@ $ python -m spacy download [model] [--direct] [--sdist] [pip_args] | pip args | Additional installation options to be passed to `pip install` when installing the pipeline package. For example, `--user` to install to the user home directory or `--no-deps` to not install package dependencies. ~~Any (option/flag)~~ | | **CREATES** | The installed pipeline package in your `site-packages` directory. | -## info {#info tag="command"} +## info {id="info",tag="command"} Print information about your spaCy installation, trained pipelines and local setup, and generate [Markdown](https://en.wikipedia.org/wiki/Markdown)-formatted markup to copy-paste into [GitHub issues](https://github.com/explosion/spaCy/issues). -```cli +```bash $ python -m spacy info [--markdown] [--silent] [--exclude] ``` > #### Example > -> ```cli +> ```bash > $ python -m spacy info en_core_web_lg --markdown > ``` -```cli +```bash $ python -m spacy info [model] [--markdown] [--silent] [--exclude] ``` @@ -87,7 +89,7 @@ $ python -m spacy info [model] [--markdown] [--silent] [--exclude] | `--help`, `-h` | Show help message and available arguments. ~~bool (flag)~~ | | **PRINTS** | Information about your spaCy installation. | -## validate {#validate new="2" tag="command"} +## validate {id="validate",version="2",tag="command"} Find all trained pipeline packages installed in the current environment and check whether they are compatible with the currently installed version of spaCy. @@ -102,7 +104,7 @@ compatible versions and command for updating are shown. > suite, to ensure all packages are up to date before proceeding. If > incompatible packages are found, it will return `1`. -```cli +```bash $ python -m spacy validate ``` @@ -110,12 +112,12 @@ $ python -m spacy validate | ---------- | -------------------------------------------------------------------- | | **PRINTS** | Details about the compatibility of your installed pipeline packages. | -## init {#init new="3"} +## init {id="init",version="3"} The `spacy init` CLI includes helpful commands for initializing training config files and pipeline directories. -### init config {#init-config new="3" tag="command"} +### init config {id="init-config",version="3",tag="command"} Initialize and save a [`config.cfg` file](/usage/training#config) using the **recommended settings** for your use case. It works just like the @@ -127,11 +129,11 @@ customize those settings in your config file later. > #### Example > -> ```cli +> ```bash > $ python -m spacy init config config.cfg --lang en --pipeline ner,textcat --optimize accuracy > ``` -```cli +```bash $ python -m spacy init config [output_file] [--lang] [--pipeline] [--optimize] [--gpu] [--pretraining] [--force] ``` @@ -147,7 +149,7 @@ $ python -m spacy init config [output_file] [--lang] [--pipeline] [--optimize] [ | `--help`, `-h` | Show help message and available arguments. ~~bool (flag)~~ | | **CREATES** | The config file for training. | -### init fill-config {#init-fill-config new="3"} +### init fill-config {id="init-fill-config",version="3"} Auto-fill a partial [.cfg file](/usage/training#config) with **all default values**, e.g. a config generated with the @@ -161,15 +163,15 @@ validation error with more details. > #### Example > -> ```cli +> ```bash > $ python -m spacy init fill-config base.cfg config.cfg --diff > ``` > > #### Example diff > -> ![Screenshot of visual diff in terminal](../images/cli_init_fill-config_diff.jpg) +> ![Screenshot of visual diff in terminal](/images/cli_init_fill-config_diff.jpg) -```cli +```bash $ python -m spacy init fill-config [base_path] [output_file] [--diff] ``` @@ -183,7 +185,7 @@ $ python -m spacy init fill-config [base_path] [output_file] [--diff] | `--help`, `-h` | Show help message and available arguments. ~~bool (flag)~~ | | **CREATES** | Complete and auto-filled config file for training. | -### init vectors {#init-vectors new="3" tag="command"} +### init vectors {id="init-vectors",version="3",tag="command"} Convert [word vectors](/usage/linguistic-features#vectors-similarity) for use with spaCy. Will export an `nlp` object that you can use in the @@ -198,7 +200,7 @@ This functionality was previously available as part of the command `init-model`. -```cli +```bash $ python -m spacy init vectors [lang] [vectors_loc] [output_dir] [--prune] [--truncate] [--name] [--verbose] ``` @@ -215,7 +217,7 @@ $ python -m spacy init vectors [lang] [vectors_loc] [output_dir] [--prune] [--tr | `--help`, `-h` | Show help message and available arguments. ~~bool (flag)~~ | | **CREATES** | A spaCy pipeline directory containing the vocab and vectors. | -### init labels {#init-labels new="3" tag="command"} +### init labels {id="init-labels",version="3",tag="command"} Generate JSON files for the labels in the data. This helps speed up the training process, since spaCy won't have to preprocess the data to extract the labels. @@ -233,7 +235,7 @@ After generating the labels, you can provide them to components that accept a > path = "corpus/labels/ner.json > ``` -```cli +```bash $ python -m spacy init labels [config_path] [output_path] [--code] [--verbose] [--gpu-id] [overrides] ``` @@ -248,7 +250,7 @@ $ python -m spacy init labels [config_path] [output_path] [--code] [--verbose] [ | overrides | Config parameters to override. Should be options starting with `--` that correspond to the config section and value to override, e.g. `--paths.train ./train.spacy`. ~~Any (option/flag)~~ | | **CREATES** | The label files. | -## convert {#convert tag="command"} +## convert {id="convert",tag="command"} Convert files into spaCy's [binary training data format](/api/data-formats#binary-training), a serialized @@ -256,7 +258,7 @@ Convert files into spaCy's management functions. The converter can be specified on the command line, or chosen based on the file extension of the input file. -```cli +```bash $ python -m spacy convert [input_file] [output_dir] [--converter] [--file-type] [--n-sents] [--seg-sents] [--base] [--morphology] [--merge-subtokens] [--ner-map] [--lang] ``` @@ -268,16 +270,16 @@ $ python -m spacy convert [input_file] [output_dir] [--converter] [--file-type] | `--file-type`, `-t` | Type of file to create. Either `spacy` (default) for binary [`DocBin`](/api/docbin) data or `json` for v2.x JSON format. ~~str (option)~~ | | `--n-sents`, `-n` | Number of sentences per document. Supported for: `conll`, `conllu`, `iob`, `ner` ~~int (option)~~ | | `--seg-sents`, `-s` | Segment sentences. Supported for: `conll`, `ner` ~~bool (flag)~~ | -| `--base`, `-b`, `--model` | Trained spaCy pipeline for sentence segmentation to use as base (for `--seg-sents`). ~~Optional[str](option)~~ | +| `--base`, `-b`, `--model` | Trained spaCy pipeline for sentence segmentation to use as base (for `--seg-sents`). ~~Optional[str] (option)~~ | | `--morphology`, `-m` | Enable appending morphology to tags. Supported for: `conllu` ~~bool (flag)~~ | | `--merge-subtokens`, `-T` | Merge CoNLL-U subtokens ~~bool (flag)~~ | -| `--ner-map`, `-nm` | NER tag mapping (as JSON-encoded dict of entity types). Supported for: `conllu` ~~Optional[Path](option)~~ | +| `--ner-map`, `-nm` | NER tag mapping (as JSON-encoded dict of entity types). Supported for: `conllu` ~~Optional[Path] (option)~~ | | `--lang`, `-l` | Language code (if tokenizer required). ~~Optional[str] \(option)~~ | | `--concatenate`, `-C` | Concatenate output to a single file ~~bool (flag)~~ | | `--help`, `-h` | Show help message and available arguments. ~~bool (flag)~~ | | **CREATES** | Binary [`DocBin`](/api/docbin) training data that can be used with [`spacy train`](/api/cli#train). | -### Converters {#converters} +### Converters {id="converters"} | ID | Description | | --------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | @@ -287,12 +289,12 @@ $ python -m spacy convert [input_file] [output_dir] [--converter] [--file-type] | `ner` / `conll` | NER with IOB/IOB2/BILUO tags, one token per line with columns separated by whitespace. The first column is the token and the final column is the NER tag. Sentences are separated by blank lines and documents are separated by the line `-DOCSTART- -X- O O`. Supports CoNLL 2003 NER format. See [sample data](%%GITHUB_SPACY/extra/example_data/ner_example_data). | | `iob` | NER with IOB/IOB2/BILUO tags, one sentence per line with tokens separated by whitespace and annotation separated by `\|`, either `word\|B-ENT`or`word\|POS\|B-ENT`. See [sample data](%%GITHUB_SPACY/extra/example_data/ner_example_data). | -## debug {#debug new="3"} +## debug {id="debug",version="3"} The `spacy debug` CLI includes helpful commands for debugging and profiling your configs, data and implementations. -### debug config {#debug-config new="3" tag="command"} +### debug config {id="debug-config",version="3",tag="command"} Debug a [`config.cfg` file](/usage/training#config) and show validation errors. The command will create all objects in the tree and validate them. Note that @@ -302,13 +304,13 @@ errors at once and some issues are only shown once previous errors have been fixed. To auto-fill a partial config and save the result, you can use the [`init fill-config`](/api/cli#init-fill-config) command. -```cli +```bash $ python -m spacy debug config [config_path] [--code] [--show-functions] [--show-variables] [overrides] ``` > #### Example > -> ```cli +> ```bash > $ python -m spacy debug config config.cfg > ``` @@ -332,7 +334,7 @@ python -m spacy init fill-config tmp/starter-config_invalid.cfg tmp/starter-conf -```cli +```bash $ python -m spacy debug config ./config.cfg --show-functions --show-variables ``` @@ -452,7 +454,7 @@ File /path/to/thinc/thinc/schedules.py (line 91) | overrides | Config parameters to override. Should be options starting with `--` that correspond to the config section and value to override, e.g. `--paths.train ./train.spacy`. ~~Any (option/flag)~~ | | **PRINTS** | Config validation errors, if available. | -### debug data {#debug-data tag="command"} +### debug data {id="debug-data",tag="command"} Analyze, debug and validate your training and development data. Get useful stats, and find problems like invalid entity annotations, cyclic dependencies, @@ -474,17 +476,17 @@ report span characteristics such as the average span length and the span (or span boundary) distinctiveness. The distinctiveness measure shows how different the tokens are with respect to the rest of the corpus using the KL-divergence of the token distributions. To learn more, you can check out Papay et al.'s work on -[*Dissecting Span Identification Tasks with Performance Prediction* (EMNLP 2020)](https://aclanthology.org/2020.emnlp-main.396/). +[_Dissecting Span Identification Tasks with Performance Prediction_ (EMNLP 2020)](https://aclanthology.org/2020.emnlp-main.396/). -```cli +```bash $ python -m spacy debug data [config_path] [--code] [--ignore-warnings] [--verbose] [--no-format] [overrides] ``` > #### Example > -> ```cli +> ```bash > $ python -m spacy debug data ./config.cfg > ``` @@ -638,7 +640,7 @@ will not be available. | overrides | Config parameters to override. Should be options starting with `--` that correspond to the config section and value to override, e.g. `--paths.train ./train.spacy`. ~~Any (option/flag)~~ | | **PRINTS** | Debugging information. | -### debug diff-config {#debug-diff tag="command"} +### debug diff-config {id="debug-diff",tag="command"} Show a diff of a config file with respect to spaCy's defaults or another config file. If additional settings were used in the creation of the config file, then @@ -646,13 +648,13 @@ you must supply these as extra parameters to the command when comparing to the default settings. The generated diff can also be used when posting to the discussion forum to provide more information for the maintainers. -```cli +```bash $ python -m spacy debug diff-config [config_path] [--compare-to] [--optimize] [--gpu] [--pretraining] [--markdown] ``` > #### Example > -> ```cli +> ```bash > $ python -m spacy debug diff-config ./config.cfg > ``` @@ -867,7 +869,7 @@ after_init = null | `markdown`, `-md` | Generate Markdown for Github issues. Defaults to `False`. ~~bool (flag)~~ | | **PRINTS** | Diff between the two config files. | -### debug profile {#debug-profile tag="command"} +### debug profile {id="debug-profile",tag="command"} Profile which functions take the most time in a spaCy pipeline. Input should be formatted as one JSON object per line with a key `"text"`. It can either be @@ -881,7 +883,7 @@ The `profile` command is now available as a subcommand of `spacy debug`. -```cli +```bash $ python -m spacy debug profile [model] [inputs] [--n-texts] ``` @@ -893,12 +895,12 @@ $ python -m spacy debug profile [model] [inputs] [--n-texts] | `--help`, `-h` | Show help message and available arguments. ~~bool (flag)~~ | | **PRINTS** | Profiling information for the pipeline. | -### debug model {#debug-model new="3" tag="command"} +### debug model {id="debug-model",version="3",tag="command"} Debug a Thinc [`Model`](https://thinc.ai/docs/api-model) by running it on a sample text and checking how it updates its internal weights and parameters. -```cli +```bash $ python -m spacy debug model [config_path] [component] [--layers] [--dimensions] [--parameters] [--gradients] [--attributes] [--print-step0] [--print-step1] [--print-step2] [--print-step3] [--gpu-id] ``` @@ -909,7 +911,7 @@ model ("Step 0"), which helps us to understand the internal structure of the Neural Network, and to focus on specific layers that we want to inspect further (see next example). -```cli +```bash $ python -m spacy debug model ./config.cfg tagger -P0 ``` @@ -955,7 +957,7 @@ an all-zero matrix determined by the `nO` and `nI` dimensions. After a first training step (Step 2), this matrix has clearly updated its values through the training feedback loop. -```cli +```bash $ python -m spacy debug model ./config.cfg tagger -l "5,15" -DIM -PAR -P0 -P1 -P2 ``` @@ -1016,7 +1018,7 @@ $ python -m spacy debug model ./config.cfg tagger -l "5,15" -DIM -PAR -P0 -P1 -P | overrides | Config parameters to override. Should be options starting with `--` that correspond to the config section and value to override, e.g. `--paths.train ./train.spacy`. ~~Any (option/flag)~~ | | **PRINTS** | Debugging information. | -## train {#train tag="command"} +## train {id="train",tag="command"} Train a pipeline. Expects data in spaCy's [binary format](/api/data-formats#training) and a @@ -1042,11 +1044,11 @@ in the section `[paths]`. > #### Example > -> ```cli +> ```bash > $ python -m spacy train config.cfg --output ./output --paths.train ./train --paths.dev ./dev > ``` -```cli +```bash $ python -m spacy train [config_path] [--output] [--code] [--verbose] [--gpu-id] [overrides] ``` @@ -1061,7 +1063,7 @@ $ python -m spacy train [config_path] [--output] [--code] [--verbose] [--gpu-id] | overrides | Config parameters to override. Should be options starting with `--` that correspond to the config section and value to override, e.g. `--paths.train ./train.spacy`. ~~Any (option/flag)~~ | | **CREATES** | The final trained pipeline and the best trained pipeline. | -### Calling the training function from Python {#train-function new="3.2"} +### Calling the training function from Python {id="train-function",version="3.2"} The training CLI exposes a `train` helper function that lets you run the training just like `spacy train`. Usually it's easier to use the command line @@ -1084,7 +1086,7 @@ directly, but if you need to kick off training from code this is how to do it. | `use_gpu` | Which GPU to use. Defaults to -1 for no GPU. ~~int~~ | | `overrides` | Values to override config settings. ~~Dict[str, Any]~~ | -## pretrain {#pretrain new="2.1" tag="command,experimental"} +## pretrain {id="pretrain",version="2.1",tag="command,experimental"} Pretrain the "token to vector" ([`Tok2vec`](/api/tok2vec)) layer of pipeline components on raw text, using an approximate language-modeling objective. @@ -1112,30 +1114,42 @@ auto-generated by setting `--pretraining` on > #### Example > -> ```cli +> ```bash > $ python -m spacy pretrain config.cfg ./output_pretrain --paths.raw_text ./data.jsonl > ``` -```cli +```bash $ python -m spacy pretrain [config_path] [output_dir] [--code] [--resume-path] [--epoch-resume] [--gpu-id] [overrides] ``` -| Name | Description | -| ----------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -| `config_path` | Path to [training config](/api/data-formats#config) file containing all settings and hyperparameters. If `-`, the data will be [read from stdin](/usage/training#config-stdin). ~~Union[Path, str] \(positional)~~ | -| `output_dir` | Directory to save binary weights to on each epoch. ~~Path (positional)~~ | -| `--code`, `-c` | Path to Python file with additional code to be imported. Allows [registering custom functions](/usage/training#custom-functions) for new architectures. ~~Optional[Path] \(option)~~ | -| `--resume-path`, `-r` | Path to pretrained weights from which to resume pretraining. ~~Optional[Path] \(option)~~ | -| `--epoch-resume`, `-er` | The epoch to resume counting from when using `--resume-path`. Prevents unintended overwriting of existing weight files. ~~Optional[int] \(option)~~ | -| `--gpu-id`, `-g` | GPU ID or `-1` for CPU. Defaults to `-1`. ~~int (option)~~ | -| `--help`, `-h` | Show help message and available arguments. ~~bool (flag)~~ | -| overrides | Config parameters to override. Should be options starting with `--` that correspond to the config section and value to override, e.g. `--training.dropout 0.2`. ~~Any (option/flag)~~ | -| **CREATES** | The pretrained weights that can be used to initialize `spacy train`. | +| Name | Description | +| -------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `config_path` | Path to [training config](/api/data-formats#config) file containing all settings and hyperparameters. If `-`, the data will be [read from stdin](/usage/training#config-stdin). ~~Union[Path, str] \(positional)~~ | +| `output_dir` | Directory to save binary weights to on each epoch. ~~Path (positional)~~ | +| `--code`, `-c` | Path to Python file with additional code to be imported. Allows [registering custom functions](/usage/training#custom-functions) for new architectures. ~~Optional[Path] \(option)~~ | +| `--resume-path`, `-r` | Path to pretrained weights from which to resume pretraining. ~~Optional[Path] \(option)~~ | +| `--epoch-resume`, `-er` | The epoch to resume counting from when using `--resume-path`. Prevents unintended overwriting of existing weight files. ~~Optional[int] \(option)~~ | +| `--gpu-id`, `-g` | GPU ID or `-1` for CPU. Defaults to `-1`. ~~int (option)~~ | +| `--skip-last`, `-L` 3.5.2 | Skip saving `model-last.bin`. Defaults to `False`. ~~bool (flag)~~ | +| `--help`, `-h` | Show help message and available arguments. ~~bool (flag)~~ | +| overrides | Config parameters to override. Should be options starting with `--` that correspond to the config section and value to override, e.g. `--training.dropout 0.2`. ~~Any (option/flag)~~ | +| **CREATES** | The pretrained weights that can be used to initialize `spacy train`. | -## evaluate {#evaluate new="2" tag="command"} +## evaluate {id="evaluate",version="2",tag="command"} -Evaluate a trained pipeline. Expects a loadable spaCy pipeline (package name or -path) and evaluation data in the +The `evaluate` subcommand is superseded by +[`spacy benchmark accuracy`](#benchmark-accuracy). `evaluate` is provided as an +alias to `benchmark accuracy` for compatibility. + +## benchmark {id="benchmark", version="3.5"} + +The `spacy benchmark` CLI includes commands for benchmarking the accuracy and +speed of your spaCy pipelines. + +### accuracy {id="benchmark-accuracy", version="3.5", tag="command"} + +Evaluate the accuracy of a trained pipeline. Expects a loadable spaCy pipeline +(package name or path) and evaluation data in the [binary `.spacy` format](/api/data-formats#binary-training). The `--gold-preproc` option sets up the evaluation examples with gold-standard sentences and tokens for the predictions. Gold preprocessing helps the @@ -1145,8 +1159,8 @@ skew. To render a sample of dependency parses in a HTML file using the [displaCy visualizations](/usage/visualizers), set as output directory as the `--displacy-path` argument. -```cli -$ python -m spacy evaluate [model] [data_path] [--output] [--code] [--gold-preproc] [--gpu-id] [--displacy-path] [--displacy-limit] +```bash +$ python -m spacy benchmark accuracy [model] [data_path] [--output] [--code] [--gold-preproc] [--gpu-id] [--displacy-path] [--displacy-limit] ``` | Name | Description | @@ -1162,7 +1176,61 @@ $ python -m spacy evaluate [model] [data_path] [--output] [--code] [--gold-prepr | `--help`, `-h` | Show help message and available arguments. ~~bool (flag)~~ | | **CREATES** | Training results and optional metrics and visualizations. | -## find-threshold {#find-threshold new="3.5" tag="command"} +### speed {id="benchmark-speed", version="3.5", tag="command"} + +Benchmark the speed of a trained pipeline with a 95% confidence interval. +Expects a loadable spaCy pipeline (package name or path) and benchmark data in +the [binary `.spacy` format](/api/data-formats#binary-training). The pipeline is +warmed up before any measurements are taken. + +```cli +$ python -m spacy benchmark speed [model] [data_path] [--batch_size] [--no-shuffle] [--gpu-id] [--batches] [--warmup] +``` + +| Name | Description | +| -------------------- | -------------------------------------------------------------------------------------------------------- | +| `model` | Pipeline to benchmark the speed of. Can be a package or a path to a data directory. ~~str (positional)~~ | +| `data_path` | Location of benchmark data in spaCy's [binary format](/api/data-formats#training). ~~Path (positional)~~ | +| `--batch-size`, `-b` | Set the batch size. If not set, the pipeline's batch size is used. ~~Optional[int] \(option)~~ | +| `--no-shuffle` | Do not shuffle documents in the benchmark data. ~~bool (flag)~~ | +| `--gpu-id`, `-g` | GPU to use, if any. Defaults to `-1` for CPU. ~~int (option)~~ | +| `--batches` | Number of batches to benchmark on. Defaults to `50`. ~~Optional[int] \(option)~~ | +| `--warmup`, `-w` | Iterations over the benchmark data for warmup. Defaults to `3` ~~Optional[int] \(option)~~ | +| `--help`, `-h` | Show help message and available arguments. ~~bool (flag)~~ | +| **PRINTS** | Pipeline speed in words per second with a 95% confidence interval. | + +## apply {id="apply", version="3.5", tag="command"} + +Applies a trained pipeline to data and stores the resulting annotated documents +in a `DocBin`. The input can be a single file or a directory. The recognized +input formats are: + +1. `.spacy` +2. `.jsonl` containing a user specified `text_key` +3. Files with any other extension are assumed to be plain text files containing + a single document. + +When a directory is provided it is traversed recursively to collect all files. + +```bash +$ python -m spacy apply [model] [data-path] [output-file] [--code] [--text-key] [--force-overwrite] [--gpu-id] [--batch-size] [--n-process] +``` + +| Name | Description | +| ------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `model` | Pipeline to apply to the data. Can be a package or a path to a data directory. ~~str (positional)~~ | +| `data_path` | Location of data to be evaluated in spaCy's [binary format](/api/data-formats#training), jsonl, or plain text. ~~Path (positional)~~ | +| `output-file`, `-o` | Output `DocBin` path. ~~str (positional)~~ | +| `--code`, `-c` | Path to Python file with additional code to be imported. Allows [registering custom functions](/usage/training#custom-functions) for new architectures. ~~Optional[Path] \(option)~~ | +| `--text-key`, `-tk` | The key for `.jsonl` files to use to grab the texts from. Defaults to `text`. ~~Optional[str] \(option)~~ | +| `--force-overwrite`, `-F` | If the provided `output-file` already exists, then force `apply` to overwrite it. If this is `False` (default) then quits with a warning instead. ~~bool (flag)~~ | +| `--gpu-id`, `-g` | GPU to use, if any. Defaults to `-1` for CPU. ~~int (option)~~ | +| `--batch-size`, `-b` | Batch size to use for prediction. Defaults to `1`. ~~int (option)~~ | +| `--n-process`, `-n` | Number of processes to use for prediction. Defaults to `1`. ~~int (option)~~ | +| `--help`, `-h` | Show help message and available arguments. ~~bool (flag)~~ | +| **CREATES** | A `DocBin` with the annotations from the `model` for all the files found in `data-path`. | + +## find-threshold {id="find-threshold",version="3.5",tag="command"} Runs prediction trials for a trained model with varying tresholds to maximize the specified metric. The search space for the threshold is traversed linearly @@ -1177,32 +1245,31 @@ be provided. > #### Examples > -> ```cli +> ```bash > # For textcat_multilabel: > $ python -m spacy find-threshold my_nlp data.spacy textcat_multilabel threshold cats_macro_f > ``` > -> ```cli +> ```bash > # For spancat: > $ python -m spacy find-threshold my_nlp data.spacy spancat threshold spans_sc_f > ``` +| Name | Description | +| ------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `model` | Pipeline to evaluate. Can be a package or a path to a data directory. ~~str (positional)~~ | +| `data_path` | Path to file with DocBin with docs to use for threshold search. ~~Path (positional)~~ | +| `pipe_name` | Name of pipe to examine thresholds for. ~~str (positional)~~ | +| `threshold_key` | Key of threshold attribute in component's configuration. ~~str (positional)~~ | +| `scores_key` | Name of score to metric to optimize. ~~str (positional)~~ | +| `--n_trials`, `-n` | Number of trials to determine optimal thresholds. ~~int (option)~~ | +| `--code`, `-c` | Path to Python file with additional code to be imported. Allows [registering custom functions](/usage/training#custom-functions) for new architectures. ~~Optional[Path] \(option)~~ | +| `--gpu-id`, `-g` | GPU to use, if any. Defaults to `-1` for CPU. ~~int (option)~~ | +| `--gold-preproc`, `-G` | Use gold preprocessing. ~~bool (flag)~~ | +| `--verbose`, `-V`, `-VV` | Display more information for debugging purposes. ~~bool (flag)~~ | +| `--help`, `-h` | Show help message and available arguments. ~~bool (flag)~~ | -| Name | Description | -| ----------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -| `model` | Pipeline to evaluate. Can be a package or a path to a data directory. ~~str (positional)~~ | -| `data_path` | Path to file with DocBin with docs to use for threshold search. ~~Path (positional)~~ | -| `pipe_name` | Name of pipe to examine thresholds for. ~~str (positional)~~ | -| `threshold_key` | Key of threshold attribute in component's configuration. ~~str (positional)~~ | -| `scores_key` | Name of score to metric to optimize. ~~str (positional)~~ | -| `--n_trials`, `-n` | Number of trials to determine optimal thresholds. ~~int (option)~~ | -| `--code`, `-c` | Path to Python file with additional code to be imported. Allows [registering custom functions](/usage/training#custom-functions) for new architectures. ~~Optional[Path] \(option)~~ | -| `--gpu-id`, `-g` | GPU to use, if any. Defaults to `-1` for CPU. ~~int (option)~~ | -| `--gold-preproc`, `-G` | Use gold preprocessing. ~~bool (flag)~~ | -| `--silent`, `-V`, `-VV` | GPU to use, if any. Defaults to `-1` for CPU. ~~int (option)~~ | -| `--help`, `-h` | Show help message and available arguments. ~~bool (flag)~~ | - -## assemble {#assemble tag="command"} +## assemble {id="assemble",tag="command"} Assemble a pipeline from a config file without additional training. Expects a [config file](/api/data-formats#config) with all settings and hyperparameters. @@ -1212,11 +1279,11 @@ config. > #### Example > -> ```cli +> ```bash > $ python -m spacy assemble config.cfg ./output > ``` -```cli +```bash $ python -m spacy assemble [config_path] [output_dir] [--code] [--verbose] [overrides] ``` @@ -1230,7 +1297,7 @@ $ python -m spacy assemble [config_path] [output_dir] [--code] [--verbose] [over | overrides | Config parameters to override. Should be options starting with `--` that correspond to the config section and value to override, e.g. `--paths.data ./data`. ~~Any (option/flag)~~ | | **CREATES** | The final assembled pipeline. | -## package {#package tag="command"} +## package {id="package",tag="command"} Generate an installable [Python package](/usage/training#models-generating) from an existing pipeline data directory. All data files are copied over. If @@ -1256,13 +1323,13 @@ the sdist and wheel by setting `--build sdist,wheel`. -```cli +```bash $ python -m spacy package [input_dir] [output_dir] [--code] [--meta-path] [--create-meta] [--build] [--name] [--version] [--force] ``` > #### Example > -> ```cli +> ```bash > $ python -m spacy package /input /output > $ cd /output/en_pipeline-0.0.0 > $ pip install dist/en_pipeline-0.0.0.tar.gz @@ -1282,13 +1349,13 @@ $ python -m spacy package [input_dir] [output_dir] [--code] [--meta-path] [--cre | `--help`, `-h` | Show help message and available arguments. ~~bool (flag)~~ | | **CREATES** | A Python package containing the spaCy pipeline. | -## project {#project new="3"} +## project {id="project",version="3"} The `spacy project` CLI includes subcommands for working with [spaCy projects](/usage/projects), end-to-end workflows for building and deploying custom spaCy pipelines. -### project clone {#project-clone tag="command"} +### project clone {id="project-clone",tag="command"} Clone a project template from a Git repository. Calls into `git` under the hood and can use the sparse checkout feature if available, so you're only downloading @@ -1297,19 +1364,19 @@ what you need. By default, spaCy's can provide any other repo (public or private) that you have access to using the `--repo` option. -```cli +```bash $ python -m spacy project clone [name] [dest] [--repo] [--branch] [--sparse] ``` > #### Example > -> ```cli +> ```bash > $ python -m spacy project clone pipelines/ner_wikiner > ``` > > Clone from custom repo: > -> ```cli +> ```bash > $ python -m spacy project clone template --repo https://github.com/your_org/your_repo > ``` @@ -1323,7 +1390,7 @@ $ python -m spacy project clone [name] [dest] [--repo] [--branch] [--sparse] | `--help`, `-h` | Show help message and available arguments. ~~bool (flag)~~ | | **CREATES** | The cloned [project directory](/usage/projects#project-files). | -### project assets {#project-assets tag="command"} +### project assets {id="project-assets",tag="command"} Fetch project assets like datasets and pretrained weights. Assets are defined in the `assets` section of the [`project.yml`](/usage/projects#project-yml). If a @@ -1334,24 +1401,25 @@ considered "private" and you have to take care of putting them into the destination directory yourself. If a local path is provided, the asset is copied into the current project. -```cli +```bash $ python -m spacy project assets [project_dir] ``` > #### Example > -> ```cli +> ```bash > $ python -m spacy project assets [--sparse] > ``` -| Name | Description | -| ---------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `project_dir` | Path to project directory. Defaults to current working directory. ~~Path (positional)~~ | -| `--sparse`, `-S` | Enable [sparse checkout](https://git-scm.com/docs/git-sparse-checkout) to only check out and download what's needed. Requires Git v22.2+. ~~bool (flag)~~ | -| `--help`, `-h` | Show help message and available arguments. ~~bool (flag)~~ | -| **CREATES** | Downloaded or copied assets defined in the `project.yml`. | +| Name | Description | +| ---------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `project_dir` | Path to project directory. Defaults to current working directory. ~~Path (positional)~~ | +| `--extra`, `-e` 3.3.1 | Download assets marked as "extra". Default false. ~~bool (flag)~~ | +| `--sparse`, `-S` | Enable [sparse checkout](https://git-scm.com/docs/git-sparse-checkout) to only check out and download what's needed. Requires Git v22.2+. ~~bool (flag)~~ | +| `--help`, `-h` | Show help message and available arguments. ~~bool (flag)~~ | +| **CREATES** | Downloaded or copied assets defined in the `project.yml`. | -### project run {#project-run tag="command"} +### project run {id="project-run",tag="command"} Run a named command or workflow defined in the [`project.yml`](/usage/projects#project-yml). If a workflow name is specified, @@ -1360,13 +1428,13 @@ all commands in the workflow are run, in order. If commands define re-run if state has changed. For example, if the input dataset changes, a preprocessing command that depends on those files will be re-run. -```cli +```bash $ python -m spacy project run [subcommand] [project_dir] [--force] [--dry] ``` > #### Example > -> ```cli +> ```bash > $ python -m spacy project run train > ``` @@ -1379,7 +1447,7 @@ $ python -m spacy project run [subcommand] [project_dir] [--force] [--dry] | `--help`, `-h` | Show help message and available arguments. ~~bool (flag)~~ | | **EXECUTES** | The command defined in the `project.yml`. | -### project push {#project-push tag="command"} +### project push {id="project-push",tag="command"} Upload all available files or directories listed as in the `outputs` section of commands to a remote storage. Outputs are archived and compressed prior to @@ -1399,13 +1467,13 @@ remote storages, so you can use any protocol that `Pathy` supports, including filesystem, although you may need to install extra dependencies to use certain protocols. -```cli +```bash $ python -m spacy project push [remote] [project_dir] ``` > #### Example > -> ```cli +> ```bash > $ python -m spacy project push my_bucket > ``` > @@ -1422,10 +1490,10 @@ $ python -m spacy project push [remote] [project_dir] | `--help`, `-h` | Show help message and available arguments. ~~bool (flag)~~ | | **UPLOADS** | All project outputs that exist and are not already stored in the remote. | -### project pull {#project-pull tag="command"} +### project pull {id="project-pull",tag="command"} Download all files or directories listed as `outputs` for commands, unless they -are not already present locally. When searching for files in the remote, `pull` +are already present locally. When searching for files in the remote, `pull` won't just look at the output path, but will also consider the **command string** and the **hashes of the dependencies**. For instance, let's say you've previously pushed a checkpoint to the remote, but now you've changed some @@ -1444,13 +1512,13 @@ remote storages, so you can use any protocol that `Pathy` supports, including filesystem, although you may need to install extra dependencies to use certain protocols. -```cli +```bash $ python -m spacy project pull [remote] [project_dir] ``` > #### Example > -> ```cli +> ```bash > $ python -m spacy project pull my_bucket > ``` > @@ -1467,7 +1535,7 @@ $ python -m spacy project pull [remote] [project_dir] | `--help`, `-h` | Show help message and available arguments. ~~bool (flag)~~ | | **DOWNLOADS** | All project outputs that do not exist locally and can be found in the remote. | -### project document {#project-document tag="command"} +### project document {id="project-document",tag="command"} Auto-generate a pretty Markdown-formatted `README` for your project, based on its [`project.yml`](/usage/projects#project-yml). Will create sections that @@ -1476,13 +1544,13 @@ content will be placed between two hidden markers, so you can add your own custom content before or after the auto-generated documentation. When you re-run the `project document` command, only the auto-generated part is replaced. -```cli +```bash $ python -m spacy project document [project_dir] [--output] [--no-emoji] ``` > #### Example > -> ```cli +> ```bash > $ python -m spacy project document --output README.md > ``` @@ -1491,7 +1559,7 @@ $ python -m spacy project document [project_dir] [--output] [--no-emoji] For more examples, see the templates in our [`projects`](https://github.com/explosion/projects) repo. -![Screenshot of auto-generated Markdown Readme](../images/project_document.jpg) +![Screenshot of auto-generated Markdown Readme](/images/project_document.jpg) @@ -1502,7 +1570,7 @@ For more examples, see the templates in our | `--no-emoji`, `-NE` | Don't use emoji in the titles. ~~bool (flag)~~ | | **CREATES** | The Markdown-formatted project documentation. | -### project dvc {#project-dvc tag="command"} +### project dvc {id="project-dvc",tag="command"} Auto-generate [Data Version Control](https://dvc.org) (DVC) config file. Calls [`dvc run`](https://dvc.org/doc/command-reference/run) with `--no-exec` under @@ -1522,13 +1590,13 @@ You'll also need to add the assets you want to track with -```cli +```bash $ python -m spacy project dvc [project_dir] [workflow] [--force] [--verbose] [--quiet] ``` > #### Example > -> ```cli +> ```bash > $ git init > $ dvc init > $ python -m spacy project dvc all @@ -1544,14 +1612,14 @@ $ python -m spacy project dvc [project_dir] [workflow] [--force] [--verbose] [-- | `--help`, `-h` | Show help message and available arguments. ~~bool (flag)~~ | | **CREATES** | A `dvc.yaml` file in the project directory, based on the steps defined in the given workflow. | -## huggingface-hub {#huggingface-hub new="3.1"} +## huggingface-hub {id="huggingface-hub",version="3.1"} The `spacy huggingface-cli` CLI includes commands for uploading your trained spaCy pipelines to the [Hugging Face Hub](https://huggingface.co/). > #### Installation > -> ```cli +> ```bash > $ pip install spacy-huggingface-hub > $ huggingface-cli login > ``` @@ -1565,19 +1633,19 @@ package installed. Installing the package will automatically add the -### huggingface-hub push {#huggingface-hub-push tag="command"} +### huggingface-hub push {id="huggingface-hub-push",tag="command"} Push a spaCy pipeline to the Hugging Face Hub. Expects a `.whl` file packaged with [`spacy package`](/api/cli#package) and `--build wheel`. For more details, see the spaCy project [integration](/usage/projects#huggingface_hub). -```cli +```bash $ python -m spacy huggingface-hub push [whl_path] [--org] [--msg] [--local-repo] [--verbose] ``` > #### Example > -> ```cli +> ```bash > $ python -m spacy huggingface-hub push en_ner_fashion-0.0.0-py3-none-any.whl > ``` diff --git a/website/docs/api/coref.md b/website/docs/api/coref.mdx similarity index 94% rename from website/docs/api/coref.md rename to website/docs/api/coref.mdx index 8f54422d6..0b9ebb888 100644 --- a/website/docs/api/coref.md +++ b/website/docs/api/coref.mdx @@ -34,7 +34,7 @@ same thing. Clusters are represented as SpanGroups that start with a prefix A `CoreferenceResolver` component can be paired with a [`SpanResolver`](/api/span-resolver) to expand single tokens to spans. -## Assigned Attributes {#assigned-attributes} +## Assigned Attributes {id="assigned-attributes"} Predictions will be saved to `Doc.spans` as a [`SpanGroup`](/api/spangroup). The span key will be a prefix plus a serial number referring to the coreference @@ -47,7 +47,7 @@ parameter. | ------------------------------------------ | ------------------------------------------------------------------------------------------------------- | | `Doc.spans[prefix + "_" + cluster_number]` | One coreference cluster, represented as single-token spans. Cluster numbers start from 1. ~~SpanGroup~~ | -## Config and implementation {#config} +## Config and implementation {id="config"} The default config is defined by the pipeline component factory and describes how the component should be configured. You can override its settings via the @@ -64,7 +64,7 @@ details on the architectures and their arguments and hyperparameters. > config={ > "model": DEFAULT_COREF_MODEL, > "span_cluster_prefix": DEFAULT_CLUSTER_PREFIX, -> }, +> } > nlp.add_pipe("experimental_coref", config=config) > ``` @@ -73,7 +73,7 @@ details on the architectures and their arguments and hyperparameters. | `model` | The [`Model`](https://thinc.ai/docs/api-model) powering the pipeline component. Defaults to [Coref](/api/architectures#Coref). ~~Model~~ | | `span_cluster_prefix` | The prefix for the keys for clusters saved to `doc.spans`. Defaults to `coref_clusters`. ~~str~~ | -## CoreferenceResolver.\_\_init\_\_ {#init tag="method"} +## CoreferenceResolver.\_\_init\_\_ {id="init",tag="method"} > #### Example > @@ -102,7 +102,7 @@ shortcut for this and instantiate the component using its string name and | _keyword-only_ | | | `span_cluster_prefix` | The prefix for the key for saving clusters of spans. ~~bool~~ | -## CoreferenceResolver.\_\_call\_\_ {#call tag="method"} +## CoreferenceResolver.\_\_call\_\_ {id="call",tag="method"} Apply the pipe to one document. The document is modified in place and returned. This usually happens under the hood when the `nlp` object is called on a text @@ -125,7 +125,7 @@ and all pipeline components are applied to the `Doc` in order. Both | `doc` | The document to process. ~~Doc~~ | | **RETURNS** | The processed document. ~~Doc~~ | -## CoreferenceResolver.pipe {#pipe tag="method"} +## CoreferenceResolver.pipe {id="pipe",tag="method"} Apply the pipe to a stream of documents. This usually happens under the hood when the `nlp` object is called on a text and all pipeline components are @@ -148,7 +148,7 @@ applied to the `Doc` in order. Both [`__call__`](/api/coref#call) and | `batch_size` | The number of documents to buffer. Defaults to `128`. ~~int~~ | | **YIELDS** | The processed documents in order. ~~Doc~~ | -## CoreferenceResolver.initialize {#initialize tag="method"} +## CoreferenceResolver.initialize {id="initialize",tag="method"} Initialize the component for training. `get_examples` should be a function that returns an iterable of [`Example`](/api/example) objects. **At least one example @@ -172,7 +172,7 @@ by [`Language.initialize`](/api/language#initialize). | _keyword-only_ | | | `nlp` | The current `nlp` object. Defaults to `None`. ~~Optional[Language]~~ | -## CoreferenceResolver.predict {#predict tag="method"} +## CoreferenceResolver.predict {id="predict",tag="method"} Apply the component's model to a batch of [`Doc`](/api/doc) objects, without modifying them. Clusters are returned as a list of `MentionClusters`, one for @@ -192,7 +192,7 @@ to token indices. | `docs` | The documents to predict. ~~Iterable[Doc]~~ | | **RETURNS** | The predicted coreference clusters for the `docs`. ~~List[MentionClusters]~~ | -## CoreferenceResolver.set_annotations {#set_annotations tag="method"} +## CoreferenceResolver.set_annotations {id="set_annotations",tag="method"} Modify a batch of documents, saving coreference clusters in `Doc.spans`. @@ -209,7 +209,7 @@ Modify a batch of documents, saving coreference clusters in `Doc.spans`. | `docs` | The documents to modify. ~~Iterable[Doc]~~ | | `clusters` | The predicted coreference clusters for the `docs`. ~~List[MentionClusters]~~ | -## CoreferenceResolver.update {#update tag="method"} +## CoreferenceResolver.update {id="update",tag="method"} Learn from a batch of [`Example`](/api/example) objects. Delegates to [`predict`](/api/coref#predict). @@ -231,7 +231,7 @@ Learn from a batch of [`Example`](/api/example) objects. Delegates to | `losses` | Optional record of the loss during training. Updated using the component name as the key. ~~Optional[Dict[str, float]]~~ | | **RETURNS** | The updated `losses` dictionary. ~~Dict[str, float]~~ | -## CoreferenceResolver.create_optimizer {#create_optimizer tag="method"} +## CoreferenceResolver.create_optimizer {id="create_optimizer",tag="method"} Create an optimizer for the pipeline component. @@ -246,7 +246,7 @@ Create an optimizer for the pipeline component. | ----------- | ---------------------------- | | **RETURNS** | The optimizer. ~~Optimizer~~ | -## CoreferenceResolver.use_params {#use_params tag="method, contextmanager"} +## CoreferenceResolver.use_params {id="use_params",tag="method, contextmanager"} Modify the pipe's model, to use the given parameter values. At the end of the context, the original parameters are restored. @@ -263,7 +263,7 @@ context, the original parameters are restored. | -------- | -------------------------------------------------- | | `params` | The parameter values to use in the model. ~~dict~~ | -## CoreferenceResolver.to_disk {#to_disk tag="method"} +## CoreferenceResolver.to_disk {id="to_disk",tag="method"} Serialize the pipe to disk. @@ -280,7 +280,7 @@ Serialize the pipe to disk. | _keyword-only_ | | | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | -## CoreferenceResolver.from_disk {#from_disk tag="method"} +## CoreferenceResolver.from_disk {id="from_disk",tag="method"} Load the pipe from disk. Modifies the object in place and returns it. @@ -298,7 +298,7 @@ Load the pipe from disk. Modifies the object in place and returns it. | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | | **RETURNS** | The modified `CoreferenceResolver` object. ~~CoreferenceResolver~~ | -## CoreferenceResolver.to_bytes {#to_bytes tag="method"} +## CoreferenceResolver.to_bytes {id="to_bytes",tag="method"} > #### Example > @@ -315,7 +315,7 @@ Serialize the pipe to a bytestring, including the `KnowledgeBase`. | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | | **RETURNS** | The serialized form of the `CoreferenceResolver` object. ~~bytes~~ | -## CoreferenceResolver.from_bytes {#from_bytes tag="method"} +## CoreferenceResolver.from_bytes {id="from_bytes",tag="method"} Load the pipe from a bytestring. Modifies the object in place and returns it. @@ -334,7 +334,7 @@ Load the pipe from a bytestring. Modifies the object in place and returns it. | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | | **RETURNS** | The `CoreferenceResolver` object. ~~CoreferenceResolver~~ | -## Serialization fields {#serialization-fields} +## Serialization fields {id="serialization-fields"} During serialization, spaCy will export several data fields used to restore different aspects of the object. If needed, you can exclude them from diff --git a/website/docs/api/corpus.md b/website/docs/api/corpus.mdx similarity index 74% rename from website/docs/api/corpus.md rename to website/docs/api/corpus.mdx index 88c4befd7..75e8f5c0f 100644 --- a/website/docs/api/corpus.md +++ b/website/docs/api/corpus.mdx @@ -3,7 +3,7 @@ title: Corpus teaser: An annotated corpus tag: class source: spacy/training/corpus.py -new: 3 +version: 3 --- This class manages annotated corpora and can be used for training and @@ -13,7 +13,7 @@ customize the data loading during training, you can register your own see the usage guide on [data utilities](/usage/training#data) for more details and examples. -## Config and implementation {#config} +## Config and implementation {id="config"} `spacy.Corpus.v1` is a registered function that creates a `Corpus` of training or evaluation data. It takes the same arguments as the `Corpus` class and @@ -49,7 +49,7 @@ streaming. %%GITHUB_SPACY/spacy/training/corpus.py ``` -## Corpus.\_\_init\_\_ {#init tag="method"} +## Corpus.\_\_init\_\_ {id="init",tag="method"} Create a `Corpus` for iterating [Example](/api/example) objects from a file or directory of [`.spacy` data files](/api/data-formats#binary-training). The @@ -81,7 +81,7 @@ train/test skew. | `augmenter` | Optional data augmentation callback. ~~Callable[[Language, Example], Iterable[Example]]~~ | | `shuffle` | Whether to shuffle the examples. Defaults to `False`. ~~bool~~ | -## Corpus.\_\_call\_\_ {#call tag="method"} +## Corpus.\_\_call\_\_ {id="call",tag="method"} Yield examples from the data. @@ -101,7 +101,7 @@ Yield examples from the data. | `nlp` | The current `nlp` object. ~~Language~~ | | **YIELDS** | The examples. ~~Example~~ | -## JsonlCorpus {#jsonlcorpus tag="class"} +## JsonlCorpus {id="jsonlcorpus",tag="class"} Iterate Doc objects from a file or directory of JSONL (newline-delimited JSON) formatted raw text files. Can be used to read the raw text corpus for language @@ -120,14 +120,13 @@ file. > srsly.write_jsonl("/path/to/text.jsonl", data) > ``` -```json -### Example +```json {title="Example"} {"text": "Can I ask where you work now and what you do, and if you enjoy it?"} {"text": "They may just pull out of the Seattle market completely, at least until they have autonomous vehicles."} {"text": "My cynical view on this is that it will never be free to the public. Reason: what would be the draw of joining the military? Right now their selling point is free Healthcare and Education. Ironically both are run horribly and most, that I've talked to, come out wishing they never went in."} ``` -### JsonlCorpus.\_\init\_\_ {#jsonlcorpus tag="method"} +### JsonlCorpus.\_\_init\_\_ {id="jsonlcorpus",tag="method"} Initialize the reader. @@ -157,7 +156,7 @@ Initialize the reader. | `max_length` | Maximum document length (in tokens). Longer documents will be skipped. Defaults to `0`, which indicates no limit. ~~int~~ | | `limit` | Limit corpus to a subset of examples, e.g. for debugging. Defaults to `0` for no limit. ~~int~~ | -### JsonlCorpus.\_\_call\_\_ {#jsonlcorpus-call tag="method"} +### JsonlCorpus.\_\_call\_\_ {id="jsonlcorpus-call",tag="method"} Yield examples from the data. @@ -176,3 +175,68 @@ Yield examples from the data. | ---------- | -------------------------------------- | | `nlp` | The current `nlp` object. ~~Language~~ | | **YIELDS** | The examples. ~~Example~~ | + +## PlainTextCorpus {id="plaintextcorpus",tag="class",version="3.5.1"} + +Iterate over documents from a plain text file. Can be used to read the raw text +corpus for language model +[pretraining](/usage/embeddings-transformers#pretraining). The expected file +format is: + +- UTF-8 encoding +- One document per line +- Blank lines are ignored. + +```text {title="Example"} +Can I ask where you work now and what you do, and if you enjoy it? +They may just pull out of the Seattle market completely, at least until they have autonomous vehicles. +My cynical view on this is that it will never be free to the public. Reason: what would be the draw of joining the military? Right now their selling point is free Healthcare and Education. Ironically both are run horribly and most, that I've talked to, come out wishing they never went in. +``` + +### PlainTextCorpus.\_\_init\_\_ {id="plaintextcorpus-init",tag="method"} + +Initialize the reader. + +> #### Example +> +> ```python +> from spacy.training import PlainTextCorpus +> +> corpus = PlainTextCorpus("./data/docs.txt") +> ``` +> +> ```ini +> ### Example config +> [corpora.pretrain] +> @readers = "spacy.PlainTextCorpus.v1" +> path = "corpus/raw_text.txt" +> min_length = 0 +> max_length = 0 +> ``` + +| Name | Description | +| -------------- | -------------------------------------------------------------------------------------------------------------------------- | +| `path` | The directory or filename to read from. Expects newline-delimited documents in UTF8 format. ~~Union[str, Path]~~ | +| _keyword-only_ | | +| `min_length` | Minimum document length (in tokens). Shorter documents will be skipped. Defaults to `0`, which indicates no limit. ~~int~~ | +| `max_length` | Maximum document length (in tokens). Longer documents will be skipped. Defaults to `0`, which indicates no limit. ~~int~~ | + +### PlainTextCorpus.\_\_call\_\_ {id="plaintextcorpus-call",tag="method"} + +Yield examples from the data. + +> #### Example +> +> ```python +> from spacy.training import PlainTextCorpus +> import spacy +> +> corpus = PlainTextCorpus("./docs.txt") +> nlp = spacy.blank("en") +> data = corpus(nlp) +> ``` + +| Name | Description | +| ---------- | -------------------------------------- | +| `nlp` | The current `nlp` object. ~~Language~~ | +| **YIELDS** | The examples. ~~Example~~ | diff --git a/website/docs/api/cython-classes.md b/website/docs/api/cython-classes.mdx similarity index 91% rename from website/docs/api/cython-classes.md rename to website/docs/api/cython-classes.mdx index a4ecf294a..ce7c03940 100644 --- a/website/docs/api/cython-classes.md +++ b/website/docs/api/cython-classes.mdx @@ -9,7 +9,7 @@ menu: - ['StringStore', 'stringstore'] --- -## Doc {#doc tag="cdef class" source="spacy/tokens/doc.pxd"} +## Doc {id="doc",tag="cdef class",source="spacy/tokens/doc.pxd"} The `Doc` object holds an array of [`TokenC`](/api/cython-structs#tokenc) structs. @@ -21,7 +21,7 @@ accessed from Python. For the Python documentation, see [`Doc`](/api/doc). -### Attributes {#doc_attributes} +### Attributes {id="doc_attributes"} | Name | Description | | ------------ | -------------------------------------------------------------------------------------------------------- | @@ -31,7 +31,7 @@ accessed from Python. For the Python documentation, see [`Doc`](/api/doc). | `length` | The number of tokens in the document. ~~int~~ | | `max_length` | The underlying size of the `Doc.c` array. ~~int~~ | -### Doc.push_back {#doc_push_back tag="method"} +### Doc.push_back {id="doc_push_back",tag="method"} Append a token to the `Doc`. The token can be provided as a [`LexemeC`](/api/cython-structs#lexemec) or @@ -55,7 +55,7 @@ Append a token to the `Doc`. The token can be provided as a | `lex_or_tok` | The word to append to the `Doc`. ~~LexemeOrToken~~ | | `has_space` | Whether the word has trailing whitespace. ~~bint~~ | -## Token {#token tag="cdef class" source="spacy/tokens/token.pxd"} +## Token {id="token",tag="cdef class",source="spacy/tokens/token.pxd"} A Cython class providing access and methods for a [`TokenC`](/api/cython-structs#tokenc) struct. Note that the `Token` object does @@ -68,7 +68,7 @@ accessed from Python. For the Python documentation, see [`Token`](/api/token). -### Attributes {#token_attributes} +### Attributes {id="token_attributes"} | Name | Description | | ------- | -------------------------------------------------------------------------- | @@ -77,7 +77,7 @@ accessed from Python. For the Python documentation, see [`Token`](/api/token). | `i` | The offset of the token within the document. ~~int~~ | | `doc` | The parent document. ~~Doc~~ | -### Token.cinit {#token_cinit tag="method"} +### Token.cinit {id="token_cinit",tag="method"} Create a `Token` object from a `TokenC*` pointer. @@ -94,7 +94,7 @@ Create a `Token` object from a `TokenC*` pointer. | `offset` | The offset of the token within the document. ~~int~~ | | `doc` | The parent document. ~~int~~ | -## Span {#span tag="cdef class" source="spacy/tokens/span.pxd"} +## Span {id="span",tag="cdef class",source="spacy/tokens/span.pxd"} A Cython class providing access and methods for a slice of a `Doc` object. @@ -105,7 +105,7 @@ accessed from Python. For the Python documentation, see [`Span`](/api/span). -### Attributes {#span_attributes} +### Attributes {id="span_attributes"} | Name | Description | | ------------ | ----------------------------------------------------------------------------- | @@ -116,7 +116,7 @@ accessed from Python. For the Python documentation, see [`Span`](/api/span). | `end_char` | The index of the last character of the span. ~~int~~ | | `label` | A label to attach to the span, e.g. for named entities. ~~attr_t (uint64_t)~~ | -## Lexeme {#lexeme tag="cdef class" source="spacy/lexeme.pxd"} +## Lexeme {id="lexeme",tag="cdef class",source="spacy/lexeme.pxd"} A Cython class providing access and methods for an entry in the vocabulary. @@ -127,7 +127,7 @@ accessed from Python. For the Python documentation, see [`Lexeme`](/api/lexeme). -### Attributes {#lexeme_attributes} +### Attributes {id="lexeme_attributes"} | Name | Description | | ------- | ----------------------------------------------------------------------------- | @@ -135,7 +135,7 @@ accessed from Python. For the Python documentation, see [`Lexeme`](/api/lexeme). | `vocab` | A reference to the shared `Vocab` object. ~~Vocab~~ | | `orth` | ID of the verbatim text content. ~~attr_t (uint64_t)~~ | -## Vocab {#vocab tag="cdef class" source="spacy/vocab.pxd"} +## Vocab {id="vocab",tag="cdef class",source="spacy/vocab.pxd"} A Cython class providing access and methods for a vocabulary and other data shared across a language. @@ -147,7 +147,7 @@ accessed from Python. For the Python documentation, see [`Vocab`](/api/vocab). -### Attributes {#vocab_attributes} +### Attributes {id="vocab_attributes"} | Name | Description | | --------- | ---------------------------------------------------------------------------------------------------------- | @@ -155,7 +155,7 @@ accessed from Python. For the Python documentation, see [`Vocab`](/api/vocab). | `strings` | A `StringStore` that maps string to hash values and vice versa. ~~StringStore~~ | | `length` | The number of entries in the vocabulary. ~~int~~ | -### Vocab.get {#vocab_get tag="method"} +### Vocab.get {id="vocab_get",tag="method"} Retrieve a [`LexemeC*`](/api/cython-structs#lexemec) pointer from the vocabulary. @@ -172,7 +172,7 @@ vocabulary. | `string` | The string of the word to look up. ~~str~~ | | **RETURNS** | The lexeme in the vocabulary. ~~const LexemeC\*~~ | -### Vocab.get_by_orth {#vocab_get_by_orth tag="method"} +### Vocab.get_by_orth {id="vocab_get_by_orth",tag="method"} Retrieve a [`LexemeC*`](/api/cython-structs#lexemec) pointer from the vocabulary. @@ -189,7 +189,7 @@ vocabulary. | `orth` | ID of the verbatim text content. ~~attr_t (uint64_t)~~ | | **RETURNS** | The lexeme in the vocabulary. ~~const LexemeC\*~~ | -## StringStore {#stringstore tag="cdef class" source="spacy/strings.pxd"} +## StringStore {id="stringstore",tag="cdef class",source="spacy/strings.pxd"} A lookup table to retrieve strings by 64-bit hashes. @@ -201,7 +201,7 @@ accessed from Python. For the Python documentation, see -### Attributes {#stringstore_attributes} +### Attributes {id="stringstore_attributes"} | Name | Description | | ------ | ---------------------------------------------------------------------------------------------------------------- | diff --git a/website/docs/api/cython-structs.md b/website/docs/api/cython-structs.mdx similarity index 94% rename from website/docs/api/cython-structs.md rename to website/docs/api/cython-structs.mdx index 4c8514b64..106a27e90 100644 --- a/website/docs/api/cython-structs.md +++ b/website/docs/api/cython-structs.mdx @@ -7,7 +7,7 @@ menu: - ['LexemeC', 'lexemec'] --- -## TokenC {#tokenc tag="C struct" source="spacy/structs.pxd"} +## TokenC {id="tokenc",tag="C struct",source="spacy/structs.pxd"} Cython data container for the `Token` object. @@ -39,7 +39,7 @@ Cython data container for the `Token` object. | `ent_type` | Named entity type. ~~attr_t (uint64_t)~~ | | `ent_id` | ID of the entity the token is an instance of, if any. Currently not used, but potentially for coreference resolution. ~~attr_t (uint64_t)~~ | -### Token.get_struct_attr {#token_get_struct_attr tag="staticmethod, nogil" source="spacy/tokens/token.pxd"} +### Token.get_struct_attr {id="token_get_struct_attr",tag="staticmethod, nogil",source="spacy/tokens/token.pxd"} Get the value of an attribute from the `TokenC` struct by attribute ID. @@ -58,7 +58,7 @@ Get the value of an attribute from the `TokenC` struct by attribute ID. | `feat_name` | The ID of the attribute to look up. The attributes are enumerated in `spacy.typedefs`. ~~attr_id_t~~ | | **RETURNS** | The value of the attribute. ~~attr_t (uint64_t)~~ | -### Token.set_struct_attr {#token_set_struct_attr tag="staticmethod, nogil" source="spacy/tokens/token.pxd"} +### Token.set_struct_attr {id="token_set_struct_attr",tag="staticmethod, nogil",source="spacy/tokens/token.pxd"} Set the value of an attribute of the `TokenC` struct by attribute ID. @@ -78,7 +78,7 @@ Set the value of an attribute of the `TokenC` struct by attribute ID. | `feat_name` | The ID of the attribute to look up. The attributes are enumerated in `spacy.typedefs`. ~~attr_id_t~~ | | `value` | The value to set. ~~attr_t (uint64_t)~~ | -### token_by_start {#token_by_start tag="function" source="spacy/tokens/doc.pxd"} +### token_by_start {id="token_by_start",tag="function",source="spacy/tokens/doc.pxd"} Find a token in a `TokenC*` array by the offset of its first character. @@ -100,7 +100,7 @@ Find a token in a `TokenC*` array by the offset of its first character. | `start_char` | The start index to search for. ~~int~~ | | **RETURNS** | The index of the token in the array or `-1` if not found. ~~int~~ | -### token_by_end {#token_by_end tag="function" source="spacy/tokens/doc.pxd"} +### token_by_end {id="token_by_end",tag="function",source="spacy/tokens/doc.pxd"} Find a token in a `TokenC*` array by the offset of its final character. @@ -122,7 +122,7 @@ Find a token in a `TokenC*` array by the offset of its final character. | `end_char` | The end index to search for. ~~int~~ | | **RETURNS** | The index of the token in the array or `-1` if not found. ~~int~~ | -### set_children_from_heads {#set_children_from_heads tag="function" source="spacy/tokens/doc.pxd"} +### set_children_from_heads {id="set_children_from_heads",tag="function",source="spacy/tokens/doc.pxd"} Set attributes that allow lookup of syntactic children on a `TokenC*` array. This function must be called after making changes to the `TokenC.head` @@ -148,7 +148,7 @@ attribute, in order to make the parse tree navigation consistent. | `tokens` | A `TokenC*` array. ~~const TokenC\*~~ | | `length` | The number of tokens in the array. ~~int~~ | -## LexemeC {#lexemec tag="C struct" source="spacy/structs.pxd"} +## LexemeC {id="lexemec",tag="C struct",source="spacy/structs.pxd"} Struct holding information about a lexical type. `LexemeC` structs are usually owned by the `Vocab`, and accessed through a read-only pointer on the `TokenC` @@ -172,7 +172,7 @@ struct. | `prefix` | Length-N substring from the start of the lexeme. Defaults to `N=1`. ~~attr_t (uint64_t)~~ | | `suffix` | Length-N substring from the end of the lexeme. Defaults to `N=3`. ~~attr_t (uint64_t)~~ | -### Lexeme.get_struct_attr {#lexeme_get_struct_attr tag="staticmethod, nogil" source="spacy/lexeme.pxd"} +### Lexeme.get_struct_attr {id="lexeme_get_struct_attr",tag="staticmethod, nogil",source="spacy/lexeme.pxd"} Get the value of an attribute from the `LexemeC` struct by attribute ID. @@ -192,7 +192,7 @@ Get the value of an attribute from the `LexemeC` struct by attribute ID. | `feat_name` | The ID of the attribute to look up. The attributes are enumerated in `spacy.typedefs`. ~~attr_id_t~~ | | **RETURNS** | The value of the attribute. ~~attr_t (uint64_t)~~ | -### Lexeme.set_struct_attr {#lexeme_set_struct_attr tag="staticmethod, nogil" source="spacy/lexeme.pxd"} +### Lexeme.set_struct_attr {id="lexeme_set_struct_attr",tag="staticmethod, nogil",source="spacy/lexeme.pxd"} Set the value of an attribute of the `LexemeC` struct by attribute ID. @@ -212,7 +212,7 @@ Set the value of an attribute of the `LexemeC` struct by attribute ID. | `feat_name` | The ID of the attribute to look up. The attributes are enumerated in `spacy.typedefs`. ~~attr_id_t~~ | | `value` | The value to set. ~~attr_t (uint64_t)~~ | -### Lexeme.c_check_flag {#lexeme_c_check_flag tag="staticmethod, nogil" source="spacy/lexeme.pxd"} +### Lexeme.c_check_flag {id="lexeme_c_check_flag",tag="staticmethod, nogil",source="spacy/lexeme.pxd"} Check the value of a binary flag attribute. @@ -232,7 +232,7 @@ Check the value of a binary flag attribute. | `flag_id` | The ID of the flag to look up. The flag IDs are enumerated in `spacy.typedefs`. ~~attr_id_t~~ | | **RETURNS** | The boolean value of the flag. ~~bint~~ | -### Lexeme.c_set_flag {#lexeme_c_set_flag tag="staticmethod, nogil" source="spacy/lexeme.pxd"} +### Lexeme.c_set_flag {id="lexeme_c_set_flag",tag="staticmethod, nogil",source="spacy/lexeme.pxd"} Set the value of a binary flag attribute. diff --git a/website/docs/api/cython.md b/website/docs/api/cython.mdx similarity index 99% rename from website/docs/api/cython.md rename to website/docs/api/cython.mdx index 16b11cead..764ff10f4 100644 --- a/website/docs/api/cython.md +++ b/website/docs/api/cython.mdx @@ -6,7 +6,7 @@ menu: - ['Conventions', 'conventions'] --- -## Overview {#overview hidden="true"} +## Overview {id="overview",hidden="true"} > #### What's Cython? > @@ -37,7 +37,7 @@ class holds a [`LexemeC`](/api/cython-structs#lexemec) struct, at `Lexeme.c`. This lets you shed the Python container, and pass a pointer to the underlying data into C-level functions. -## Conventions {#conventions} +## Conventions {id="conventions"} spaCy's core data structures are implemented as [Cython](http://cython.org/) `cdef` classes. Memory is managed through the diff --git a/website/docs/api/data-formats.md b/website/docs/api/data-formats.mdx similarity index 98% rename from website/docs/api/data-formats.md rename to website/docs/api/data-formats.mdx index 768844cf3..c9d88f87c 100644 --- a/website/docs/api/data-formats.md +++ b/website/docs/api/data-formats.mdx @@ -14,7 +14,7 @@ vocabulary data. For an overview of label schemes used by the models, see the [models directory](/models). Each trained pipeline documents the label schemes used in its components, depending on the data it was trained on. -## Training config {#config new="3"} +## Training config {id="config",version="3"} Config files define the training process and pipeline and can be passed to [`spacy train`](/api/cli#train). They use @@ -52,7 +52,7 @@ your config and check that it's valid, you can run the -### nlp {#config-nlp tag="section"} +### nlp {id="config-nlp",tag="section"} > #### Example > @@ -83,7 +83,7 @@ Defines the `nlp` object, its tokenizer and | `tokenizer` | The tokenizer to use. Defaults to [`Tokenizer`](/api/tokenizer). ~~Callable[[str], Doc]~~ | | `batch_size` | Default batch size for [`Language.pipe`](/api/language#pipe) and [`Language.evaluate`](/api/language#evaluate). ~~int~~ | -### components {#config-components tag="section"} +### components {id="config-components",tag="section"} > #### Example > @@ -106,7 +106,7 @@ function to use to create component) or a `source` (name of path of trained pipeline to copy components from). See the docs on [defining pipeline components](/usage/training#config-components) for details. -### paths, system {#config-variables tag="variables"} +### paths, system {id="config-variables",tag="variables"} These sections define variables that can be referenced across the other sections as variables. For example `${paths.train}` uses the value of `train` defined in @@ -116,11 +116,11 @@ need paths, you can define them here. All config values can also be [`spacy train`](/api/cli#train), which is especially relevant for data paths that you don't want to hard-code in your config file. -```cli +```bash $ python -m spacy train config.cfg --paths.train ./corpus/train.spacy ``` -### corpora {#config-corpora tag="section"} +### corpora {id="config-corpora",tag="section"} > #### Example > @@ -176,7 +176,7 @@ single corpus once and then divide it up into `train` and `dev` partitions. | --------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | | `corpora` | A dictionary keyed by string names, mapped to corpus functions that receive the current `nlp` object and return an iterator of [`Example`](/api/example) objects. ~~Dict[str, Callable[[Language], Iterator[Example]]]~~ | -### training {#config-training tag="section"} +### training {id="config-training",tag="section"} This section defines settings and controls for the training and evaluation process that are used when you run [`spacy train`](/api/cli#train). @@ -186,7 +186,7 @@ process that are used when you run [`spacy train`](/api/cli#train). | `accumulate_gradient` | Whether to divide the batch up into substeps. Defaults to `1`. ~~int~~ | | `batcher` | Callable that takes an iterator of [`Doc`](/api/doc) objects and yields batches of `Doc`s. Defaults to [`batch_by_words`](/api/top-level#batch_by_words). ~~Callable[[Iterator[Doc], Iterator[List[Doc]]]]~~ | | `before_to_disk` | Optional callback to modify `nlp` object right before it is saved to disk during and after training. Can be used to remove or reset config values or disable components. Defaults to `null`. ~~Optional[Callable[[Language], Language]]~~ | -| `before_update` | Optional callback that is invoked at the start of each training step with the `nlp` object and a `Dict` containing the following entries: `step`, `epoch`. Can be used to make deferred changes to components. Defaults to `null`. ~~Optional[Callable[[Language, Dict[str, Any]], None]]~~ | +| `before_update` 3.5 | Optional callback that is invoked at the start of each training step with the `nlp` object and a `Dict` containing the following entries: `step`, `epoch`. Can be used to make deferred changes to components. Defaults to `null`. ~~Optional[Callable[[Language, Dict[str, Any]], None]]~~ | | `dev_corpus` | Dot notation of the config location defining the dev corpus. Defaults to `corpora.dev`. ~~str~~ | | `dropout` | The dropout rate. Defaults to `0.1`. ~~float~~ | | `eval_frequency` | How often to evaluate during training (steps). Defaults to `200`. ~~int~~ | @@ -202,7 +202,7 @@ process that are used when you run [`spacy train`](/api/cli#train). | `seed` | The random seed. Defaults to variable `${system.seed}`. ~~int~~ | | `train_corpus` | Dot notation of the config location defining the train corpus. Defaults to `corpora.train`. ~~str~~ | -### pretraining {#config-pretraining tag="section,optional"} +### pretraining {id="config-pretraining",tag="section,optional"} This section is optional and defines settings and controls for [language model pretraining](/usage/embeddings-transformers#pretraining). It's @@ -220,7 +220,7 @@ used when you run [`spacy pretrain`](/api/cli#pretrain). | `component` | Component name to identify the layer with the model to pretrain. Defaults to `"tok2vec"`. ~~str~~ | | `layer` | The specific layer of the model to pretrain. If empty, the whole model will be used. ~~str~~ | -### initialize {#config-initialize tag="section"} +### initialize {id="config-initialize",tag="section"} This config block lets you define resources for **initializing the pipeline**. It's used by [`Language.initialize`](/api/language#initialize) and typically @@ -255,9 +255,9 @@ Also see the usage guides on the | `vectors` | Name or path of pipeline containing pretrained word vectors to use, e.g. created with [`init vectors`](/api/cli#init-vectors). Defaults to `null`. ~~Optional[str]~~ | | `vocab_data` | Path to JSONL-formatted [vocabulary file](/api/data-formats#vocab-jsonl) to initialize vocabulary. ~~Optional[str]~~ | -## Training data {#training} +## Training data {id="training"} -### Binary training format {#binary-training new="3"} +### Binary training format {id="binary-training",version="3"} > #### Example > @@ -288,7 +288,7 @@ Note that while this is the format used to save training data, you do not have to understand the internal details to use it or create training data. See the section on [preparing training data](/usage/training#training-data). -### JSON training format {#json-input tag="deprecated"} +### JSON training format {id="json-input",tag="deprecated"} @@ -300,7 +300,7 @@ objects to JSON, you can now serialize them directly using the [`spacy convert`](/api/cli) lets you convert your JSON data to the new `.spacy` format: -```cli +```bash $ python -m spacy convert ./data.json . ``` @@ -317,8 +317,7 @@ $ python -m spacy convert ./data.json . > [`offsets_to_biluo_tags`](/api/top-level#offsets_to_biluo_tags) function can > help you convert entity offsets to the right format. -```python -### Example structure +```python {title="Example structure"} [{ "id": int, # ID of the document within the corpus "paragraphs": [{ # list of paragraphs in the corpus @@ -357,7 +356,7 @@ https://github.com/explosion/spaCy/blob/v2.3.x/examples/training/training-data.j -### Annotation format for creating training examples {#dict-input} +### Annotation format for creating training examples {id="dict-input"} An [`Example`](/api/example) object holds the information for one training instance. It stores two [`Doc`](/api/doc) objects: one for holding the @@ -436,8 +435,7 @@ file to keep track of your settings and hyperparameters and your own -```python -### Examples +```python {title="Examples"} # Training data for a part-of-speech tagger doc = Doc(vocab, words=["I", "like", "stuff"]) gold_dict = {"tags": ["NOUN", "VERB", "NOUN"]} @@ -466,7 +464,7 @@ gold_dict = {"entities": [(0, 12, "PERSON")], example = Example.from_dict(doc, gold_dict) ``` -## Lexical data for vocabulary {#vocab-jsonl new="2"} +## Lexical data for vocabulary {id="vocab-jsonl",version="2"} This data file can be provided via the `vocab_data` setting in the `[initialize]` block of the training config to pre-define the lexical data to @@ -483,13 +481,11 @@ spaCy's [`Lexeme`](/api/lexeme#attributes) object. > vocab_data = "/path/to/vocab-data.jsonl" > ``` -```python -### First line +```python {title="First line"} {"lang": "en", "settings": {"oov_prob": -20.502029418945312}} ``` -```python -### Entry structure +```python {title="Entry structure"} { "orth": string, # the word text "id": int, # can correspond to row in vectors table @@ -526,7 +522,7 @@ Here's an example of the 20 most frequent lexemes in the English training data: %%GITHUB_SPACY/extra/example_data/vocab-data.jsonl ``` -## Pipeline meta {#meta} +## Pipeline meta {id="meta"} The pipeline meta is available as the file `meta.json` and exported automatically when you save an `nlp` object to disk. Its contents are available diff --git a/website/docs/api/dependencymatcher.md b/website/docs/api/dependencymatcher.mdx similarity index 69% rename from website/docs/api/dependencymatcher.md rename to website/docs/api/dependencymatcher.mdx index cae4221bf..d0971da55 100644 --- a/website/docs/api/dependencymatcher.md +++ b/website/docs/api/dependencymatcher.mdx @@ -2,7 +2,7 @@ title: DependencyMatcher teaser: Match subtrees within a dependency parse tag: class -new: 3 +version: 3 source: spacy/matcher/dependencymatcher.pyx --- @@ -14,7 +14,7 @@ It requires a pretrained [`DependencyParser`](/api/parser) or other component that sets the `Token.dep` and `Token.head` attributes. See the [usage guide](/usage/rule-based-matching#dependencymatcher) for examples. -## Pattern format {#patterns} +## Pattern format {id="patterns"} > ```python > ### Example @@ -62,33 +62,36 @@ of relations, see the usage guide on -### Operators {#operators} +### Operators {id="operators"} The following operators are supported by the `DependencyMatcher`, most of which come directly from [Semgrex](https://nlp.stanford.edu/nlp/javadoc/javanlp/edu/stanford/nlp/semgraph/semgrex/SemgrexPattern.html): -| Symbol | Description | -| --------- | -------------------------------------------------------------------------------------------------------------------- | -| `A < B` | `A` is the immediate dependent of `B`. | -| `A > B` | `A` is the immediate head of `B`. | -| `A << B` | `A` is the dependent in a chain to `B` following dep → head paths. | -| `A >> B` | `A` is the head in a chain to `B` following head → dep paths. | -| `A . B` | `A` immediately precedes `B`, i.e. `A.i == B.i - 1`, and both are within the same dependency tree. | -| `A .* B` | `A` precedes `B`, i.e. `A.i < B.i`, and both are within the same dependency tree _(not in Semgrex)_. | -| `A ; B` | `A` immediately follows `B`, i.e. `A.i == B.i + 1`, and both are within the same dependency tree _(not in Semgrex)_. | -| `A ;* B` | `A` follows `B`, i.e. `A.i > B.i`, and both are within the same dependency tree _(not in Semgrex)_. | -| `A $+ B` | `B` is a right immediate sibling of `A`, i.e. `A` and `B` have the same parent and `A.i == B.i - 1`. | -| `A $- B` | `B` is a left immediate sibling of `A`, i.e. `A` and `B` have the same parent and `A.i == B.i + 1`. | -| `A $++ B` | `B` is a right sibling of `A`, i.e. `A` and `B` have the same parent and `A.i < B.i`. | -| `A $-- B` | `B` is a left sibling of `A`, i.e. `A` and `B` have the same parent and `A.i > B.i`. | -| `A >++ B` | `B` is a right child of `A`, i.e. `A` is a parent of `B` and `A.i < B.i` _(not in Semgrex)_. | -| `A >-- B` | `B` is a left child of `A`, i.e. `A` is a parent of `B` and `A.i > B.i` _(not in Semgrex)_. | -| `A <++ B` | `B` is a right parent of `A`, i.e. `A` is a child of `B` and `A.i < B.i` _(not in Semgrex)_. | -| `A <-- B` | `B` is a left parent of `A`, i.e. `A` is a child of `B` and `A.i > B.i` _(not in Semgrex)_. | +| Symbol | Description | +| --------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------- | +| `A < B` | `A` is the immediate dependent of `B`. | +| `A > B` | `A` is the immediate head of `B`. | +| `A << B` | `A` is the dependent in a chain to `B` following dep → head paths. | +| `A >> B` | `A` is the head in a chain to `B` following head → dep paths. | +| `A . B` | `A` immediately precedes `B`, i.e. `A.i == B.i - 1`, and both are within the same dependency tree. | +| `A .* B` | `A` precedes `B`, i.e. `A.i < B.i`, and both are within the same dependency tree _(Semgrex counterpart: `..`)_. | +| `A ; B` | `A` immediately follows `B`, i.e. `A.i == B.i + 1`, and both are within the same dependency tree _(Semgrex counterpart: `-`)_. | +| `A ;* B` | `A` follows `B`, i.e. `A.i > B.i`, and both are within the same dependency tree _(Semgrex counterpart: `--`)_. | +| `A $+ B` | `B` is a right immediate sibling of `A`, i.e. `A` and `B` have the same parent and `A.i == B.i - 1`. | +| `A $- B` | `B` is a left immediate sibling of `A`, i.e. `A` and `B` have the same parent and `A.i == B.i + 1`. | +| `A $++ B` | `B` is a right sibling of `A`, i.e. `A` and `B` have the same parent and `A.i < B.i`. | +| `A $-- B` | `B` is a left sibling of `A`, i.e. `A` and `B` have the same parent and `A.i > B.i`. | +| `A >+ B` 3.5.1 | `B` is a right immediate child of `A`, i.e. `A` is a parent of `B` and `A.i == B.i - 1` _(not in Semgrex)_. | +| `A >- B` 3.5.1 | `B` is a left immediate child of `A`, i.e. `A` is a parent of `B` and `A.i == B.i + 1` _(not in Semgrex)_. | +| `A >++ B` | `B` is a right child of `A`, i.e. `A` is a parent of `B` and `A.i < B.i`. | +| `A >-- B` | `B` is a left child of `A`, i.e. `A` is a parent of `B` and `A.i > B.i`. | +| `A <+ B` 3.5.1 | `B` is a right immediate parent of `A`, i.e. `A` is a child of `B` and `A.i == B.i - 1` _(not in Semgrex)_. | +| `A <- B` 3.5.1 | `B` is a left immediate parent of `A`, i.e. `A` is a child of `B` and `A.i == B.i + 1` _(not in Semgrex)_. | +| `A <++ B` | `B` is a right parent of `A`, i.e. `A` is a child of `B` and `A.i < B.i`. | +| `A <-- B` | `B` is a left parent of `A`, i.e. `A` is a child of `B` and `A.i > B.i`. | - -## DependencyMatcher.\_\_init\_\_ {#init tag="method"} +## DependencyMatcher.\_\_init\_\_ {id="init",tag="method"} Create a `DependencyMatcher`. @@ -105,7 +108,7 @@ Create a `DependencyMatcher`. | _keyword-only_ | | | `validate` | Validate all patterns added to this matcher. ~~bool~~ | -## DependencyMatcher.\_\call\_\_ {#call tag="method"} +## DependencyMatcher.\_\_call\_\_ {id="call",tag="method"} Find all tokens matching the supplied patterns on the `Doc` or `Span`. @@ -127,7 +130,7 @@ Find all tokens matching the supplied patterns on the `Doc` or `Span`. | `doclike` | The `Doc` or `Span` to match over. ~~Union[Doc, Span]~~ | | **RETURNS** | A list of `(match_id, token_ids)` tuples, describing the matches. The `match_id` is the ID of the match pattern and `token_ids` is a list of token indices matched by the pattern, where the position of each token in the list corresponds to the position of the node specification in the pattern. ~~List[Tuple[int, List[int]]]~~ | -## DependencyMatcher.\_\_len\_\_ {#len tag="method"} +## DependencyMatcher.\_\_len\_\_ {id="len",tag="method"} Get the number of rules added to the dependency matcher. Note that this only returns the number of rules (identical with the number of IDs), not the number @@ -148,7 +151,7 @@ of individual patterns. | ----------- | ---------------------------- | | **RETURNS** | The number of rules. ~~int~~ | -## DependencyMatcher.\_\_contains\_\_ {#contains tag="method"} +## DependencyMatcher.\_\_contains\_\_ {id="contains",tag="method"} Check whether the matcher contains rules for a match ID. @@ -166,7 +169,7 @@ Check whether the matcher contains rules for a match ID. | `key` | The match ID. ~~str~~ | | **RETURNS** | Whether the matcher contains rules for this match ID. ~~bool~~ | -## DependencyMatcher.add {#add tag="method"} +## DependencyMatcher.add {id="add",tag="method"} Add a rule to the matcher, consisting of an ID key, one or more patterns, and an optional callback function to act on the matches. The callback function will @@ -191,7 +194,7 @@ will be overwritten. | _keyword-only_ | | | `on_match` | Callback function to act on matches. Takes the arguments `matcher`, `doc`, `i` and `matches`. ~~Optional[Callable[[DependencyMatcher, Doc, int, List[Tuple], Any]]~~ | -## DependencyMatcher.get {#get tag="method"} +## DependencyMatcher.get {id="get",tag="method"} Retrieve the pattern stored for a key. Returns the rule as an `(on_match, patterns)` tuple containing the callback and available patterns. @@ -208,7 +211,7 @@ Retrieve the pattern stored for a key. Returns the rule as an | `key` | The ID of the match rule. ~~str~~ | | **RETURNS** | The rule, as an `(on_match, patterns)` tuple. ~~Tuple[Optional[Callable], List[List[Union[Dict, Tuple]]]]~~ | -## DependencyMatcher.remove {#remove tag="method"} +## DependencyMatcher.remove {id="remove",tag="method"} Remove a rule from the dependency matcher. A `KeyError` is raised if the match ID does not exist. diff --git a/website/docs/api/dependencyparser.md b/website/docs/api/dependencyparser.mdx similarity index 95% rename from website/docs/api/dependencyparser.md rename to website/docs/api/dependencyparser.mdx index 27e315592..a6bc48cdf 100644 --- a/website/docs/api/dependencyparser.md +++ b/website/docs/api/dependencyparser.mdx @@ -25,7 +25,7 @@ current state. The weights are updated such that the scores assigned to the set of optimal actions is increased, while scores assigned to other actions are decreased. Note that more than one action may be optimal for a given state. -## Assigned Attributes {#assigned-attributes} +## Assigned Attributes {id="assigned-attributes"} Dependency predictions are assigned to the `Token.dep` and `Token.head` fields. Beside the dependencies themselves, the parser decides sentence boundaries, @@ -39,7 +39,7 @@ which are saved in `Token.is_sent_start` and accessible via `Doc.sents`. | `Token.is_sent_start` | A boolean value indicating whether the token starts a sentence. After the parser runs this will be `True` or `False` for all tokens. ~~bool~~ | | `Doc.sents` | An iterator over sentences in the `Doc`, determined by `Token.is_sent_start` values. ~~Iterator[Span]~~ | -## Config and implementation {#config} +## Config and implementation {id="config"} The default config is defined by the pipeline component factory and describes how the component should be configured. You can override its settings via the @@ -74,7 +74,7 @@ architectures and their arguments and hyperparameters. %%GITHUB_SPACY/spacy/pipeline/dep_parser.pyx ``` -## DependencyParser.\_\_init\_\_ {#init tag="method"} +## DependencyParser.\_\_init\_\_ {id="init",tag="method"} > #### Example > @@ -107,7 +107,7 @@ shortcut for this and instantiate the component using its string name and | `min_action_freq` | The minimum frequency of labelled actions to retain. Rarer labelled actions have their label backed-off to "dep". While this primarily affects the label accuracy, it can also affect the attachment structure, as the labels are used to represent the pseudo-projectivity transformation. ~~int~~ | | `scorer` | The scoring method. Defaults to [`Scorer.score_deps`](/api/scorer#score_deps) for the attribute `"dep"` ignoring the labels `p` and `punct` and [`Scorer.score_spans`](/api/scorer/#score_spans) for the attribute `"sents"`. ~~Optional[Callable]~~ | -## DependencyParser.\_\_call\_\_ {#call tag="method"} +## DependencyParser.\_\_call\_\_ {id="call",tag="method"} Apply the pipe to one document. The document is modified in place, and returned. This usually happens under the hood when the `nlp` object is called on a text @@ -131,7 +131,7 @@ and all pipeline components are applied to the `Doc` in order. Both | `doc` | The document to process. ~~Doc~~ | | **RETURNS** | The processed document. ~~Doc~~ | -## DependencyParser.pipe {#pipe tag="method"} +## DependencyParser.pipe {id="pipe",tag="method"} Apply the pipe to a stream of documents. This usually happens under the hood when the `nlp` object is called on a text and all pipeline components are @@ -155,7 +155,7 @@ applied to the `Doc` in order. Both [`__call__`](/api/dependencyparser#call) and | `batch_size` | The number of documents to buffer. Defaults to `128`. ~~int~~ | | **YIELDS** | The processed documents in order. ~~Doc~~ | -## DependencyParser.initialize {#initialize tag="method" new="3"} +## DependencyParser.initialize {id="initialize",tag="method",version="3"} Initialize the component for training. `get_examples` should be a function that returns an iterable of [`Example`](/api/example) objects. **At least one example @@ -198,7 +198,7 @@ This method was previously called `begin_training`. | `nlp` | The current `nlp` object. Defaults to `None`. ~~Optional[Language]~~ | | `labels` | The label information to add to the component, as provided by the [`label_data`](#label_data) property after initialization. To generate a reusable JSON file from your data, you should run the [`init labels`](/api/cli#init-labels) command. If no labels are provided, the `get_examples` callback is used to extract the labels from the data, which may be a lot slower. ~~Optional[Dict[str, Dict[str, int]]]~~ | -## DependencyParser.predict {#predict tag="method"} +## DependencyParser.predict {id="predict",tag="method"} Apply the component's model to a batch of [`Doc`](/api/doc) objects, without modifying them. @@ -215,7 +215,7 @@ modifying them. | `docs` | The documents to predict. ~~Iterable[Doc]~~ | | **RETURNS** | A helper class for the parse state (internal). ~~StateClass~~ | -## DependencyParser.set_annotations {#set_annotations tag="method"} +## DependencyParser.set_annotations {id="set_annotations",tag="method"} Modify a batch of [`Doc`](/api/doc) objects, using pre-computed scores. @@ -232,7 +232,7 @@ Modify a batch of [`Doc`](/api/doc) objects, using pre-computed scores. | `docs` | The documents to modify. ~~Iterable[Doc]~~ | | `scores` | The scores to set, produced by `DependencyParser.predict`. Returns an internal helper class for the parse state. ~~List[StateClass]~~ | -## DependencyParser.update {#update tag="method"} +## DependencyParser.update {id="update",tag="method"} Learn from a batch of [`Example`](/api/example) objects, updating the pipe's model. Delegates to [`predict`](/api/dependencyparser#predict) and @@ -255,7 +255,7 @@ model. Delegates to [`predict`](/api/dependencyparser#predict) and | `losses` | Optional record of the loss during training. Updated using the component name as the key. ~~Optional[Dict[str, float]]~~ | | **RETURNS** | The updated `losses` dictionary. ~~Dict[str, float]~~ | -## DependencyParser.get_loss {#get_loss tag="method"} +## DependencyParser.get_loss {id="get_loss",tag="method"} Find the loss and gradient of loss for the batch of documents and their predicted scores. @@ -274,7 +274,7 @@ predicted scores. | `scores` | Scores representing the model's predictions. ~~StateClass~~ | | **RETURNS** | The loss and the gradient, i.e. `(loss, gradient)`. ~~Tuple[float, float]~~ | -## DependencyParser.create_optimizer {#create_optimizer tag="method"} +## DependencyParser.create_optimizer {id="create_optimizer",tag="method"} Create an [`Optimizer`](https://thinc.ai/docs/api-optimizers) for the pipeline component. @@ -290,7 +290,7 @@ component. | ----------- | ---------------------------- | | **RETURNS** | The optimizer. ~~Optimizer~~ | -## DependencyParser.use_params {#use_params tag="method, contextmanager"} +## DependencyParser.use_params {id="use_params",tag="method, contextmanager"} Modify the pipe's model, to use the given parameter values. At the end of the context, the original parameters are restored. @@ -307,7 +307,7 @@ context, the original parameters are restored. | -------- | -------------------------------------------------- | | `params` | The parameter values to use in the model. ~~dict~~ | -## DependencyParser.add_label {#add_label tag="method"} +## DependencyParser.add_label {id="add_label",tag="method"} Add a new label to the pipe. Note that you don't have to call this method if you provide a **representative data sample** to the [`initialize`](#initialize) @@ -327,7 +327,7 @@ to the model, and the output dimension will be | `label` | The label to add. ~~str~~ | | **RETURNS** | `0` if the label is already present, otherwise `1`. ~~int~~ | -## DependencyParser.set_output {#set_output tag="method"} +## DependencyParser.set_output {id="set_output",tag="method"} Change the output dimension of the component's model by calling the model's attribute `resize_output`. This is a function that takes the original model and @@ -346,7 +346,7 @@ forgetting" problem. | ---- | --------------------------------- | | `nO` | The new output dimension. ~~int~~ | -## DependencyParser.to_disk {#to_disk tag="method"} +## DependencyParser.to_disk {id="to_disk",tag="method"} Serialize the pipe to disk. @@ -363,7 +363,7 @@ Serialize the pipe to disk. | _keyword-only_ | | | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | -## DependencyParser.from_disk {#from_disk tag="method"} +## DependencyParser.from_disk {id="from_disk",tag="method"} Load the pipe from disk. Modifies the object in place and returns it. @@ -381,7 +381,7 @@ Load the pipe from disk. Modifies the object in place and returns it. | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | | **RETURNS** | The modified `DependencyParser` object. ~~DependencyParser~~ | -## DependencyParser.to_bytes {#to_bytes tag="method"} +## DependencyParser.to_bytes {id="to_bytes",tag="method"} > #### Example > @@ -398,7 +398,7 @@ Serialize the pipe to a bytestring. | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | | **RETURNS** | The serialized form of the `DependencyParser` object. ~~bytes~~ | -## DependencyParser.from_bytes {#from_bytes tag="method"} +## DependencyParser.from_bytes {id="from_bytes",tag="method"} Load the pipe from a bytestring. Modifies the object in place and returns it. @@ -417,7 +417,7 @@ Load the pipe from a bytestring. Modifies the object in place and returns it. | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | | **RETURNS** | The `DependencyParser` object. ~~DependencyParser~~ | -## DependencyParser.labels {#labels tag="property"} +## DependencyParser.labels {id="labels",tag="property"} The labels currently added to the component. @@ -432,7 +432,7 @@ The labels currently added to the component. | ----------- | ------------------------------------------------------ | | **RETURNS** | The labels added to the component. ~~Tuple[str, ...]~~ | -## DependencyParser.label_data {#label_data tag="property" new="3"} +## DependencyParser.label_data {id="label_data",tag="property",version="3"} The labels currently added to the component and their internal meta information. This is the data generated by [`init labels`](/api/cli#init-labels) and used by @@ -450,7 +450,7 @@ the model with a pre-defined label set. | ----------- | ------------------------------------------------------------------------------- | | **RETURNS** | The label data added to the component. ~~Dict[str, Dict[str, Dict[str, int]]]~~ | -## Serialization fields {#serialization-fields} +## Serialization fields {id="serialization-fields"} During serialization, spaCy will export several data fields used to restore different aspects of the object. If needed, you can exclude them from diff --git a/website/docs/api/doc.md b/website/docs/api/doc.mdx similarity index 90% rename from website/docs/api/doc.md rename to website/docs/api/doc.mdx index 090489d83..0a5826500 100644 --- a/website/docs/api/doc.md +++ b/website/docs/api/doc.mdx @@ -12,7 +12,7 @@ compressed binary strings. The `Doc` object holds an array of [`Span`](/api/span) objects are views of this array, i.e. they don't own the data themselves. -## Doc.\_\_init\_\_ {#init tag="method"} +## Doc.\_\_init\_\_ {id="init",tag="method"} Construct a `Doc` object. The most common way to get a `Doc` object is via the `nlp` object. @@ -37,7 +37,7 @@ Construct a `Doc` object. The most common way to get a `Doc` object is via the | `words` | A list of strings or integer hash values to add to the document as words. ~~Optional[List[Union[str,int]]]~~ | | `spaces` | A list of boolean values indicating whether each word has a subsequent space. Must have the same length as `words`, if specified. Defaults to a sequence of `True`. ~~Optional[List[bool]]~~ | | _keyword-only_ | | -| `user\_data` | Optional extra data to attach to the Doc. ~~Dict~~ | +| `user_data` | Optional extra data to attach to the Doc. ~~Dict~~ | | `tags` 3 | A list of strings, of the same length as `words`, to assign as `token.tag` for each word. Defaults to `None`. ~~Optional[List[str]]~~ | | `pos` 3 | A list of strings, of the same length as `words`, to assign as `token.pos` for each word. Defaults to `None`. ~~Optional[List[str]]~~ | | `morphs` 3 | A list of strings, of the same length as `words`, to assign as `token.morph` for each word. Defaults to `None`. ~~Optional[List[str]]~~ | @@ -47,7 +47,7 @@ Construct a `Doc` object. The most common way to get a `Doc` object is via the | `sent_starts` 3 | A list of values, of the same length as `words`, to assign as `token.is_sent_start`. Will be overridden by heads if `heads` is provided. Defaults to `None`. ~~Optional[List[Union[bool, int, None]]]~~ | | `ents` 3 | A list of strings, of the same length of `words`, to assign the token-based IOB tag. Defaults to `None`. ~~Optional[List[str]]~~ | -## Doc.\_\_getitem\_\_ {#getitem tag="method"} +## Doc.\_\_getitem\_\_ {id="getitem",tag="method"} Get a [`Token`](/api/token) object at position `i`, where `i` is an integer. Negative indexing is supported, and follows the usual Python semantics, i.e. @@ -80,7 +80,7 @@ semantics. | `start_end` | The slice of the document to get. ~~Tuple[int, int]~~ | | **RETURNS** | The span at `doc[start:end]`. ~~Span~~ | -## Doc.\_\_iter\_\_ {#iter tag="method"} +## Doc.\_\_iter\_\_ {id="iter",tag="method"} Iterate over `Token` objects, from which the annotations can be easily accessed. @@ -100,7 +100,7 @@ underlying C data directly from Cython. | ---------- | --------------------------- | | **YIELDS** | A `Token` object. ~~Token~~ | -## Doc.\_\_len\_\_ {#len tag="method"} +## Doc.\_\_len\_\_ {id="len",tag="method"} Get the number of tokens in the document. @@ -115,7 +115,7 @@ Get the number of tokens in the document. | ----------- | --------------------------------------------- | | **RETURNS** | The number of tokens in the document. ~~int~~ | -## Doc.set_extension {#set_extension tag="classmethod" new="2"} +## Doc.set_extension {id="set_extension",tag="classmethod",version="2"} Define a custom attribute on the `Doc` which becomes available via `Doc._`. For details, see the documentation on @@ -140,7 +140,7 @@ details, see the documentation on | `setter` | Setter function that takes the `Doc` and a value, and modifies the object. Is called when the user writes to the `Doc._` attribute. ~~Optional[Callable[[Doc, Any], None]]~~ | | `force` | Force overwriting existing attribute. ~~bool~~ | -## Doc.get_extension {#get_extension tag="classmethod" new="2"} +## Doc.get_extension {id="get_extension",tag="classmethod",version="2"} Look up a previously registered extension by name. Returns a 4-tuple `(default, method, getter, setter)` if the extension is registered. Raises a @@ -160,7 +160,7 @@ Look up a previously registered extension by name. Returns a 4-tuple | `name` | Name of the extension. ~~str~~ | | **RETURNS** | A `(default, method, getter, setter)` tuple of the extension. ~~Tuple[Optional[Any], Optional[Callable], Optional[Callable], Optional[Callable]]~~ | -## Doc.has_extension {#has_extension tag="classmethod" new="2"} +## Doc.has_extension {id="has_extension",tag="classmethod",version="2"} Check whether an extension has been registered on the `Doc` class. @@ -177,7 +177,7 @@ Check whether an extension has been registered on the `Doc` class. | `name` | Name of the extension to check. ~~str~~ | | **RETURNS** | Whether the extension has been registered. ~~bool~~ | -## Doc.remove_extension {#remove_extension tag="classmethod" new="2.0.12"} +## Doc.remove_extension {id="remove_extension",tag="classmethod",version="2.0.12"} Remove a previously registered extension. @@ -195,7 +195,7 @@ Remove a previously registered extension. | `name` | Name of the extension. ~~str~~ | | **RETURNS** | A `(default, method, getter, setter)` tuple of the removed extension. ~~Tuple[Optional[Any], Optional[Callable], Optional[Callable], Optional[Callable]]~~ | -## Doc.char_span {#char_span tag="method" new="2"} +## Doc.char_span {id="char_span",tag="method",version="2"} Create a `Span` object from the slice `doc.text[start_idx:end_idx]`. Returns `None` if the character indices don't map to a valid span using the default @@ -209,17 +209,18 @@ alignment mode `"strict". > assert span.text == "New York" > ``` -| Name | Description | -| ---------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `start` | The index of the first character of the span. ~~int~~ | -| `end` | The index of the last character after the span. ~~int~~ | -| `label` | A label to attach to the span, e.g. for named entities. ~~Union[int, str]~~ | -| `kb_id` | An ID from a knowledge base to capture the meaning of a named entity. ~~Union[int, str]~~ | -| `vector` | A meaning representation of the span. ~~numpy.ndarray[ndim=1, dtype=float32]~~ | -| `alignment_mode` | How character indices snap to token boundaries. Options: `"strict"` (no snapping), `"contract"` (span of all tokens completely within the character span), `"expand"` (span of all tokens at least partially covered by the character span). Defaults to `"strict"`. ~~str~~ | -| **RETURNS** | The newly constructed object or `None`. ~~Optional[Span]~~ | +| Name | Description | +| ---------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `start` | The index of the first character of the span. ~~int~~ | +| `end` | The index of the last character after the span. ~~int~~ | +| `label` | A label to attach to the span, e.g. for named entities. ~~Union[int, str]~~ | +| `kb_id` | An ID from a knowledge base to capture the meaning of a named entity. ~~Union[int, str]~~ | +| `vector` | A meaning representation of the span. ~~numpy.ndarray[ndim=1, dtype=float32]~~ | +| `alignment_mode` | How character indices snap to token boundaries. Options: `"strict"` (no snapping), `"contract"` (span of all tokens completely within the character span), `"expand"` (span of all tokens at least partially covered by the character span). Defaults to `"strict"`. ~~str~~ | +| `span_id` 3.3.1 | An identifier to associate with the span. ~~Union[int, str]~~ | +| **RETURNS** | The newly constructed object or `None`. ~~Optional[Span]~~ | -## Doc.set_ents {#set_ents tag="method" new="3"} +## Doc.set_ents {id="set_ents",tag="method",version="3"} Set the named entities in the document. @@ -243,7 +244,7 @@ Set the named entities in the document. | `outside` | Spans outside of entities (O in IOB). ~~Optional[List[Span]]~~ | | `default` | How to set entity annotation for tokens outside of any provided spans. Options: `"blocked"`, `"missing"`, `"outside"` and `"unmodified"` (preserve current state). Defaults to `"outside"`. ~~str~~ | -## Doc.similarity {#similarity tag="method" model="vectors"} +## Doc.similarity {id="similarity",tag="method",model="vectors"} Make a semantic similarity estimate. The default estimate is cosine similarity using an average of word vectors. @@ -263,7 +264,7 @@ using an average of word vectors. | `other` | The object to compare with. By default, accepts `Doc`, `Span`, `Token` and `Lexeme` objects. ~~Union[Doc, Span, Token, Lexeme]~~ | | **RETURNS** | A scalar similarity score. Higher is more similar. ~~float~~ | -## Doc.count_by {#count_by tag="method"} +## Doc.count_by {id="count_by",tag="method"} Count the frequencies of a given attribute. Produces a dict of `{attr (int): count (ints)}` frequencies, keyed by the values of the given @@ -284,7 +285,7 @@ attribute ID. | `attr_id` | The attribute ID. ~~int~~ | | **RETURNS** | A dictionary mapping attributes to integer counts. ~~Dict[int, int]~~ | -## Doc.get_lca_matrix {#get_lca_matrix tag="method"} +## Doc.get_lca_matrix {id="get_lca_matrix",tag="method"} Calculates the lowest common ancestor matrix for a given `Doc`. Returns LCA matrix containing the integer index of the ancestor, or `-1` if no common @@ -302,7 +303,7 @@ ancestor is found, e.g. if span excludes a necessary ancestor. | ----------- | -------------------------------------------------------------------------------------- | | **RETURNS** | The lowest common ancestor matrix of the `Doc`. ~~numpy.ndarray[ndim=2, dtype=int32]~~ | -## Doc.has_annotation {#has_annotation tag="method"} +## Doc.has_annotation {id="has_annotation",tag="method"} Check whether the doc contains annotation on a [`Token` attribute](/api/token#attributes). @@ -327,7 +328,7 @@ doc = nlp("This is a text") | `require_complete` | Whether to check that the attribute is set on every token in the doc. Defaults to `False`. ~~bool~~ | | **RETURNS** | Whether specified annotation is present in the doc. ~~bool~~ | -## Doc.to_array {#to_array tag="method"} +## Doc.to_array {id="to_array",tag="method"} Export given token attributes to a numpy `ndarray`. If `attr_ids` is a sequence of `M` attributes, the output array will be of shape `(N, M)`, where `N` is the @@ -355,7 +356,7 @@ Returns a 2D array with one row per token and one column per attribute (when | `attr_ids` | A list of attributes (int IDs or string names) or a single attribute (int ID or string name). ~~Union[int, str, List[Union[int, str]]]~~ | | **RETURNS** | The exported attributes as a numpy array. ~~Union[numpy.ndarray[ndim=2, dtype=uint64], numpy.ndarray[ndim=1, dtype=uint64]]~~ | -## Doc.from_array {#from_array tag="method"} +## Doc.from_array {id="from_array",tag="method"} Load attributes from a numpy array. Write to a `Doc` object, from an `(M, N)` array of attributes. @@ -379,7 +380,7 @@ array of attributes. | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | | **RETURNS** | The `Doc` itself. ~~Doc~~ | -## Doc.from_docs {#from_docs tag="staticmethod" new="3"} +## Doc.from_docs {id="from_docs",tag="staticmethod",version="3"} Concatenate multiple `Doc` objects to form a new one. Raises an error if the `Doc` objects do not all share the same `Vocab`. @@ -408,7 +409,7 @@ Concatenate multiple `Doc` objects to form a new one. Raises an error if the | `exclude` 3.3 | String names of Doc attributes to exclude. Supported: `spans`, `tensor`, `user_data`. ~~Iterable[str]~~ | | **RETURNS** | The new `Doc` object that is containing the other docs or `None`, if `docs` is empty or `None`. ~~Optional[Doc]~~ | -## Doc.to_disk {#to_disk tag="method" new="2"} +## Doc.to_disk {id="to_disk",tag="method",version="2"} Save the current state to a directory. @@ -424,7 +425,7 @@ Save the current state to a directory. | _keyword-only_ | | | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | -## Doc.from_disk {#from_disk tag="method" new="2"} +## Doc.from_disk {id="from_disk",tag="method",version="2"} Loads state from a directory. Modifies the object in place and returns it. @@ -443,7 +444,7 @@ Loads state from a directory. Modifies the object in place and returns it. | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | | **RETURNS** | The modified `Doc` object. ~~Doc~~ | -## Doc.to_bytes {#to_bytes tag="method"} +## Doc.to_bytes {id="to_bytes",tag="method"} Serialize, i.e. export the document contents to a binary string. @@ -460,7 +461,7 @@ Serialize, i.e. export the document contents to a binary string. | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | | **RETURNS** | A losslessly serialized copy of the `Doc`, including all annotations. ~~bytes~~ | -## Doc.from_bytes {#from_bytes tag="method"} +## Doc.from_bytes {id="from_bytes",tag="method"} Deserialize, i.e. import the document contents from a binary string. @@ -481,7 +482,7 @@ Deserialize, i.e. import the document contents from a binary string. | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | | **RETURNS** | The `Doc` object. ~~Doc~~ | -## Doc.to_json {#to_json tag="method"} +## Doc.to_json {id="to_json",tag="method"} Serializes a document to JSON. Note that this is format differs from the deprecated [`JSON training format`](/api/data-formats#json-input). @@ -498,7 +499,7 @@ deprecated [`JSON training format`](/api/data-formats#json-input). | `underscore` | Optional list of string names of custom `Doc` attributes. Attribute values need to be JSON-serializable. Values will be added to an `"_"` key in the data, e.g. `"_": {"foo": "bar"}`. ~~Optional[List[str]]~~ | | **RETURNS** | The data in JSON format. ~~Dict[str, Any]~~ | -## Doc.from_json {#from_json tag="method" new="3.3.1"} +## Doc.from_json {id="from_json",tag="method",version="3.3.1"} Deserializes a document from JSON, i.e. generates a document from the provided JSON data as generated by [`Doc.to_json()`](/api/doc#to_json). @@ -520,7 +521,7 @@ JSON data as generated by [`Doc.to_json()`](/api/doc#to_json). | `validate` | Whether to validate the JSON input against the expected schema for detailed debugging. Defaults to `False`. ~~bool~~ | | **RETURNS** | A `Doc` corresponding to the provided JSON. ~~Doc~~ | -## Doc.retokenize {#retokenize tag="contextmanager" new="2.1"} +## Doc.retokenize {id="retokenize",tag="contextmanager",version="2.1"} Context manager to handle retokenization of the `Doc`. Modifications to the `Doc`'s tokenization are stored, and then made all at once when the context @@ -540,7 +541,7 @@ invalidated, although they may accidentally continue to work. | ----------- | -------------------------------- | | **RETURNS** | The retokenizer. ~~Retokenizer~~ | -### Retokenizer.merge {#retokenizer.merge tag="method"} +### Retokenizer.merge {id="retokenizer.merge",tag="method"} Mark a span for merging. The `attrs` will be applied to the resulting token (if they're context-dependent token attributes like `LEMMA` or `DEP`) or to the @@ -563,7 +564,7 @@ values. | `span` | The span to merge. ~~Span~~ | | `attrs` | Attributes to set on the merged token. ~~Dict[Union[str, int], Any]~~ | -### Retokenizer.split {#retokenizer.split tag="method"} +### Retokenizer.split {id="retokenizer.split",tag="method"} Mark a token for splitting, into the specified `orths`. The `heads` are required to specify how the new subtokens should be integrated into the dependency tree. @@ -599,7 +600,7 @@ underlying lexeme (if they're context-independent lexical attributes like | `heads` | List of `token` or `(token, subtoken)` tuples specifying the tokens to attach the newly split subtokens to. ~~List[Union[Token, Tuple[Token, int]]]~~ | | `attrs` | Attributes to set on all split tokens. Attribute names mapped to list of per-token attribute values. ~~Dict[Union[str, int], List[Any]]~~ | -## Doc.ents {#ents tag="property" model="NER"} +## Doc.ents {id="ents",tag="property",model="NER"} The named entities in the document. Returns a tuple of named entity `Span` objects, if the entity recognizer has been applied. @@ -617,7 +618,7 @@ objects, if the entity recognizer has been applied. | ----------- | ---------------------------------------------------------------- | | **RETURNS** | Entities in the document, one `Span` per entity. ~~Tuple[Span]~~ | -## Doc.spans {#spans tag="property"} +## Doc.spans {id="spans",tag="property"} A dictionary of named span groups, to store and access additional span annotations. You can write to it by assigning a list of [`Span`](/api/span) @@ -634,7 +635,7 @@ objects or a [`SpanGroup`](/api/spangroup) to a given key. | ----------- | ------------------------------------------------------------------ | | **RETURNS** | The span groups assigned to the document. ~~Dict[str, SpanGroup]~~ | -## Doc.cats {#cats tag="property" model="text classifier"} +## Doc.cats {id="cats",tag="property",model="text classifier"} Maps a label to a score for categories applied to the document. Typically set by the [`TextCategorizer`](/api/textcategorizer). @@ -650,7 +651,7 @@ the [`TextCategorizer`](/api/textcategorizer). | ----------- | ---------------------------------------------------------- | | **RETURNS** | The text categories mapped to scores. ~~Dict[str, float]~~ | -## Doc.noun_chunks {#noun_chunks tag="property" model="parser"} +## Doc.noun_chunks {id="noun_chunks",tag="property",model="parser"} Iterate over the base noun phrases in the document. Yields base noun-phrase `Span` objects, if the document has been syntactically parsed. A base noun @@ -677,7 +678,7 @@ implemented for the given language, a `NotImplementedError` is raised. | ---------- | ------------------------------------- | | **YIELDS** | Noun chunks in the document. ~~Span~~ | -## Doc.sents {#sents tag="property" model="sentences"} +## Doc.sents {id="sents",tag="property",model="sentences"} Iterate over the sentences in the document. Sentence spans have no label. @@ -699,7 +700,7 @@ will raise an error otherwise. | ---------- | ----------------------------------- | | **YIELDS** | Sentences in the document. ~~Span~~ | -## Doc.has_vector {#has_vector tag="property" model="vectors"} +## Doc.has_vector {id="has_vector",tag="property",model="vectors"} A boolean value indicating whether a word vector is associated with the object. @@ -714,7 +715,7 @@ A boolean value indicating whether a word vector is associated with the object. | ----------- | --------------------------------------------------------- | | **RETURNS** | Whether the document has a vector data attached. ~~bool~~ | -## Doc.vector {#vector tag="property" model="vectors"} +## Doc.vector {id="vector",tag="property",model="vectors"} A real-valued meaning representation. Defaults to an average of the token vectors. @@ -731,7 +732,7 @@ vectors. | ----------- | -------------------------------------------------------------------------------------------------- | | **RETURNS** | A 1-dimensional array representing the document's vector. ~~numpy.ndarray[ndim=1, dtype=float32]~~ | -## Doc.vector_norm {#vector_norm tag="property" model="vectors"} +## Doc.vector_norm {id="vector_norm",tag="property",model="vectors"} The L2 norm of the document's vector representation. @@ -749,7 +750,7 @@ The L2 norm of the document's vector representation. | ----------- | --------------------------------------------------- | | **RETURNS** | The L2 norm of the vector representation. ~~float~~ | -## Attributes {#attributes} +## Attributes {id="attributes"} | Name | Description | | -------------------- | ----------------------------------------------------------------------------------------------------------------------------------- | @@ -768,7 +769,7 @@ The L2 norm of the document's vector representation. | `has_unknown_spaces` | Whether the document was constructed without known spacing between tokens (typically when created from gold tokenization). ~~bool~~ | | `_` | User space for adding custom [attribute extensions](/usage/processing-pipelines#custom-components-attributes). ~~Underscore~~ | -## Serialization fields {#serialization-fields} +## Serialization fields {id="serialization-fields"} During serialization, spaCy will export several data fields used to restore different aspects of the object. If needed, you can exclude them from diff --git a/website/docs/api/docbin.md b/website/docs/api/docbin.mdx similarity index 93% rename from website/docs/api/docbin.md rename to website/docs/api/docbin.mdx index b1d1798ba..b5cf29df7 100644 --- a/website/docs/api/docbin.md +++ b/website/docs/api/docbin.mdx @@ -1,7 +1,7 @@ --- title: DocBin tag: class -new: 2.2 +version: 2.2 teaser: Pack Doc objects for binary serialization source: spacy/tokens/_serialize.py --- @@ -15,8 +15,7 @@ notable downside to this format is that you can't easily extract just one document from the `DocBin`. The serialization format is gzipped msgpack, where the msgpack object has the following structure: -```python -### msgpack object structure +```python {title="msgpack object structure"} { "version": str, # DocBin version number "attrs": List[uint64], # e.g. [TAG, HEAD, ENT_IOB, ENT_TYPE] @@ -33,7 +32,7 @@ object. This means the storage is more efficient if you pack more documents together, because you have less duplication in the strings. For usage examples, see the docs on [serializing `Doc` objects](/usage/saving-loading#docs). -## DocBin.\_\_init\_\_ {#init tag="method"} +## DocBin.\_\_init\_\_ {id="init",tag="method"} Create a `DocBin` object to hold serialized annotations. @@ -50,7 +49,7 @@ Create a `DocBin` object to hold serialized annotations. | `store_user_data` | Whether to write the `Doc.user_data` and the values of custom extension attributes to file/bytes. Defaults to `False`. ~~bool~~ | | `docs` | `Doc` objects to add on initialization. ~~Iterable[Doc]~~ | -## DocBin.\_\len\_\_ {#len tag="method"} +## DocBin.\_\_len\_\_ {id="len",tag="method"} Get the number of `Doc` objects that were added to the `DocBin`. @@ -67,7 +66,7 @@ Get the number of `Doc` objects that were added to the `DocBin`. | ----------- | --------------------------------------------------- | | **RETURNS** | The number of `Doc`s added to the `DocBin`. ~~int~~ | -## DocBin.add {#add tag="method"} +## DocBin.add {id="add",tag="method"} Add a `Doc`'s annotations to the `DocBin` for serialization. @@ -83,7 +82,7 @@ Add a `Doc`'s annotations to the `DocBin` for serialization. | -------- | -------------------------------- | | `doc` | The `Doc` object to add. ~~Doc~~ | -## DocBin.get_docs {#get_docs tag="method"} +## DocBin.get_docs {id="get_docs",tag="method"} Recover `Doc` objects from the annotations, using the given vocab. @@ -98,7 +97,7 @@ Recover `Doc` objects from the annotations, using the given vocab. | `vocab` | The shared vocab. ~~Vocab~~ | | **YIELDS** | The `Doc` objects. ~~Doc~~ | -## DocBin.merge {#merge tag="method"} +## DocBin.merge {id="merge",tag="method"} Extend the annotations of this `DocBin` with the annotations from another. Will raise an error if the pre-defined `attrs` of the two `DocBin`s don't match. @@ -118,7 +117,7 @@ raise an error if the pre-defined `attrs` of the two `DocBin`s don't match. | -------- | ------------------------------------------------------ | | `other` | The `DocBin` to merge into the current bin. ~~DocBin~~ | -## DocBin.to_bytes {#to_bytes tag="method"} +## DocBin.to_bytes {id="to_bytes",tag="method"} Serialize the `DocBin`'s annotations to a bytestring. @@ -134,7 +133,7 @@ Serialize the `DocBin`'s annotations to a bytestring. | ----------- | ---------------------------------- | | **RETURNS** | The serialized `DocBin`. ~~bytes~~ | -## DocBin.from_bytes {#from_bytes tag="method"} +## DocBin.from_bytes {id="from_bytes",tag="method"} Deserialize the `DocBin`'s annotations from a bytestring. @@ -150,7 +149,7 @@ Deserialize the `DocBin`'s annotations from a bytestring. | `bytes_data` | The data to load from. ~~bytes~~ | | **RETURNS** | The loaded `DocBin`. ~~DocBin~~ | -## DocBin.to_disk {#to_disk tag="method" new="3"} +## DocBin.to_disk {id="to_disk",tag="method",version="3"} Save the serialized `DocBin` to a file. Typically uses the `.spacy` extension and the result can be used as the input data for @@ -168,7 +167,7 @@ and the result can be used as the input data for | -------- | -------------------------------------------------------------------------- | | `path` | The file path, typically with the `.spacy` extension. ~~Union[str, Path]~~ | -## DocBin.from_disk {#from_disk tag="method" new="3"} +## DocBin.from_disk {id="from_disk",tag="method",version="3"} Load a serialized `DocBin` from a file. Typically uses the `.spacy` extension. diff --git a/website/docs/api/edittreelemmatizer.md b/website/docs/api/edittreelemmatizer.mdx similarity index 95% rename from website/docs/api/edittreelemmatizer.md rename to website/docs/api/edittreelemmatizer.mdx index 63e4bf910..82967482c 100644 --- a/website/docs/api/edittreelemmatizer.md +++ b/website/docs/api/edittreelemmatizer.mdx @@ -2,7 +2,7 @@ title: EditTreeLemmatizer tag: class source: spacy/pipeline/edit_tree_lemmatizer.py -new: 3.3 +version: 3.3 teaser: 'Pipeline component for lemmatization' api_base_class: /api/pipe api_string_name: trainable_lemmatizer @@ -18,7 +18,7 @@ and construction method used by this lemmatizer were proposed in For a lookup and rule-based lemmatizer, see [`Lemmatizer`](/api/lemmatizer). -## Assigned Attributes {#assigned-attributes} +## Assigned Attributes {id="assigned-attributes"} Predictions are assigned to `Token.lemma`. @@ -27,7 +27,7 @@ Predictions are assigned to `Token.lemma`. | `Token.lemma` | The lemma (hash). ~~int~~ | | `Token.lemma_` | The lemma. ~~str~~ | -## Config and implementation {#config} +## Config and implementation {id="config"} The default config is defined by the pipeline component factory and describes how the component should be configured. You can override its settings via the @@ -57,7 +57,7 @@ architectures and their arguments and hyperparameters. %%GITHUB_SPACY/spacy/pipeline/edit_tree_lemmatizer.py ``` -## EditTreeLemmatizer.\_\_init\_\_ {#init tag="method"} +## EditTreeLemmatizer.\_\_init\_\_ {id="init",tag="method"} > #### Example > @@ -90,7 +90,7 @@ shortcut for this and instantiate the component using its string name and | `top_k` | The number of most probable edit trees to try before resorting to `backoff`. Defaults to `1`. ~~int~~ | | `scorer` | The scoring method. Defaults to [`Scorer.score_token_attr`](/api/scorer#score_token_attr) for the attribute `"lemma"`. ~~Optional[Callable]~~ | -## EditTreeLemmatizer.\_\_call\_\_ {#call tag="method"} +## EditTreeLemmatizer.\_\_call\_\_ {id="call",tag="method"} Apply the pipe to one document. The document is modified in place, and returned. This usually happens under the hood when the `nlp` object is called on a text @@ -114,7 +114,7 @@ and all pipeline components are applied to the `Doc` in order. Both | `doc` | The document to process. ~~Doc~~ | | **RETURNS** | The processed document. ~~Doc~~ | -## EditTreeLemmatizer.pipe {#pipe tag="method"} +## EditTreeLemmatizer.pipe {id="pipe",tag="method"} Apply the pipe to a stream of documents. This usually happens under the hood when the `nlp` object is called on a text and all pipeline components are @@ -138,7 +138,7 @@ and [`pipe`](/api/edittreelemmatizer#pipe) delegate to the | `batch_size` | The number of documents to buffer. Defaults to `128`. ~~int~~ | | **YIELDS** | The processed documents in order. ~~Doc~~ | -## EditTreeLemmatizer.initialize {#initialize tag="method" new="3"} +## EditTreeLemmatizer.initialize {id="initialize",tag="method",version="3"} Initialize the component for training. `get_examples` should be a function that returns an iterable of [`Example`](/api/example) objects. **At least one example @@ -175,7 +175,7 @@ config. | `nlp` | The current `nlp` object. Defaults to `None`. ~~Optional[Language]~~ | | `labels` | The label information to add to the component, as provided by the [`label_data`](#label_data) property after initialization. To generate a reusable JSON file from your data, you should run the [`init labels`](/api/cli#init-labels) command. If no labels are provided, the `get_examples` callback is used to extract the labels from the data, which may be a lot slower. ~~Optional[Iterable[str]]~~ | -## EditTreeLemmatizer.predict {#predict tag="method"} +## EditTreeLemmatizer.predict {id="predict",tag="method"} Apply the component's model to a batch of [`Doc`](/api/doc) objects, without modifying them. @@ -192,7 +192,7 @@ modifying them. | `docs` | The documents to predict. ~~Iterable[Doc]~~ | | **RETURNS** | The model's prediction for each document. | -## EditTreeLemmatizer.set_annotations {#set_annotations tag="method"} +## EditTreeLemmatizer.set_annotations {id="set_annotations",tag="method"} Modify a batch of [`Doc`](/api/doc) objects, using pre-computed tree identifiers. @@ -210,7 +210,7 @@ identifiers. | `docs` | The documents to modify. ~~Iterable[Doc]~~ | | `tree_ids` | The identifiers of the edit trees to apply, produced by `EditTreeLemmatizer.predict`. | -## EditTreeLemmatizer.update {#update tag="method"} +## EditTreeLemmatizer.update {id="update",tag="method"} Learn from a batch of [`Example`](/api/example) objects containing the predictions and gold-standard annotations, and update the component's model. @@ -234,7 +234,7 @@ Delegates to [`predict`](/api/edittreelemmatizer#predict) and | `losses` | Optional record of the loss during training. Updated using the component name as the key. ~~Optional[Dict[str, float]]~~ | | **RETURNS** | The updated `losses` dictionary. ~~Dict[str, float]~~ | -## EditTreeLemmatizer.get_loss {#get_loss tag="method"} +## EditTreeLemmatizer.get_loss {id="get_loss",tag="method"} Find the loss and gradient of loss for the batch of documents and their predicted scores. @@ -253,7 +253,7 @@ predicted scores. | `scores` | Scores representing the model's predictions. | | **RETURNS** | The loss and the gradient, i.e. `(loss, gradient)`. ~~Tuple[float, float]~~ | -## EditTreeLemmatizer.create_optimizer {#create_optimizer tag="method"} +## EditTreeLemmatizer.create_optimizer {id="create_optimizer",tag="method"} Create an optimizer for the pipeline component. @@ -268,7 +268,7 @@ Create an optimizer for the pipeline component. | ----------- | ---------------------------- | | **RETURNS** | The optimizer. ~~Optimizer~~ | -## EditTreeLemmatizer.use_params {#use_params tag="method, contextmanager"} +## EditTreeLemmatizer.use_params {id="use_params",tag="method, contextmanager"} Modify the pipe's model, to use the given parameter values. At the end of the context, the original parameters are restored. @@ -285,7 +285,7 @@ context, the original parameters are restored. | -------- | -------------------------------------------------- | | `params` | The parameter values to use in the model. ~~dict~~ | -## EditTreeLemmatizer.to_disk {#to_disk tag="method"} +## EditTreeLemmatizer.to_disk {id="to_disk",tag="method"} Serialize the pipe to disk. @@ -302,7 +302,7 @@ Serialize the pipe to disk. | _keyword-only_ | | | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | -## EditTreeLemmatizer.from_disk {#from_disk tag="method"} +## EditTreeLemmatizer.from_disk {id="from_disk",tag="method"} Load the pipe from disk. Modifies the object in place and returns it. @@ -320,7 +320,7 @@ Load the pipe from disk. Modifies the object in place and returns it. | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | | **RETURNS** | The modified `EditTreeLemmatizer` object. ~~EditTreeLemmatizer~~ | -## EditTreeLemmatizer.to_bytes {#to_bytes tag="method"} +## EditTreeLemmatizer.to_bytes {id="to_bytes",tag="method"} > #### Example > @@ -337,7 +337,7 @@ Serialize the pipe to a bytestring. | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | | **RETURNS** | The serialized form of the `EditTreeLemmatizer` object. ~~bytes~~ | -## EditTreeLemmatizer.from_bytes {#from_bytes tag="method"} +## EditTreeLemmatizer.from_bytes {id="from_bytes",tag="method"} Load the pipe from a bytestring. Modifies the object in place and returns it. @@ -356,7 +356,7 @@ Load the pipe from a bytestring. Modifies the object in place and returns it. | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | | **RETURNS** | The `EditTreeLemmatizer` object. ~~EditTreeLemmatizer~~ | -## EditTreeLemmatizer.labels {#labels tag="property"} +## EditTreeLemmatizer.labels {id="labels",tag="property"} The labels currently added to the component. @@ -371,7 +371,7 @@ identifiers of edit trees. | ----------- | ------------------------------------------------------ | | **RETURNS** | The labels added to the component. ~~Tuple[str, ...]~~ | -## EditTreeLemmatizer.label_data {#label_data tag="property" new="3"} +## EditTreeLemmatizer.label_data {id="label_data",tag="property",version="3"} The labels currently added to the component and their internal meta information. This is the data generated by [`init labels`](/api/cli#init-labels) and used by @@ -389,7 +389,7 @@ initialize the model with a pre-defined label set. | ----------- | ---------------------------------------------------------- | | **RETURNS** | The label data added to the component. ~~Tuple[str, ...]~~ | -## Serialization fields {#serialization-fields} +## Serialization fields {id="serialization-fields"} During serialization, spaCy will export several data fields used to restore different aspects of the object. If needed, you can exclude them from diff --git a/website/docs/api/entitylinker.md b/website/docs/api/entitylinker.mdx similarity index 82% rename from website/docs/api/entitylinker.md rename to website/docs/api/entitylinker.mdx index 40ec8afb5..d84dd3ca9 100644 --- a/website/docs/api/entitylinker.md +++ b/website/docs/api/entitylinker.mdx @@ -2,7 +2,7 @@ title: EntityLinker tag: class source: spacy/pipeline/entity_linker.py -new: 2.2 +version: 2.2 teaser: 'Pipeline component for named entity linking and disambiguation' api_base_class: /api/pipe api_string_name: entity_linker @@ -15,9 +15,9 @@ world". It requires a `KnowledgeBase`, as well as a function to generate plausible candidates from that `KnowledgeBase` given a certain textual mention, and a machine learning model to pick the right candidate, given the local context of the mention. `EntityLinker` defaults to using the -[`InMemoryLookupKB`](/api/kb_in_memory) implementation. +[`InMemoryLookupKB`](/api/inmemorylookupkb) implementation. -## Assigned Attributes {#assigned-attributes} +## Assigned Attributes {id="assigned-attributes"} Predictions, in the form of knowledge base IDs, will be assigned to `Token.ent_kb_id_`. @@ -27,7 +27,7 @@ Predictions, in the form of knowledge base IDs, will be assigned to | `Token.ent_kb_id` | Knowledge base ID (hash). ~~int~~ | | `Token.ent_kb_id_` | Knowledge base ID. ~~str~~ | -## Config and implementation {#config} +## Config and implementation {id="config"} The default config is defined by the pipeline component factory and describes how the component should be configured. You can override its settings via the @@ -53,25 +53,27 @@ architectures and their arguments and hyperparameters. > nlp.add_pipe("entity_linker", config=config) > ``` -| Setting | Description | -| ---------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `labels_discard` | NER labels that will automatically get a "NIL" prediction. Defaults to `[]`. ~~Iterable[str]~~ | -| `n_sents` | The number of neighbouring sentences to take into account. Defaults to 0. ~~int~~ | -| `incl_prior` | Whether or not to include prior probabilities from the KB in the model. Defaults to `True`. ~~bool~~ | -| `incl_context` | Whether or not to include the local context in the model. Defaults to `True`. ~~bool~~ | -| `model` | The [`Model`](https://thinc.ai/docs/api-model) powering the pipeline component. Defaults to [EntityLinker](/api/architectures#EntityLinker). ~~Model~~ | -| `entity_vector_length` | Size of encoding vectors in the KB. Defaults to `64`. ~~int~~ | -| `use_gold_ents` | Whether to copy entities from the gold docs or not. Defaults to `True`. If `False`, entities must be set in the training data or by an annotating component in the pipeline. ~~int~~ | -| `get_candidates` | Function that generates plausible candidates for a given `Span` object. Defaults to [CandidateGenerator](/api/architectures#CandidateGenerator), a function looking up exact, case-dependent aliases in the KB. ~~Callable[[KnowledgeBase, Span], Iterable[Candidate]]~~ | -| `overwrite` 3.2 | Whether existing annotation is overwritten. Defaults to `True`. ~~bool~~ | -| `scorer` 3.2 | The scoring method. Defaults to [`Scorer.score_links`](/api/scorer#score_links). ~~Optional[Callable]~~ | -| `threshold` 3.4 | Confidence threshold for entity predictions. The default of `None` implies that all predictions are accepted, otherwise those with a score beneath the treshold are discarded. If there are no predictions with scores above the threshold, the linked entity is `NIL`. ~~Optional[float]~~ | +| Setting | Description | +| --------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `labels_discard` | NER labels that will automatically get a "NIL" prediction. Defaults to `[]`. ~~Iterable[str]~~ | +| `n_sents` | The number of neighbouring sentences to take into account. Defaults to 0. ~~int~~ | +| `incl_prior` | Whether or not to include prior probabilities from the KB in the model. Defaults to `True`. ~~bool~~ | +| `incl_context` | Whether or not to include the local context in the model. Defaults to `True`. ~~bool~~ | +| `model` | The [`Model`](https://thinc.ai/docs/api-model) powering the pipeline component. Defaults to [EntityLinker](/api/architectures#EntityLinker). ~~Model~~ | +| `entity_vector_length` | Size of encoding vectors in the KB. Defaults to `64`. ~~int~~ | +| `use_gold_ents` | Whether to copy entities from the gold docs or not. Defaults to `True`. If `False`, entities must be set in the training data or by an annotating component in the pipeline. ~~int~~ | +| `get_candidates` | Function that generates plausible candidates for a given `Span` object. Defaults to [CandidateGenerator](/api/architectures#CandidateGenerator), a function looking up exact, case-dependent aliases in the KB. ~~Callable[[KnowledgeBase, Span], Iterable[Candidate]]~~ | +| `get_candidates_batch` 3.5 | Function that generates plausible candidates for a given batch of `Span` objects. Defaults to [CandidateBatchGenerator](/api/architectures#CandidateBatchGenerator), a function looking up exact, case-dependent aliases in the KB. ~~Callable[[KnowledgeBase, Iterable[Span]], Iterable[Iterable[Candidate]]]~~ | +| `generate_empty_kb` 3.6 | Function that generates an empty `KnowledgeBase` object. Defaults to [`spacy.EmptyKB.v2`](/api/architectures#EmptyKB), which generates an empty [`InMemoryLookupKB`](/api/inmemorylookupkb). ~~Callable[[Vocab, int], KnowledgeBase]~~ | +| `overwrite` 3.2 | Whether existing annotation is overwritten. Defaults to `True`. ~~bool~~ | +| `scorer` 3.2 | The scoring method. Defaults to [`Scorer.score_links`](/api/scorer#score_links). ~~Optional[Callable]~~ | +| `threshold` 3.4 | Confidence threshold for entity predictions. The default of `None` implies that all predictions are accepted, otherwise those with a score beneath the treshold are discarded. If there are no predictions with scores above the threshold, the linked entity is `NIL`. ~~Optional[float]~~ | ```python %%GITHUB_SPACY/spacy/pipeline/entity_linker.py ``` -## EntityLinker.\_\_init\_\_ {#init tag="method"} +## EntityLinker.\_\_init\_\_ {id="init",tag="method"} > #### Example > @@ -114,7 +116,7 @@ custom knowledge base, you should either call | `scorer` 3.2 | The scoring method. Defaults to [`Scorer.score_links`](/api/scorer#score_links). ~~Optional[Callable]~~ | | `threshold` 3.4 | Confidence threshold for entity predictions. The default of `None` implies that all predictions are accepted, otherwise those with a score beneath the treshold are discarded. If there are no predictions with scores above the threshold, the linked entity is `NIL`. ~~Optional[float]~~ | -## EntityLinker.\_\_call\_\_ {#call tag="method"} +## EntityLinker.\_\_call\_\_ {id="call",tag="method"} Apply the pipe to one document. The document is modified in place and returned. This usually happens under the hood when the `nlp` object is called on a text @@ -137,7 +139,7 @@ delegate to the [`predict`](/api/entitylinker#predict) and | `doc` | The document to process. ~~Doc~~ | | **RETURNS** | The processed document. ~~Doc~~ | -## EntityLinker.pipe {#pipe tag="method"} +## EntityLinker.pipe {id="pipe",tag="method"} Apply the pipe to a stream of documents. This usually happens under the hood when the `nlp` object is called on a text and all pipeline components are @@ -161,7 +163,7 @@ applied to the `Doc` in order. Both [`__call__`](/api/entitylinker#call) and | `batch_size` | The number of documents to buffer. Defaults to `128`. ~~int~~ | | **YIELDS** | The processed documents in order. ~~Doc~~ | -## EntityLinker.set_kb {#set_kb tag="method" new="3"} +## EntityLinker.set_kb {id="set_kb",tag="method",version="3"} The `kb_loader` should be a function that takes a `Vocab` instance and creates the `KnowledgeBase`, ensuring that the strings of the knowledge base are synced @@ -183,7 +185,7 @@ with the current vocab. | ----------- | ---------------------------------------------------------------------------------------------------------------- | | `kb_loader` | Function that creates a [`KnowledgeBase`](/api/kb) from a `Vocab` instance. ~~Callable[[Vocab], KnowledgeBase]~~ | -## EntityLinker.initialize {#initialize tag="method" new="3"} +## EntityLinker.initialize {id="initialize",tag="method",version="3"} Initialize the component for training. `get_examples` should be a function that returns an iterable of [`Example`](/api/example) objects. **At least one example @@ -219,7 +221,7 @@ This method was previously called `begin_training`. | `nlp` | The current `nlp` object. Defaults to `None`. ~~Optional[Language]~~ | | `kb_loader` | Function that creates a [`KnowledgeBase`](/api/kb) from a `Vocab` instance. ~~Callable[[Vocab], KnowledgeBase]~~ | -## EntityLinker.predict {#predict tag="method"} +## EntityLinker.predict {id="predict",tag="method"} Apply the component's model to a batch of [`Doc`](/api/doc) objects, without modifying them. Returns the KB IDs for each entity in each doc, including `NIL` @@ -237,7 +239,7 @@ if there is no prediction. | `docs` | The documents to predict. ~~Iterable[Doc]~~ | | **RETURNS** | The predicted KB identifiers for the entities in the `docs`. ~~List[str]~~ | -## EntityLinker.set_annotations {#set_annotations tag="method"} +## EntityLinker.set_annotations {id="set_annotations",tag="method"} Modify a batch of documents, using pre-computed entity IDs for a list of named entities. @@ -255,7 +257,7 @@ entities. | `docs` | The documents to modify. ~~Iterable[Doc]~~ | | `kb_ids` | The knowledge base identifiers for the entities in the docs, predicted by `EntityLinker.predict`. ~~List[str]~~ | -## EntityLinker.update {#update tag="method"} +## EntityLinker.update {id="update",tag="method"} Learn from a batch of [`Example`](/api/example) objects, updating both the pipe's entity linking model and context encoder. Delegates to @@ -278,7 +280,7 @@ pipe's entity linking model and context encoder. Delegates to | `losses` | Optional record of the loss during training. Updated using the component name as the key. ~~Optional[Dict[str, float]]~~ | | **RETURNS** | The updated `losses` dictionary. ~~Dict[str, float]~~ | -## EntityLinker.create_optimizer {#create_optimizer tag="method"} +## EntityLinker.create_optimizer {id="create_optimizer",tag="method"} Create an optimizer for the pipeline component. @@ -293,7 +295,7 @@ Create an optimizer for the pipeline component. | ----------- | ---------------------------- | | **RETURNS** | The optimizer. ~~Optimizer~~ | -## EntityLinker.use_params {#use_params tag="method, contextmanager"} +## EntityLinker.use_params {id="use_params",tag="method, contextmanager"} Modify the pipe's model, to use the given parameter values. At the end of the context, the original parameters are restored. @@ -310,7 +312,7 @@ context, the original parameters are restored. | -------- | -------------------------------------------------- | | `params` | The parameter values to use in the model. ~~dict~~ | -## EntityLinker.to_disk {#to_disk tag="method"} +## EntityLinker.to_disk {id="to_disk",tag="method"} Serialize the pipe to disk. @@ -327,7 +329,7 @@ Serialize the pipe to disk. | _keyword-only_ | | | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | -## EntityLinker.from_disk {#from_disk tag="method"} +## EntityLinker.from_disk {id="from_disk",tag="method"} Load the pipe from disk. Modifies the object in place and returns it. @@ -345,7 +347,7 @@ Load the pipe from disk. Modifies the object in place and returns it. | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | | **RETURNS** | The modified `EntityLinker` object. ~~EntityLinker~~ | -## EntityLinker.to_bytes {#to_bytes tag="method"} +## EntityLinker.to_bytes {id="to_bytes",tag="method"} > #### Example > @@ -362,7 +364,7 @@ Serialize the pipe to a bytestring, including the `KnowledgeBase`. | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | | **RETURNS** | The serialized form of the `EntityLinker` object. ~~bytes~~ | -## EntityLinker.from_bytes {#from_bytes tag="method"} +## EntityLinker.from_bytes {id="from_bytes",tag="method"} Load the pipe from a bytestring. Modifies the object in place and returns it. @@ -381,7 +383,7 @@ Load the pipe from a bytestring. Modifies the object in place and returns it. | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | | **RETURNS** | The `EntityLinker` object. ~~EntityLinker~~ | -## Serialization fields {#serialization-fields} +## Serialization fields {id="serialization-fields"} During serialization, spaCy will export several data fields used to restore different aspects of the object. If needed, you can exclude them from diff --git a/website/docs/api/entityrecognizer.md b/website/docs/api/entityrecognizer.mdx similarity index 95% rename from website/docs/api/entityrecognizer.md rename to website/docs/api/entityrecognizer.mdx index a535e8316..c80406a5b 100644 --- a/website/docs/api/entityrecognizer.md +++ b/website/docs/api/entityrecognizer.mdx @@ -20,7 +20,7 @@ your entities will be close to their initial tokens. If your entities are long and characterized by tokens in their middle, the component will likely not be a good fit for your task. -## Assigned Attributes {#assigned-attributes} +## Assigned Attributes {id="assigned-attributes"} Predictions will be saved to `Doc.ents` as a tuple. Each label will also be reflected to each underlying token, where it is saved in the `Token.ent_type` @@ -38,7 +38,7 @@ non-overlapping, or an error will be thrown. | `Token.ent_type` | The label part of the named entity tag (hash). ~~int~~ | | `Token.ent_type_` | The label part of the named entity tag. ~~str~~ | -## Config and implementation {#config} +## Config and implementation {id="config"} The default config is defined by the pipeline component factory and describes how the component should be configured. You can override its settings via the @@ -72,7 +72,7 @@ architectures and their arguments and hyperparameters. %%GITHUB_SPACY/spacy/pipeline/ner.pyx ``` -## EntityRecognizer.\_\_init\_\_ {#init tag="method"} +## EntityRecognizer.\_\_init\_\_ {id="init",tag="method"} > #### Example > @@ -103,7 +103,7 @@ shortcut for this and instantiate the component using its string name and | `update_with_oracle_cut_size` | During training, cut long sequences into shorter segments by creating intermediate states based on the gold-standard history. The model is not very sensitive to this parameter, so you usually won't need to change it. Defaults to `100`. ~~int~~ | | `incorrect_spans_key` | Identifies spans that are known to be incorrect entity annotations. The incorrect entity annotations can be stored in the span group in [`Doc.spans`](/api/doc#spans), under this key. Defaults to `None`. ~~Optional[str]~~ | -## EntityRecognizer.\_\_call\_\_ {#call tag="method"} +## EntityRecognizer.\_\_call\_\_ {id="call",tag="method"} Apply the pipe to one document. The document is modified in place and returned. This usually happens under the hood when the `nlp` object is called on a text @@ -127,7 +127,7 @@ and all pipeline components are applied to the `Doc` in order. Both | `doc` | The document to process. ~~Doc~~ | | **RETURNS** | The processed document. ~~Doc~~ | -## EntityRecognizer.pipe {#pipe tag="method"} +## EntityRecognizer.pipe {id="pipe",tag="method"} Apply the pipe to a stream of documents. This usually happens under the hood when the `nlp` object is called on a text and all pipeline components are @@ -151,7 +151,7 @@ applied to the `Doc` in order. Both [`__call__`](/api/entityrecognizer#call) and | `batch_size` | The number of documents to buffer. Defaults to `128`. ~~int~~ | | **YIELDS** | The processed documents in order. ~~Doc~~ | -## EntityRecognizer.initialize {#initialize tag="method" new="3"} +## EntityRecognizer.initialize {id="initialize",tag="method",version="3"} Initialize the component for training. `get_examples` should be a function that returns an iterable of [`Example`](/api/example) objects. **At least one example @@ -194,7 +194,7 @@ This method was previously called `begin_training`. | `nlp` | The current `nlp` object. Defaults to `None`. ~~Optional[Language]~~ | | `labels` | The label information to add to the component, as provided by the [`label_data`](#label_data) property after initialization. To generate a reusable JSON file from your data, you should run the [`init labels`](/api/cli#init-labels) command. If no labels are provided, the `get_examples` callback is used to extract the labels from the data, which may be a lot slower. ~~Optional[Dict[str, Dict[str, int]]]~~ | -## EntityRecognizer.predict {#predict tag="method"} +## EntityRecognizer.predict {id="predict",tag="method"} Apply the component's model to a batch of [`Doc`](/api/doc) objects, without modifying them. @@ -211,7 +211,7 @@ modifying them. | `docs` | The documents to predict. ~~Iterable[Doc]~~ | | **RETURNS** | A helper class for the parse state (internal). ~~StateClass~~ | -## EntityRecognizer.set_annotations {#set_annotations tag="method"} +## EntityRecognizer.set_annotations {id="set_annotations",tag="method"} Modify a batch of [`Doc`](/api/doc) objects, using pre-computed scores. @@ -228,7 +228,7 @@ Modify a batch of [`Doc`](/api/doc) objects, using pre-computed scores. | `docs` | The documents to modify. ~~Iterable[Doc]~~ | | `scores` | The scores to set, produced by `EntityRecognizer.predict`. Returns an internal helper class for the parse state. ~~List[StateClass]~~ | -## EntityRecognizer.update {#update tag="method"} +## EntityRecognizer.update {id="update",tag="method"} Learn from a batch of [`Example`](/api/example) objects, updating the pipe's model. Delegates to [`predict`](/api/entityrecognizer#predict) and @@ -251,7 +251,7 @@ model. Delegates to [`predict`](/api/entityrecognizer#predict) and | `losses` | Optional record of the loss during training. Updated using the component name as the key. ~~Optional[Dict[str, float]]~~ | | **RETURNS** | The updated `losses` dictionary. ~~Dict[str, float]~~ | -## EntityRecognizer.get_loss {#get_loss tag="method"} +## EntityRecognizer.get_loss {id="get_loss",tag="method"} Find the loss and gradient of loss for the batch of documents and their predicted scores. @@ -270,7 +270,7 @@ predicted scores. | `scores` | Scores representing the model's predictions. ~~StateClass~~ | | **RETURNS** | The loss and the gradient, i.e. `(loss, gradient)`. ~~Tuple[float, float]~~ | -## EntityRecognizer.create_optimizer {#create_optimizer tag="method"} +## EntityRecognizer.create_optimizer {id="create_optimizer",tag="method"} Create an optimizer for the pipeline component. @@ -285,7 +285,7 @@ Create an optimizer for the pipeline component. | ----------- | ---------------------------- | | **RETURNS** | The optimizer. ~~Optimizer~~ | -## EntityRecognizer.use_params {#use_params tag="method, contextmanager"} +## EntityRecognizer.use_params {id="use_params",tag="method, contextmanager"} Modify the pipe's model, to use the given parameter values. At the end of the context, the original parameters are restored. @@ -302,7 +302,7 @@ context, the original parameters are restored. | -------- | -------------------------------------------------- | | `params` | The parameter values to use in the model. ~~dict~~ | -## EntityRecognizer.add_label {#add_label tag="method"} +## EntityRecognizer.add_label {id="add_label",tag="method"} Add a new label to the pipe. Note that you don't have to call this method if you provide a **representative data sample** to the [`initialize`](#initialize) @@ -322,7 +322,7 @@ to the model, and the output dimension will be | `label` | The label to add. ~~str~~ | | **RETURNS** | `0` if the label is already present, otherwise `1`. ~~int~~ | -## EntityRecognizer.set_output {#set_output tag="method"} +## EntityRecognizer.set_output {id="set_output",tag="method"} Change the output dimension of the component's model by calling the model's attribute `resize_output`. This is a function that takes the original model and @@ -341,7 +341,7 @@ forgetting" problem. | ---- | --------------------------------- | | `nO` | The new output dimension. ~~int~~ | -## EntityRecognizer.to_disk {#to_disk tag="method"} +## EntityRecognizer.to_disk {id="to_disk",tag="method"} Serialize the pipe to disk. @@ -358,7 +358,7 @@ Serialize the pipe to disk. | _keyword-only_ | | | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | -## EntityRecognizer.from_disk {#from_disk tag="method"} +## EntityRecognizer.from_disk {id="from_disk",tag="method"} Load the pipe from disk. Modifies the object in place and returns it. @@ -376,7 +376,7 @@ Load the pipe from disk. Modifies the object in place and returns it. | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | | **RETURNS** | The modified `EntityRecognizer` object. ~~EntityRecognizer~~ | -## EntityRecognizer.to_bytes {#to_bytes tag="method"} +## EntityRecognizer.to_bytes {id="to_bytes",tag="method"} > #### Example > @@ -393,7 +393,7 @@ Serialize the pipe to a bytestring. | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | | **RETURNS** | The serialized form of the `EntityRecognizer` object. ~~bytes~~ | -## EntityRecognizer.from_bytes {#from_bytes tag="method"} +## EntityRecognizer.from_bytes {id="from_bytes",tag="method"} Load the pipe from a bytestring. Modifies the object in place and returns it. @@ -412,7 +412,7 @@ Load the pipe from a bytestring. Modifies the object in place and returns it. | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | | **RETURNS** | The `EntityRecognizer` object. ~~EntityRecognizer~~ | -## EntityRecognizer.labels {#labels tag="property"} +## EntityRecognizer.labels {id="labels",tag="property"} The labels currently added to the component. @@ -427,7 +427,7 @@ The labels currently added to the component. | ----------- | ------------------------------------------------------ | | **RETURNS** | The labels added to the component. ~~Tuple[str, ...]~~ | -## EntityRecognizer.label_data {#label_data tag="property" new="3"} +## EntityRecognizer.label_data {id="label_data",tag="property",version="3"} The labels currently added to the component and their internal meta information. This is the data generated by [`init labels`](/api/cli#init-labels) and used by @@ -445,7 +445,7 @@ the model with a pre-defined label set. | ----------- | ------------------------------------------------------------------------------- | | **RETURNS** | The label data added to the component. ~~Dict[str, Dict[str, Dict[str, int]]]~~ | -## Serialization fields {#serialization-fields} +## Serialization fields {id="serialization-fields"} During serialization, spaCy will export several data fields used to restore different aspects of the object. If needed, you can exclude them from diff --git a/website/docs/api/entityruler.md b/website/docs/api/entityruler.mdx similarity index 70% rename from website/docs/api/entityruler.md rename to website/docs/api/entityruler.mdx index c2ba33f01..27624398e 100644 --- a/website/docs/api/entityruler.md +++ b/website/docs/api/entityruler.mdx @@ -2,7 +2,7 @@ title: EntityRuler tag: class source: spacy/pipeline/entityruler.py -new: 2.1 +version: 2.1 teaser: 'Pipeline component for rule-based named entity recognition' api_string_name: entity_ruler api_trainable: false @@ -15,7 +15,7 @@ used on its own to implement a purely rule-based entity recognition system. For usage examples, see the docs on [rule-based entity recognition](/usage/rule-based-matching#entityruler). -## Assigned Attributes {#assigned-attributes} +## Assigned Attributes {id="assigned-attributes"} This component assigns predictions basically the same way as the [`EntityRecognizer`](/api/entityrecognizer). @@ -36,7 +36,7 @@ non-overlapping, or an error will be thrown. | `Token.ent_type` | The label part of the named entity tag (hash). ~~int~~ | | `Token.ent_type_` | The label part of the named entity tag. ~~str~~ | -## Config and implementation {#config} +## Config and implementation {id="config"} The default config is defined by the pipeline component factory and describes how the component should be configured. You can override its settings via the @@ -55,19 +55,20 @@ how the component should be configured. You can override its settings via the > nlp.add_pipe("entity_ruler", config=config) > ``` -| Setting | Description | -| --------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `phrase_matcher_attr` | Optional attribute name match on for the internal [`PhraseMatcher`](/api/phrasematcher), e.g. `LOWER` to match on the lowercase token text. Defaults to `None`. ~~Optional[Union[int, str]]~~ | -| `validate` | Whether patterns should be validated (passed to the `Matcher` and `PhraseMatcher`). Defaults to `False`. ~~bool~~ | -| `overwrite_ents` | If existing entities are present, e.g. entities added by the model, overwrite them by matches if necessary. Defaults to `False`. ~~bool~~ | -| `ent_id_sep` | Separator used internally for entity IDs. Defaults to `"\|\|"`. ~~str~~ | -| `scorer` | The scoring method. Defaults to [`spacy.scorer.get_ner_prf`](/api/scorer#get_ner_prf). ~~Optional[Callable]~~ | +| Setting | Description | +| ---------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `phrase_matcher_attr` | Optional attribute name match on for the internal [`PhraseMatcher`](/api/phrasematcher), e.g. `LOWER` to match on the lowercase token text. Defaults to `None`. ~~Optional[Union[int, str]]~~ | +| `matcher_fuzzy_compare` 3.5 | The fuzzy comparison method, passed on to the internal `Matcher`. Defaults to `spacy.matcher.levenshtein.levenshtein_compare`. ~~Callable~~ | +| `validate` | Whether patterns should be validated (passed to the `Matcher` and `PhraseMatcher`). Defaults to `False`. ~~bool~~ | +| `overwrite_ents` | If existing entities are present, e.g. entities added by the model, overwrite them by matches if necessary. Defaults to `False`. ~~bool~~ | +| `ent_id_sep` | Separator used internally for entity IDs. Defaults to `"\|\|"`. ~~str~~ | +| `scorer` | The scoring method. Defaults to [`spacy.scorer.get_ner_prf`](/api/scorer#get_ner_prf). ~~Optional[Callable]~~ | ```python %%GITHUB_SPACY/spacy/pipeline/entityruler.py ``` -## EntityRuler.\_\_init\_\_ {#init tag="method"} +## EntityRuler.\_\_init\_\_ {id="init",tag="method"} Initialize the entity ruler. If patterns are supplied here, they need to be a list of dictionaries with a `"label"` and `"pattern"` key. A pattern can either @@ -85,23 +86,25 @@ be a token pattern (list) or a phrase pattern (string). For example: > ruler = EntityRuler(nlp, overwrite_ents=True) > ``` -| Name | Description | -| --------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `nlp` | The shared nlp object to pass the vocab to the matchers and process phrase patterns. ~~Language~~ | -| `name` 3 | Instance name of the current pipeline component. Typically passed in automatically from the factory when the component is added. Used to disable the current entity ruler while creating phrase patterns with the nlp object. ~~str~~ | -| _keyword-only_ | | -| `phrase_matcher_attr` | Optional attribute name match on for the internal [`PhraseMatcher`](/api/phrasematcher), e.g. `LOWER` to match on the lowercase token text. Defaults to `None`. ~~Optional[Union[int, str]]~~ | -| `validate` | Whether patterns should be validated, passed to Matcher and PhraseMatcher as `validate`. Defaults to `False`. ~~bool~~ | -| `overwrite_ents` | If existing entities are present, e.g. entities added by the model, overwrite them by matches if necessary. Defaults to `False`. ~~bool~~ | -| `ent_id_sep` | Separator used internally for entity IDs. Defaults to `"\|\|"`. ~~str~~ | -| `patterns` | Optional patterns to load in on initialization. ~~Optional[List[Dict[str, Union[str, List[dict]]]]]~~ | +| Name | Description | +| ---------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `nlp` | The shared nlp object to pass the vocab to the matchers and process phrase patterns. ~~Language~~ | +| `name` 3 | Instance name of the current pipeline component. Typically passed in automatically from the factory when the component is added. Used to disable the current entity ruler while creating phrase patterns with the nlp object. ~~str~~ | +| _keyword-only_ | | +| `phrase_matcher_attr` | Optional attribute name match on for the internal [`PhraseMatcher`](/api/phrasematcher), e.g. `LOWER` to match on the lowercase token text. Defaults to `None`. ~~Optional[Union[int, str]]~~ | +| `matcher_fuzzy_compare` 3.5 | The fuzzy comparison method, passed on to the internal `Matcher`. Defaults to `spacy.matcher.levenshtein.levenshtein_compare`. ~~Callable~~ | +| `validate` | Whether patterns should be validated, passed to Matcher and PhraseMatcher as `validate`. Defaults to `False`. ~~bool~~ | +| `overwrite_ents` | If existing entities are present, e.g. entities added by the model, overwrite them by matches if necessary. Defaults to `False`. ~~bool~~ | +| `ent_id_sep` | Separator used internally for entity IDs. Defaults to `"\|\|"`. ~~str~~ | +| `patterns` | Optional patterns to load in on initialization. ~~Optional[List[Dict[str, Union[str, List[dict]]]]]~~ | +| `scorer` | The scoring method. Defaults to [`spacy.scorer.get_ner_prf`](/api/scorer#get_ner_prf). ~~Optional[Callable]~~ | -## EntityRuler.initialize {#initialize tag="method" new="3"} +## EntityRuler.initialize {id="initialize",tag="method",version="3"} Initialize the component with data and used before training to load in rules -from a [pattern file](/usage/rule-based-matching/#entityruler-files). This method -is typically called by [`Language.initialize`](/api/language#initialize) and -lets you customize arguments it receives via the +from a [pattern file](/usage/rule-based-matching/#entityruler-files). This +method is typically called by [`Language.initialize`](/api/language#initialize) +and lets you customize arguments it receives via the [`[initialize.components]`](/api/data-formats#config-initialize) block in the config. @@ -128,7 +131,7 @@ config. | `nlp` | The current `nlp` object. Defaults to `None`. ~~Optional[Language]~~ | | `patterns` | The list of patterns. Defaults to `None`. ~~Optional[Sequence[Dict[str, Union[str, List[Dict[str, Any]]]]]]~~ | -## EntityRuler.\_\len\_\_ {#len tag="method"} +## EntityRuler.\_\_len\_\_ {id="len",tag="method"} The number of all patterns added to the entity ruler. @@ -145,7 +148,7 @@ The number of all patterns added to the entity ruler. | ----------- | ------------------------------- | | **RETURNS** | The number of patterns. ~~int~~ | -## EntityRuler.\_\_contains\_\_ {#contains tag="method"} +## EntityRuler.\_\_contains\_\_ {id="contains",tag="method"} Whether a label is present in the patterns. @@ -163,7 +166,7 @@ Whether a label is present in the patterns. | `label` | The label to check. ~~str~~ | | **RETURNS** | Whether the entity ruler contains the label. ~~bool~~ | -## EntityRuler.\_\_call\_\_ {#call tag="method"} +## EntityRuler.\_\_call\_\_ {id="call",tag="method"} Find matches in the `Doc` and add them to the `doc.ents`. Typically, this happens automatically after the component has been added to the pipeline using @@ -189,7 +192,7 @@ is chosen. | `doc` | The `Doc` object to process, e.g. the `Doc` in the pipeline. ~~Doc~~ | | **RETURNS** | The modified `Doc` with added entities, if available. ~~Doc~~ | -## EntityRuler.add_patterns {#add_patterns tag="method"} +## EntityRuler.add_patterns {id="add_patterns",tag="method"} Add patterns to the entity ruler. A pattern can either be a token pattern (list of dicts) or a phrase pattern (string). For more details, see the usage guide on @@ -210,10 +213,10 @@ of dicts) or a phrase pattern (string). For more details, see the usage guide on | ---------- | ---------------------------------------------------------------- | | `patterns` | The patterns to add. ~~List[Dict[str, Union[str, List[dict]]]]~~ | +## EntityRuler.remove {id="remove",tag="method",version="3.2.1"} -## EntityRuler.remove {#remove tag="method" new="3.2.1"} - -Remove a pattern by its ID from the entity ruler. A `ValueError` is raised if the ID does not exist. +Remove a pattern by its ID from the entity ruler. A `ValueError` is raised if +the ID does not exist. > #### Example > @@ -224,11 +227,11 @@ Remove a pattern by its ID from the entity ruler. A `ValueError` is raised if th > ruler.remove("apple") > ``` -| Name | Description | -| ---------- | ---------------------------------------------------------------- | -| `id` | The ID of the pattern rule. ~~str~~ | +| Name | Description | +| ---- | ----------------------------------- | +| `id` | The ID of the pattern rule. ~~str~~ | -## EntityRuler.to_disk {#to_disk tag="method"} +## EntityRuler.to_disk {id="to_disk",tag="method"} Save the entity ruler patterns to a directory. The patterns will be saved as newline-delimited JSON (JSONL). If a file with the suffix `.jsonl` is provided, @@ -247,7 +250,7 @@ only the patterns are saved as JSONL. If a directory name is provided, a | ------ | -------------------------------------------------------------------------------------------------------------------------------------------------------- | | `path` | A path to a JSONL file or directory, which will be created if it doesn't exist. Paths may be either strings or `Path`-like objects. ~~Union[str, Path]~~ | -## EntityRuler.from_disk {#from_disk tag="method"} +## EntityRuler.from_disk {id="from_disk",tag="method"} Load the entity ruler from a path. Expects either a file containing newline-delimited JSON (JSONL) with one entry per line, or a directory @@ -267,7 +270,7 @@ configuration. | `path` | A path to a JSONL file or directory. Paths may be either strings or `Path`-like objects. ~~Union[str, Path]~~ | | **RETURNS** | The modified `EntityRuler` object. ~~EntityRuler~~ | -## EntityRuler.to_bytes {#to_bytes tag="method"} +## EntityRuler.to_bytes {id="to_bytes",tag="method"} Serialize the entity ruler patterns to a bytestring. @@ -282,7 +285,7 @@ Serialize the entity ruler patterns to a bytestring. | ----------- | ---------------------------------- | | **RETURNS** | The serialized patterns. ~~bytes~~ | -## EntityRuler.from_bytes {#from_bytes tag="method"} +## EntityRuler.from_bytes {id="from_bytes",tag="method"} Load the pipe from a bytestring. Modifies the object in place and returns it. @@ -299,7 +302,7 @@ Load the pipe from a bytestring. Modifies the object in place and returns it. | `bytes_data` | The bytestring to load. ~~bytes~~ | | **RETURNS** | The modified `EntityRuler` object. ~~EntityRuler~~ | -## EntityRuler.labels {#labels tag="property"} +## EntityRuler.labels {id="labels",tag="property"} All labels present in the match patterns. @@ -307,7 +310,7 @@ All labels present in the match patterns. | ----------- | -------------------------------------- | | **RETURNS** | The string labels. ~~Tuple[str, ...]~~ | -## EntityRuler.ent_ids {#ent_ids tag="property" new="2.2.2"} +## EntityRuler.ent_ids {id="ent_ids",tag="property",version="2.2.2"} All entity IDs present in the `id` properties of the match patterns. @@ -315,7 +318,7 @@ All entity IDs present in the `id` properties of the match patterns. | ----------- | ----------------------------------- | | **RETURNS** | The string IDs. ~~Tuple[str, ...]~~ | -## EntityRuler.patterns {#patterns tag="property"} +## EntityRuler.patterns {id="patterns",tag="property"} Get all patterns that were added to the entity ruler. @@ -323,7 +326,7 @@ Get all patterns that were added to the entity ruler. | ----------- | ---------------------------------------------------------------------------------------- | | **RETURNS** | The original patterns, one dictionary per pattern. ~~List[Dict[str, Union[str, dict]]]~~ | -## Attributes {#attributes} +## Attributes {id="attributes"} | Name | Description | | ----------------- | --------------------------------------------------------------------------------------------------------------------- | diff --git a/website/docs/api/example.md b/website/docs/api/example.mdx similarity index 92% rename from website/docs/api/example.md rename to website/docs/api/example.mdx index 63768d58f..a29d5a7e0 100644 --- a/website/docs/api/example.md +++ b/website/docs/api/example.mdx @@ -3,7 +3,7 @@ title: Example teaser: A training instance tag: class source: spacy/training/example.pyx -new: 3.0 +version: 3.0 --- An `Example` holds the information for one training instance. It stores two @@ -12,7 +12,7 @@ holding the predictions of the pipeline. An [`Alignment`](/api/example#alignment-object) object stores the alignment between these two documents, as they can differ in tokenization. -## Example.\_\_init\_\_ {#init tag="method"} +## Example.\_\_init\_\_ {id="init",tag="method"} Construct an `Example` object from the `predicted` document and the `reference` document. If `alignment` is `None`, it will be initialized from the words in @@ -40,7 +40,7 @@ both documents. | _keyword-only_ | | | `alignment` | An object holding the alignment between the tokens of the `predicted` and `reference` documents. ~~Optional[Alignment]~~ | -## Example.from_dict {#from_dict tag="classmethod"} +## Example.from_dict {id="from_dict",tag="classmethod"} Construct an `Example` object from the `predicted` document and the reference annotations provided as a dictionary. For more details on the required format, @@ -64,7 +64,7 @@ see the [training format documentation](/api/data-formats#dict-input). | `example_dict` | The gold-standard annotations as a dictionary. Cannot be `None`. ~~Dict[str, Any]~~ | | **RETURNS** | The newly constructed object. ~~Example~~ | -## Example.text {#text tag="property"} +## Example.text {id="text",tag="property"} The text of the `predicted` document in this `Example`. @@ -78,7 +78,7 @@ The text of the `predicted` document in this `Example`. | ----------- | --------------------------------------------- | | **RETURNS** | The text of the `predicted` document. ~~str~~ | -## Example.predicted {#predicted tag="property"} +## Example.predicted {id="predicted",tag="property"} The `Doc` holding the predictions. Occasionally also referred to as `example.x`. @@ -94,7 +94,7 @@ The `Doc` holding the predictions. Occasionally also referred to as `example.x`. | ----------- | ------------------------------------------------------ | | **RETURNS** | The document containing (partial) predictions. ~~Doc~~ | -## Example.reference {#reference tag="property"} +## Example.reference {id="reference",tag="property"} The `Doc` holding the gold-standard annotations. Occasionally also referred to as `example.y`. @@ -111,7 +111,7 @@ as `example.y`. | ----------- | ---------------------------------------------------------- | | **RETURNS** | The document containing gold-standard annotations. ~~Doc~~ | -## Example.alignment {#alignment tag="property"} +## Example.alignment {id="alignment",tag="property"} The [`Alignment`](/api/example#alignment-object) object mapping the tokens of the `predicted` document to those of the `reference` document. @@ -131,7 +131,7 @@ the `predicted` document to those of the `reference` document. | ----------- | ---------------------------------------------------------------- | | **RETURNS** | The document containing gold-standard annotations. ~~Alignment~~ | -## Example.get_aligned {#get_aligned tag="method"} +## Example.get_aligned {id="get_aligned",tag="method"} Get the aligned view of a certain token attribute, denoted by its int ID or string name. @@ -152,7 +152,7 @@ string name. | `as_string` | Whether or not to return the list of values as strings. Defaults to `False`. ~~bool~~ | | **RETURNS** | List of integer values, or string values if `as_string` is `True`. ~~Union[List[int], List[str]]~~ | -## Example.get_aligned_parse {#get_aligned_parse tag="method"} +## Example.get_aligned_parse {id="get_aligned_parse",tag="method"} Get the aligned view of the dependency parse. If `projectivize` is set to `True`, non-projective dependency trees are made projective through the @@ -172,7 +172,7 @@ Pseudo-Projective Dependency Parsing algorithm by Nivre and Nilsson (2005). | `projectivize` | Whether or not to projectivize the dependency trees. Defaults to `True`. ~~bool~~ | | **RETURNS** | List of integer values, or string values if `as_string` is `True`. ~~Union[List[int], List[str]]~~ | -## Example.get_aligned_ner {#get_aligned_ner tag="method"} +## Example.get_aligned_ner {id="get_aligned_ner",tag="method"} Get the aligned view of the NER [BILUO](/usage/linguistic-features#accessing-ner) tags. @@ -193,7 +193,7 @@ Get the aligned view of the NER | ----------- | ------------------------------------------------------------------------------------------------- | | **RETURNS** | List of BILUO values, denoting whether tokens are part of an NER annotation or not. ~~List[str]~~ | -## Example.get_aligned_spans_y2x {#get_aligned_spans_y2x tag="method"} +## Example.get_aligned_spans_y2x {id="get_aligned_spans_y2x",tag="method"} Get the aligned view of any set of [`Span`](/api/span) objects defined over [`Example.reference`](/api/example#reference). The resulting span indices will @@ -219,7 +219,7 @@ align to the tokenization in [`Example.predicted`](/api/example#predicted). | `allow_overlap` | Whether the resulting `Span` objects may overlap or not. Set to `False` by default. ~~bool~~ | | **RETURNS** | `Span` objects aligned to the tokenization of `predicted`. ~~List[Span]~~ | -## Example.get_aligned_spans_x2y {#get_aligned_spans_x2y tag="method"} +## Example.get_aligned_spans_x2y {id="get_aligned_spans_x2y",tag="method"} Get the aligned view of any set of [`Span`](/api/span) objects defined over [`Example.predicted`](/api/example#predicted). The resulting span indices will @@ -247,7 +247,7 @@ against the original gold-standard annotation. | `allow_overlap` | Whether the resulting `Span` objects may overlap or not. Set to `False` by default. ~~bool~~ | | **RETURNS** | `Span` objects aligned to the tokenization of `reference`. ~~List[Span]~~ | -## Example.to_dict {#to_dict tag="method"} +## Example.to_dict {id="to_dict",tag="method"} Return a [dictionary representation](/api/data-formats#dict-input) of the reference annotation contained in this `Example`. @@ -262,7 +262,7 @@ reference annotation contained in this `Example`. | ----------- | ------------------------------------------------------------------------- | | **RETURNS** | Dictionary representation of the reference annotation. ~~Dict[str, Any]~~ | -## Example.split_sents {#split_sents tag="method"} +## Example.split_sents {id="split_sents",tag="method"} Split one `Example` into multiple `Example` objects, one for each sentence. @@ -282,15 +282,15 @@ Split one `Example` into multiple `Example` objects, one for each sentence. | ----------- | ---------------------------------------------------------------------------- | | **RETURNS** | List of `Example` objects, one for each original sentence. ~~List[Example]~~ | -## Alignment {#alignment-object new="3"} +## Alignment {id="alignment-object",version="3"} Calculate alignment tables between two tokenizations. -### Alignment attributes {#alignment-attributes"} +### Alignment attributes {id="alignment-attributes"} -Alignment attributes are managed using `AlignmentArray`, which is a -simplified version of Thinc's [Ragged](https://thinc.ai/docs/api-types#ragged) -type that only supports the `data` and `length` attributes. +Alignment attributes are managed using `AlignmentArray`, which is a simplified +version of Thinc's [Ragged](https://thinc.ai/docs/api-types#ragged) type that +only supports the `data` and `length` attributes. | Name | Description | | ----- | ------------------------------------------------------------------------------------- | @@ -321,7 +321,7 @@ tokenizations add up to the same string. For example, you'll be able to align > If `a2b.data[1] == a2b.data[2] == 1`, that means that `A[1]` (`"'"`) and > `A[2]` (`"s"`) both align to `B[1]` (`"'s"`). -### Alignment.from_strings {#classmethod tag="function"} +### Alignment.from_strings {id="classmethod",tag="function"} | Name | Description | | ----------- | ------------------------------------------------------------- | diff --git a/website/docs/api/index.md b/website/docs/api/index.mdx similarity index 58% rename from website/docs/api/index.md rename to website/docs/api/index.mdx index a9dc408f6..6c6e1fff4 100644 --- a/website/docs/api/index.md +++ b/website/docs/api/index.mdx @@ -3,6 +3,4 @@ title: Library Architecture next: /api/architectures --- -import Architecture101 from 'usage/101/\_architecture.md' - diff --git a/website/docs/api/kb_in_memory.md b/website/docs/api/inmemorylookupkb.mdx similarity index 86% rename from website/docs/api/kb_in_memory.md rename to website/docs/api/inmemorylookupkb.mdx index 9e3279e6a..c24fe78d6 100644 --- a/website/docs/api/kb_in_memory.md +++ b/website/docs/api/inmemorylookupkb.mdx @@ -5,7 +5,7 @@ teaser: information in-memory. tag: class source: spacy/kb/kb_in_memory.pyx -new: 3.5 +version: 3.5 --- The `InMemoryLookupKB` class inherits from [`KnowledgeBase`](/api/kb) and @@ -14,7 +14,7 @@ implements all of its methods. It stores all KB data in-memory and generates entity names. It's highly optimized for both a low memory footprint and speed of retrieval. -## InMemoryLookupKB.\_\_init\_\_ {#init tag="method"} +## InMemoryLookupKB.\_\_init\_\_ {id="init",tag="method"} Create the knowledge base. @@ -31,7 +31,7 @@ Create the knowledge base. | `vocab` | The shared vocabulary. ~~Vocab~~ | | `entity_vector_length` | Length of the fixed-size entity vectors. ~~int~~ | -## InMemoryLookupKB.entity_vector_length {#entity_vector_length tag="property"} +## InMemoryLookupKB.entity_vector_length {id="entity_vector_length",tag="property"} The length of the fixed-size entity vectors in the knowledge base. @@ -39,11 +39,11 @@ The length of the fixed-size entity vectors in the knowledge base. | ----------- | ------------------------------------------------ | | **RETURNS** | Length of the fixed-size entity vectors. ~~int~~ | -## InMemoryLookupKB.add_entity {#add_entity tag="method"} +## InMemoryLookupKB.add_entity {id="add_entity",tag="method"} Add an entity to the knowledge base, specifying its corpus frequency and entity vector, which should be of length -[`entity_vector_length`](/api/kb_in_memory#entity_vector_length). +[`entity_vector_length`](/api/inmemorylookupkb#entity_vector_length). > #### Example > @@ -58,7 +58,7 @@ vector, which should be of length | `freq` | The frequency of the entity in a typical corpus. ~~float~~ | | `entity_vector` | The pretrained vector of the entity. ~~numpy.ndarray~~ | -## InMemoryLookupKB.set_entities {#set_entities tag="method"} +## InMemoryLookupKB.set_entities {id="set_entities",tag="method"} Define the full list of entities in the knowledge base, specifying the corpus frequency and entity vector for each entity. @@ -75,12 +75,13 @@ frequency and entity vector for each entity. | `freq_list` | List of entity frequencies. ~~Iterable[int]~~ | | `vector_list` | List of entity vectors. ~~Iterable[numpy.ndarray]~~ | -## InMemoryLookupKB.add_alias {#add_alias tag="method"} +## InMemoryLookupKB.add_alias {id="add_alias",tag="method"} Add an alias or mention to the knowledge base, specifying its potential KB identifiers and their prior probabilities. The entity identifiers should refer -to entities previously added with [`add_entity`](/api/kb_in_memory#add_entity) -or [`set_entities`](/api/kb_in_memory#set_entities). The sum of the prior +to entities previously added with +[`add_entity`](/api/inmemorylookupkb#add_entity) or +[`set_entities`](/api/inmemorylookupkb#set_entities). The sum of the prior probabilities should not exceed 1. Note that an empty string can not be used as alias. @@ -96,7 +97,7 @@ alias. | `entities` | The potential entities that the alias may refer to. ~~Iterable[Union[str, int]]~~ | | `probabilities` | The prior probabilities of each entity. ~~Iterable[float]~~ | -## InMemoryLookupKB.\_\_len\_\_ {#len tag="method"} +## InMemoryLookupKB.\_\_len\_\_ {id="len",tag="method"} Get the total number of entities in the knowledge base. @@ -110,7 +111,7 @@ Get the total number of entities in the knowledge base. | ----------- | ----------------------------------------------------- | | **RETURNS** | The number of entities in the knowledge base. ~~int~~ | -## InMemoryLookupKB.get_entity_strings {#get_entity_strings tag="method"} +## InMemoryLookupKB.get_entity_strings {id="get_entity_strings",tag="method"} Get a list of all entity IDs in the knowledge base. @@ -124,7 +125,7 @@ Get a list of all entity IDs in the knowledge base. | ----------- | --------------------------------------------------------- | | **RETURNS** | The list of entities in the knowledge base. ~~List[str]~~ | -## InMemoryLookupKB.get_size_aliases {#get_size_aliases tag="method"} +## InMemoryLookupKB.get_size_aliases {id="get_size_aliases",tag="method"} Get the total number of aliases in the knowledge base. @@ -138,7 +139,7 @@ Get the total number of aliases in the knowledge base. | ----------- | ---------------------------------------------------- | | **RETURNS** | The number of aliases in the knowledge base. ~~int~~ | -## InMemoryLookupKB.get_alias_strings {#get_alias_strings tag="method"} +## InMemoryLookupKB.get_alias_strings {id="get_alias_strings",tag="method"} Get a list of all aliases in the knowledge base. @@ -152,11 +153,11 @@ Get a list of all aliases in the knowledge base. | ----------- | -------------------------------------------------------- | | **RETURNS** | The list of aliases in the knowledge base. ~~List[str]~~ | -## InMemoryLookupKB.get_candidates {#get_candidates tag="method"} +## InMemoryLookupKB.get_candidates {id="get_candidates",tag="method"} Given a certain textual mention as input, retrieve a list of candidate entities of type [`Candidate`](/api/kb#candidate). Wraps -[`get_alias_candidates()`](/api/kb_in_memory#get_alias_candidates). +[`get_alias_candidates()`](/api/inmemorylookupkb#get_alias_candidates). > #### Example > @@ -172,9 +173,9 @@ of type [`Candidate`](/api/kb#candidate). Wraps | `mention` | The textual mention or alias. ~~Span~~ | | **RETURNS** | An iterable of relevant `Candidate` objects. ~~Iterable[Candidate]~~ | -## InMemoryLookupKB.get_candidates_batch {#get_candidates_batch tag="method"} +## InMemoryLookupKB.get_candidates_batch {id="get_candidates_batch",tag="method"} -Same as [`get_candidates()`](/api/kb_in_memory#get_candidates), but for an +Same as [`get_candidates()`](/api/inmemorylookupkb#get_candidates), but for an arbitrary number of mentions. The [`EntityLinker`](/api/entitylinker) component will call `get_candidates_batch()` instead of `get_candidates()`, if the config parameter `candidates_batch_size` is greater or equal than 1. @@ -198,7 +199,7 @@ to you. | `mentions` | The textual mention or alias. ~~Iterable[Span]~~ | | **RETURNS** | An iterable of iterable with relevant `Candidate` objects. ~~Iterable[Iterable[Candidate]]~~ | -## InMemoryLookupKB.get_alias_candidates {#get_alias_candidates tag="method"} +## InMemoryLookupKB.get_alias_candidates {id="get_alias_candidates",tag="method"} Given a certain textual mention as input, retrieve a list of candidate entities of type [`Candidate`](/api/kb#candidate). @@ -214,7 +215,7 @@ of type [`Candidate`](/api/kb#candidate). | `alias` | The textual mention or alias. ~~str~~ | | **RETURNS** | The list of relevant `Candidate` objects. ~~List[Candidate]~~ | -## InMemoryLookupKB.get_vector {#get_vector tag="method"} +## InMemoryLookupKB.get_vector {id="get_vector",tag="method"} Given a certain entity ID, retrieve its pretrained entity vector. @@ -229,9 +230,9 @@ Given a certain entity ID, retrieve its pretrained entity vector. | `entity` | The entity ID. ~~str~~ | | **RETURNS** | The entity vector. ~~numpy.ndarray~~ | -## InMemoryLookupKB.get_vectors {#get_vectors tag="method"} +## InMemoryLookupKB.get_vectors {id="get_vectors",tag="method"} -Same as [`get_vector()`](/api/kb_in_memory#get_vector), but for an arbitrary +Same as [`get_vector()`](/api/inmemorylookupkb#get_vector), but for an arbitrary number of entity IDs. The default implementation of `get_vectors()` executes `get_vector()` in a loop. @@ -249,7 +250,7 @@ entities at once, if performance is of concern to you. | `entities` | The entity IDs. ~~Iterable[str]~~ | | **RETURNS** | The entity vectors. ~~Iterable[Iterable[numpy.ndarray]]~~ | -## InMemoryLookupKB.get_prior_prob {#get_prior_prob tag="method"} +## InMemoryLookupKB.get_prior_prob {id="get_prior_prob",tag="method"} Given a certain entity ID and a certain textual mention, retrieve the prior probability of the fact that the mention links to the entity ID. @@ -266,7 +267,7 @@ probability of the fact that the mention links to the entity ID. | `alias` | The textual mention or alias. ~~str~~ | | **RETURNS** | The prior probability of the `alias` referring to the `entity`. ~~float~~ | -## InMemoryLookupKB.to_disk {#to_disk tag="method"} +## InMemoryLookupKB.to_disk {id="to_disk",tag="method"} Save the current state of the knowledge base to a directory. @@ -281,7 +282,7 @@ Save the current state of the knowledge base to a directory. | `path` | A path to a directory, which will be created if it doesn't exist. Paths may be either strings or `Path`-like objects. ~~Union[str, Path]~~ | | `exclude` | List of components to exclude. ~~Iterable[str]~~ | -## InMemoryLookupKB.from_disk {#from_disk tag="method"} +## InMemoryLookupKB.from_disk {id="from_disk",tag="method"} Restore the state of the knowledge base from a given directory. Note that the [`Vocab`](/api/vocab) should also be the same as the one used to create the KB. diff --git a/website/docs/api/kb.md b/website/docs/api/kb.mdx similarity index 85% rename from website/docs/api/kb.md rename to website/docs/api/kb.mdx index b217a1678..2b0d4d9d6 100644 --- a/website/docs/api/kb.md +++ b/website/docs/api/kb.mdx @@ -5,7 +5,7 @@ teaser: (ontology) tag: class source: spacy/kb/kb.pyx -new: 2.2 +version: 2.2 --- The `KnowledgeBase` object is an abstract class providing a method to generate @@ -21,12 +21,12 @@ functions called by the [`EntityLinker`](/api/entitylinker) component. This class was not abstract up to spaCy version 3.5. The `KnowledgeBase` -implementation up to that point is available as `InMemoryLookupKB` from 3.5 -onwards. +implementation up to that point is available as +[`InMemoryLookupKB`](/api/inmemorylookupkb) from 3.5 onwards. -## KnowledgeBase.\_\_init\_\_ {#init tag="method"} +## KnowledgeBase.\_\_init\_\_ {id="init",tag="method"} `KnowledgeBase` is an abstract class and cannot be instantiated. Its child classes should call `__init__()` to set up some necessary attributes. @@ -50,7 +50,7 @@ classes should call `__init__()` to set up some necessary attributes. | `vocab` | The shared vocabulary. ~~Vocab~~ | | `entity_vector_length` | Length of the fixed-size entity vectors. ~~int~~ | -## KnowledgeBase.entity_vector_length {#entity_vector_length tag="property"} +## KnowledgeBase.entity_vector_length {id="entity_vector_length",tag="property"} The length of the fixed-size entity vectors in the knowledge base. @@ -58,7 +58,7 @@ The length of the fixed-size entity vectors in the knowledge base. | ----------- | ------------------------------------------------ | | **RETURNS** | Length of the fixed-size entity vectors. ~~int~~ | -## KnowledgeBase.get_candidates {#get_candidates tag="method"} +## KnowledgeBase.get_candidates {id="get_candidates",tag="method"} Given a certain textual mention as input, retrieve a list of candidate entities of type [`Candidate`](/api/kb#candidate). @@ -77,7 +77,7 @@ of type [`Candidate`](/api/kb#candidate). | `mention` | The textual mention or alias. ~~Span~~ | | **RETURNS** | An iterable of relevant `Candidate` objects. ~~Iterable[Candidate]~~ | -## KnowledgeBase.get_candidates_batch {#get_candidates_batch tag="method"} +## KnowledgeBase.get_candidates_batch {id="get_candidates_batch",tag="method"} Same as [`get_candidates()`](/api/kb#get_candidates), but for an arbitrary number of mentions. The [`EntityLinker`](/api/entitylinker) component will call @@ -103,23 +103,24 @@ to you. | `mentions` | The textual mention or alias. ~~Iterable[Span]~~ | | **RETURNS** | An iterable of iterable with relevant `Candidate` objects. ~~Iterable[Iterable[Candidate]]~~ | -## KnowledgeBase.get_alias_candidates {#get_alias_candidates tag="method"} +## KnowledgeBase.get_alias_candidates {id="get_alias_candidates",tag="method"} -This method is _not_ available from spaCy 3.5 onwards. + This method is _not_ available from spaCy 3.5 onwards. From spaCy 3.5 on `KnowledgeBase` is an abstract class (with -[`InMemoryLookupKB`](/api/kb_in_memory) being a drop-in replacement) to allow -more flexibility in customizing knowledge bases. Some of its methods were moved -to [`InMemoryLookupKB`](/api/kb_in_memory) during this refactoring, one of those -being `get_alias_candidates()`. This method is now available as -[`InMemoryLookupKB.get_alias_candidates()`](/api/kb_in_memory#get_alias_candidates). -Note: [`InMemoryLookupKB.get_candidates()`](/api/kb_in_memory#get_candidates) +[`InMemoryLookupKB`](/api/inmemorylookupkb) being a drop-in replacement) to +allow more flexibility in customizing knowledge bases. Some of its methods were +moved to [`InMemoryLookupKB`](/api/inmemorylookupkb) during this refactoring, +one of those being `get_alias_candidates()`. This method is now available as +[`InMemoryLookupKB.get_alias_candidates()`](/api/inmemorylookupkb#get_alias_candidates). +Note: +[`InMemoryLookupKB.get_candidates()`](/api/inmemorylookupkb#get_candidates) defaults to -[`InMemoryLookupKB.get_alias_candidates()`](/api/kb_in_memory#get_alias_candidates). +[`InMemoryLookupKB.get_alias_candidates()`](/api/inmemorylookupkb#get_alias_candidates). -## KnowledgeBase.get_vector {#get_vector tag="method"} +## KnowledgeBase.get_vector {id="get_vector",tag="method"} Given a certain entity ID, retrieve its pretrained entity vector. @@ -134,7 +135,7 @@ Given a certain entity ID, retrieve its pretrained entity vector. | `entity` | The entity ID. ~~str~~ | | **RETURNS** | The entity vector. ~~Iterable[float]~~ | -## KnowledgeBase.get_vectors {#get_vectors tag="method"} +## KnowledgeBase.get_vectors {id="get_vectors",tag="method"} Same as [`get_vector()`](/api/kb#get_vector), but for an arbitrary number of entity IDs. @@ -154,7 +155,7 @@ entities at once, if performance is of concern to you. | `entities` | The entity IDs. ~~Iterable[str]~~ | | **RETURNS** | The entity vectors. ~~Iterable[Iterable[numpy.ndarray]]~~ | -## KnowledgeBase.to_disk {#to_disk tag="method"} +## KnowledgeBase.to_disk {id="to_disk",tag="method"} Save the current state of the knowledge base to a directory. @@ -169,7 +170,7 @@ Save the current state of the knowledge base to a directory. | `path` | A path to a directory, which will be created if it doesn't exist. Paths may be either strings or `Path`-like objects. ~~Union[str, Path]~~ | | `exclude` | List of components to exclude. ~~Iterable[str]~~ | -## KnowledgeBase.from_disk {#from_disk tag="method"} +## KnowledgeBase.from_disk {id="from_disk",tag="method"} Restore the state of the knowledge base from a given directory. Note that the [`Vocab`](/api/vocab) should also be the same as the one used to create the KB. @@ -189,7 +190,7 @@ Restore the state of the knowledge base from a given directory. Note that the | `exclude` | List of components to exclude. ~~Iterable[str]~~ | | **RETURNS** | The modified `KnowledgeBase` object. ~~KnowledgeBase~~ | -## Candidate {#candidate tag="class"} +## Candidate {id="candidate",tag="class"} A `Candidate` object refers to a textual mention (alias) that may or may not be resolved to a specific entity from a `KnowledgeBase`. This will be used as input @@ -197,7 +198,7 @@ for the entity linking algorithm which will disambiguate the various candidates to the correct one. Each candidate `(alias, entity)` pair is assigned to a certain prior probability. -### Candidate.\_\_init\_\_ {#candidate-init tag="method"} +### Candidate.\_\_init\_\_ {id="candidate-init",tag="method"} Construct a `Candidate` object. Usually this constructor is not called directly, but instead these objects are returned by the `get_candidates` method of the @@ -218,7 +219,7 @@ but instead these objects are returned by the `get_candidates` method of the | `alias_hash` | The hash of the textual mention or alias. ~~int~~ | | `prior_prob` | The prior probability of the `alias` referring to the `entity`. ~~float~~ | -## Candidate attributes {#candidate-attributes} +## Candidate attributes {id="candidate-attributes"} | Name | Description | | --------------- | ------------------------------------------------------------------------ | diff --git a/website/docs/api/language.md b/website/docs/api/language.mdx similarity index 96% rename from website/docs/api/language.md rename to website/docs/api/language.mdx index ad0ac2a46..93ddd79a2 100644 --- a/website/docs/api/language.md +++ b/website/docs/api/language.mdx @@ -15,7 +15,7 @@ the tagger or parser that are called on a document in order. You can also add your own processing pipeline components that take a `Doc` object, modify it and return it. -## Language.\_\_init\_\_ {#init tag="method"} +## Language.\_\_init\_\_ {id="init",tag="method"} Initialize a `Language` object. Note that the `meta` is only used for meta information in [`Language.meta`](/api/language#meta) and not to configure the @@ -44,7 +44,7 @@ information in [`Language.meta`](/api/language#meta) and not to configure the | `create_tokenizer` | Optional function that receives the `nlp` object and returns a tokenizer. ~~Callable[[Language], Callable[[str], Doc]]~~ | | `batch_size` | Default batch size for [`pipe`](#pipe) and [`evaluate`](#evaluate). Defaults to `1000`. ~~int~~ | -## Language.from_config {#from_config tag="classmethod" new="3"} +## Language.from_config {id="from_config",tag="classmethod",version="3"} Create a `Language` object from a loaded config. Will set up the tokenizer and language data, add pipeline components based on the pipeline and add pipeline @@ -76,7 +76,7 @@ spaCy loads a model under the hood based on its | `validate` | Whether to validate the component config and arguments against the types expected by the factory. Defaults to `True`. ~~bool~~ | | **RETURNS** | The initialized object. ~~Language~~ | -## Language.component {#component tag="classmethod" new="3"} +## Language.component {id="component",tag="classmethod",version="3"} Register a custom pipeline component under a given name. This allows initializing the component by name using @@ -112,7 +112,7 @@ decorator. For more details and examples, see the | `retokenizes` | Whether the component changes tokenization. Used for [pipe analysis](/usage/processing-pipelines#analysis). ~~bool~~ | | `func` | Optional function if not used as a decorator. ~~Optional[Callable[[Doc], Doc]]~~ | -## Language.factory {#factory tag="classmethod"} +## Language.factory {id="factory",tag="classmethod"} Register a custom pipeline component factory under a given name. This allows initializing the component by name using @@ -159,7 +159,7 @@ examples, see the | `default_score_weights` | The scores to report during training, and their default weight towards the final score used to select the best model. Weights should sum to `1.0` per component and will be combined and normalized for the whole pipeline. If a weight is set to `None`, the score will not be logged or weighted. ~~Dict[str, Optional[float]]~~ | | `func` | Optional function if not used as a decorator. ~~Optional[Callable[[...], Callable[[Doc], Doc]]]~~ | -## Language.\_\_call\_\_ {#call tag="method"} +## Language.\_\_call\_\_ {id="call",tag="method"} Apply the pipeline to some text. The text can span multiple sentences, and can contain arbitrary whitespace. Alignment into the original string is preserved. @@ -182,7 +182,7 @@ skipped, but the rest of the pipeline is run. | `component_cfg` | Optional dictionary of keyword arguments for components, keyed by component names. Defaults to `None`. ~~Optional[Dict[str, Dict[str, Any]]]~~ | | **RETURNS** | A container for accessing the annotations. ~~Doc~~ | -## Language.pipe {#pipe tag="method"} +## Language.pipe {id="pipe",tag="method"} Process texts as a stream, and yield `Doc` objects in order. This is usually more efficient than processing texts one-by-one. @@ -209,7 +209,7 @@ tokenization is skipped but the rest of the pipeline is run. | `n_process` | Number of processors to use. Defaults to `1`. ~~int~~ | | **YIELDS** | Documents in the order of the original text. ~~Doc~~ | -## Language.set_error_handler {#set_error_handler tag="method" new="3"} +## Language.set_error_handler {id="set_error_handler",tag="method",version="3"} Define a callback that will be invoked when an error is thrown during processing of one or more documents. Specifically, this function will call @@ -231,7 +231,7 @@ being processed, and the original error. | --------------- | -------------------------------------------------------------------------------------------------------------- | | `error_handler` | A function that performs custom error handling. ~~Callable[[str, Callable[[Doc], Doc], List[Doc], Exception]~~ | -## Language.initialize {#initialize tag="method" new="3"} +## Language.initialize {id="initialize",tag="method",version="3"} Initialize the pipeline for training and return an [`Optimizer`](https://thinc.ai/docs/api-optimizers). Under the hood, it uses the @@ -282,7 +282,7 @@ objects. | `sgd` | An optimizer. Will be created via [`create_optimizer`](#create_optimizer) if not set. ~~Optional[Optimizer]~~ | | **RETURNS** | The optimizer. ~~Optimizer~~ | -## Language.resume_training {#resume_training tag="method,experimental" new="3"} +## Language.resume_training {id="resume_training",tag="method,experimental",version="3"} Continue training a trained pipeline. Create and return an optimizer, and initialize "rehearsal" for any pipeline component that has a `rehearse` method. @@ -304,7 +304,7 @@ a batch of [Example](/api/example) objects. | `sgd` | An optimizer. Will be created via [`create_optimizer`](#create_optimizer) if not set. ~~Optional[Optimizer]~~ | | **RETURNS** | The optimizer. ~~Optimizer~~ | -## Language.update {#update tag="method"} +## Language.update {id="update",tag="method"} Update the models in the pipeline. @@ -342,7 +342,7 @@ and custom registered functions if needed. See the | `component_cfg` | Optional dictionary of keyword arguments for components, keyed by component names. Defaults to `None`. ~~Optional[Dict[str, Dict[str, Any]]]~~ | | **RETURNS** | The updated `losses` dictionary. ~~Dict[str, float]~~ | -## Language.rehearse {#rehearse tag="method,experimental" new="3"} +## Language.rehearse {id="rehearse",tag="method,experimental",version="3"} Perform a "rehearsal" update from a batch of data. Rehearsal updates teach the current model to make predictions similar to an initial model, to try to address @@ -364,7 +364,7 @@ the "catastrophic forgetting" problem. This feature is experimental. | `losses` | Dictionary to update with the loss, keyed by pipeline component. ~~Optional[Dict[str, float]]~~ | | **RETURNS** | The updated `losses` dictionary. ~~Dict[str, float]~~ | -## Language.evaluate {#evaluate tag="method"} +## Language.evaluate {id="evaluate",tag="method"} Evaluate a pipeline's components. @@ -392,7 +392,7 @@ objects instead of tuples of `Doc` and `GoldParse` objects. | `scorer_cfg` | Optional dictionary of keyword arguments for the `Scorer`. Defaults to `None`. ~~Optional[Dict[str, Any]]~~ | | **RETURNS** | A dictionary of evaluation scores. ~~Dict[str, Union[float, Dict[str, float]]]~~ | -## Language.use_params {#use_params tag="contextmanager, method"} +## Language.use_params {id="use_params",tag="contextmanager, method"} Replace weights of models in the pipeline with those provided in the params dictionary. Can be used as a context manager, in which case, models go back to @@ -409,7 +409,7 @@ their original weights after the block. | -------- | ------------------------------------------------------ | | `params` | A dictionary of parameters keyed by model ID. ~~dict~~ | -## Language.add_pipe {#add_pipe tag="method" new="2"} +## Language.add_pipe {id="add_pipe",tag="method",version="2"} Add a component to the processing pipeline. Expects a name that maps to a component factory registered using @@ -458,7 +458,7 @@ component, adds it to the pipeline and returns it. | `validate` 3 | Whether to validate the component config and arguments against the types expected by the factory. Defaults to `True`. ~~bool~~ | | **RETURNS** | The pipeline component. ~~Callable[[Doc], Doc]~~ | -## Language.create_pipe {#create_pipe tag="method" new="2"} +## Language.create_pipe {id="create_pipe",tag="method",version="2"} Create a pipeline component from a factory. @@ -487,7 +487,7 @@ To create a component and add it to the pipeline, you should always use | `validate` 3 | Whether to validate the component config and arguments against the types expected by the factory. Defaults to `True`. ~~bool~~ | | **RETURNS** | The pipeline component. ~~Callable[[Doc], Doc]~~ | -## Language.has_factory {#has_factory tag="classmethod" new="3"} +## Language.has_factory {id="has_factory",tag="classmethod",version="3"} Check whether a factory name is registered on the `Language` class or subclass. Will check for @@ -514,7 +514,7 @@ the `Language` base class, available to all subclasses. | `name` | Name of the pipeline factory to check. ~~str~~ | | **RETURNS** | Whether a factory of that name is registered on the class. ~~bool~~ | -## Language.has_pipe {#has_pipe tag="method" new="2"} +## Language.has_pipe {id="has_pipe",tag="method",version="2"} Check whether a component is present in the pipeline. Equivalent to `name in nlp.pipe_names`. @@ -536,7 +536,7 @@ Check whether a component is present in the pipeline. Equivalent to | `name` | Name of the pipeline component to check. ~~str~~ | | **RETURNS** | Whether a component of that name exists in the pipeline. ~~bool~~ | -## Language.get_pipe {#get_pipe tag="method" new="2"} +## Language.get_pipe {id="get_pipe",tag="method",version="2"} Get a pipeline component for a given component name. @@ -552,7 +552,7 @@ Get a pipeline component for a given component name. | `name` | Name of the pipeline component to get. ~~str~~ | | **RETURNS** | The pipeline component. ~~Callable[[Doc], Doc]~~ | -## Language.replace_pipe {#replace_pipe tag="method" new="2"} +## Language.replace_pipe {id="replace_pipe",tag="method",version="2"} Replace a component in the pipeline and return the new component. @@ -580,7 +580,7 @@ and instead expects the **name of a component factory** registered using | `validate` 3 | Whether to validate the component config and arguments against the types expected by the factory. Defaults to `True`. ~~bool~~ | | **RETURNS** | The new pipeline component. ~~Callable[[Doc], Doc]~~ | -## Language.rename_pipe {#rename_pipe tag="method" new="2"} +## Language.rename_pipe {id="rename_pipe",tag="method",version="2"} Rename a component in the pipeline. Useful to create custom names for pre-defined and pre-loaded components. To change the default name of a component @@ -598,7 +598,7 @@ added to the pipeline, you can also use the `name` argument on | `old_name` | Name of the component to rename. ~~str~~ | | `new_name` | New name of the component. ~~str~~ | -## Language.remove_pipe {#remove_pipe tag="method" new="2"} +## Language.remove_pipe {id="remove_pipe",tag="method",version="2"} Remove a component from the pipeline. Returns the removed component name and component function. @@ -615,7 +615,7 @@ component function. | `name` | Name of the component to remove. ~~str~~ | | **RETURNS** | A `(name, component)` tuple of the removed component. ~~Tuple[str, Callable[[Doc], Doc]]~~ | -## Language.disable_pipe {#disable_pipe tag="method" new="3"} +## Language.disable_pipe {id="disable_pipe",tag="method",version="3"} Temporarily disable a pipeline component so it's not run as part of the pipeline. Disabled components are listed in @@ -641,7 +641,7 @@ does nothing. | ------ | ----------------------------------------- | | `name` | Name of the component to disable. ~~str~~ | -## Language.enable_pipe {#enable_pipe tag="method" new="3"} +## Language.enable_pipe {id="enable_pipe",tag="method",version="3"} Enable a previously disabled component (e.g. via [`Language.disable_pipes`](/api/language#disable_pipes)) so it's run as part of @@ -663,7 +663,7 @@ already enabled, this method does nothing. | ------ | ---------------------------------------- | | `name` | Name of the component to enable. ~~str~~ | -## Language.select_pipes {#select_pipes tag="contextmanager, method" new="3"} +## Language.select_pipes {id="select_pipes",tag="contextmanager, method",version="3"} Disable one or more pipeline components. If used as a context manager, the pipeline will be restored to the initial state at the end of the block. @@ -706,7 +706,7 @@ As of spaCy v3.0, the `disable_pipes` method has been renamed to `select_pipes`: | `enable` | Name(s) of pipeline component(s) that will not be disabled. ~~Optional[Union[str, Iterable[str]]]~~ | | **RETURNS** | The disabled pipes that can be restored by calling the object's `.restore()` method. ~~DisabledPipes~~ | -## Language.get_factory_meta {#get_factory_meta tag="classmethod" new="3"} +## Language.get_factory_meta {id="get_factory_meta",tag="classmethod",version="3"} Get the factory meta information for a given pipeline component name. Expects the name of the component **factory**. The factory meta is an instance of the @@ -728,7 +728,7 @@ information about the component and its default provided by the | `name` | The factory name. ~~str~~ | | **RETURNS** | The factory meta. ~~FactoryMeta~~ | -## Language.get_pipe_meta {#get_pipe_meta tag="method" new="3"} +## Language.get_pipe_meta {id="get_pipe_meta",tag="method",version="3"} Get the factory meta information for a given pipeline component name. Expects the name of the component **instance** in the pipeline. The factory meta is an @@ -751,7 +751,7 @@ contains the information about the component and its default provided by the | `name` | The pipeline component name. ~~str~~ | | **RETURNS** | The factory meta. ~~FactoryMeta~~ | -## Language.analyze_pipes {#analyze_pipes tag="method" new="3"} +## Language.analyze_pipes {id="analyze_pipes",tag="method",version="3"} Analyze the current pipeline components and show a summary of the attributes they assign and require, and the scores they set. The data is based on the @@ -780,8 +780,7 @@ doesn't, the pipeline analysis won't catch that. -```json -### Structured +```json {title="Structured"} { "summary": { "tagger": { @@ -799,7 +798,12 @@ doesn't, the pipeline analysis won't catch that. }, "problems": { "tagger": [], - "entity_linker": ["doc.ents", "doc.sents", "token.ent_iob", "token.ent_type"] + "entity_linker": [ + "doc.ents", + "doc.sents", + "token.ent_iob", + "token.ent_type" + ] }, "attrs": { "token.ent_iob": { "assigns": [], "requires": ["entity_linker"] }, @@ -840,7 +844,7 @@ token.ent_iob, token.ent_type | `pretty` | Pretty-print the results as a table. Defaults to `False`. ~~bool~~ | | **RETURNS** | Dictionary containing the pipe analysis, keyed by `"summary"` (component meta by pipe), `"problems"` (attribute names by pipe) and `"attrs"` (pipes that assign and require an attribute, keyed by attribute). ~~Optional[Dict[str, Any]]~~ | -## Language.replace_listeners {#replace_listeners tag="method" new="3"} +## Language.replace_listeners {id="replace_listeners",tag="method",version="3"} Find [listener layers](/usage/embeddings-transformers#embedding-layers) (connecting to a shared token-to-vector embedding component) of a given pipeline @@ -885,7 +889,7 @@ when loading a config with | `pipe_name` | Name of pipeline component to replace listeners for. ~~str~~ | | `listeners` | The paths to the listeners, relative to the component config, e.g. `["model.tok2vec"]`. Typically, implementations will only connect to one tok2vec component, `model.tok2vec`, but in theory, custom models can use multiple listeners. The value here can either be an empty list to not replace any listeners, or a _complete_ list of the paths to all listener layers used by the model that should be replaced.~~Iterable[str]~~ | -## Language.meta {#meta tag="property"} +## Language.meta {id="meta",tag="property"} Meta data for the `Language` class, including name, version, data sources, license, author information and more. If a trained pipeline is loaded, this @@ -911,7 +915,7 @@ information is expressed in the [`config.cfg`](/api/data-formats#config). | ----------- | --------------------------------- | | **RETURNS** | The meta data. ~~Dict[str, Any]~~ | -## Language.config {#config tag="property" new="3"} +## Language.config {id="config",tag="property",version="3"} Export a trainable [`config.cfg`](/api/data-formats#config) for the current `nlp` object. Includes the current pipeline, all configs used to create the @@ -932,7 +936,7 @@ subclass of the built-in `dict`. It supports the additional methods `to_disk` | ----------- | ---------------------- | | **RETURNS** | The config. ~~Config~~ | -## Language.to_disk {#to_disk tag="method" new="2"} +## Language.to_disk {id="to_disk",tag="method",version="2"} Save the current state to a directory. Under the hood, this method delegates to the `to_disk` methods of the individual pipeline components, if available. This @@ -951,7 +955,7 @@ will be saved to disk. | _keyword-only_ | | | `exclude` | Names of pipeline components or [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | -## Language.from_disk {#from_disk tag="method" new="2"} +## Language.from_disk {id="from_disk",tag="method",version="2"} Loads state from a directory, including all data that was saved with the `Language` object. Modifies the object in place and returns it. @@ -984,7 +988,7 @@ you want to load a serialized pipeline from a directory, you should use | `exclude` | Names of pipeline components or [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | | **RETURNS** | The modified `Language` object. ~~Language~~ | -## Language.to_bytes {#to_bytes tag="method"} +## Language.to_bytes {id="to_bytes",tag="method"} Serialize the current state to a binary string. @@ -1000,7 +1004,7 @@ Serialize the current state to a binary string. | `exclude` | Names of pipeline components or [serialization fields](#serialization-fields) to exclude. ~~iterable~~ | | **RETURNS** | The serialized form of the `Language` object. ~~bytes~~ | -## Language.from_bytes {#from_bytes tag="method"} +## Language.from_bytes {id="from_bytes",tag="method"} Load state from a binary string. Note that this method is commonly used via the subclasses like `English` or `German` to make language-specific functionality @@ -1028,7 +1032,7 @@ details. | `exclude` | Names of pipeline components or [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | | **RETURNS** | The `Language` object. ~~Language~~ | -## Attributes {#attributes} +## Attributes {id="attributes"} | Name | Description | | -------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------- | @@ -1046,7 +1050,7 @@ details. | `disabled` 3 | Names of components that are currently disabled and don't run as part of the pipeline. ~~List[str]~~ | | `path` | Path to the pipeline data directory, if a pipeline is loaded from a path or package. Otherwise `None`. ~~Optional[Path]~~ | -## Class attributes {#class-attributes} +## Class attributes {id="class-attributes"} | Name | Description | | ---------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | @@ -1054,7 +1058,7 @@ details. | `lang` | [IETF language tag](https://www.w3.org/International/articles/language-tags/), such as 'en' for English. ~~str~~ | | `default_config` | Base [config](/usage/training#config) to use for [Language.config](/api/language#config). Defaults to [`default_config.cfg`](%%GITHUB_SPACY/spacy/default_config.cfg). ~~Config~~ | -## Defaults {#defaults} +## Defaults {id="defaults"} The following attributes can be set on the `Language.Defaults` class to customize the default language data: @@ -1097,7 +1101,7 @@ customize the default language data: | `writing_system` | Information about the language's writing system, available via `Vocab.writing_system`. Defaults to: `{"direction": "ltr", "has_case": True, "has_letters": True}.`.
**Example:** [`zh/__init__.py`](%%GITHUB_SPACY/spacy/lang/zh/__init__.py) ~~Dict[str, Any]~~ | | `config` | Default [config](/usage/training#config) added to `nlp.config`. This can include references to custom tokenizers or lemmatizers.
**Example:** [`zh/__init__.py`](%%GITHUB_SPACY/spacy/lang/zh/__init__.py) ~~Config~~ | -## Serialization fields {#serialization-fields} +## Serialization fields {id="serialization-fields"} During serialization, spaCy will export several data fields used to restore different aspects of the object. If needed, you can exclude them from @@ -1117,7 +1121,7 @@ serialization by passing in the string names via the `exclude` argument. | `meta` | The meta data, available as [`Language.meta`](/api/language#meta). | | ... | String names of pipeline components, e.g. `"ner"`. | -## FactoryMeta {#factorymeta new="3" tag="dataclass"} +## FactoryMeta {id="factorymeta",version="3",tag="dataclass"} The `FactoryMeta` contains the information about the component and its default provided by the [`@Language.component`](/api/language#component) or diff --git a/website/docs/api/legacy.md b/website/docs/api/legacy.mdx similarity index 95% rename from website/docs/api/legacy.md rename to website/docs/api/legacy.mdx index d9167c76f..ea6d3a899 100644 --- a/website/docs/api/legacy.md +++ b/website/docs/api/legacy.mdx @@ -12,11 +12,11 @@ functions that may still be used in projects. You can find the detailed documentation of each such legacy function on this page. -## Architectures {#architectures} +## Architectures {id="architectures"} These functions are available from `@spacy.registry.architectures`. -### spacy.Tok2Vec.v1 {#Tok2Vec_v1} +### spacy.Tok2Vec.v1 {id="Tok2Vec_v1"} The `spacy.Tok2Vec.v1` architecture was expecting an `encode` model of type `Model[Floats2D, Floats2D]` such as `spacy.MaxoutWindowEncoder.v1` or @@ -48,7 +48,7 @@ blog post for background. | `encode` | Encode context into the embeddings, using an architecture such as a CNN, BiLSTM or transformer. For example, [MaxoutWindowEncoder.v1](/api/legacy#MaxoutWindowEncoder_v1). ~~Model[Floats2d, Floats2d]~~ | | **CREATES** | The model using the architecture. ~~Model[List[Doc], List[Floats2d]]~~ | -### spacy.MaxoutWindowEncoder.v1 {#MaxoutWindowEncoder_v1} +### spacy.MaxoutWindowEncoder.v1 {id="MaxoutWindowEncoder_v1"} The `spacy.MaxoutWindowEncoder.v1` architecture was producing a model of type `Model[Floats2D, Floats2D]`. Since `spacy.MaxoutWindowEncoder.v2`, this has been @@ -76,7 +76,7 @@ and residual connections. | `depth` | The number of convolutional layers. Recommended value is `4`. ~~int~~ | | **CREATES** | The model using the architecture. ~~Model[Floats2d, Floats2d]~~ | -### spacy.MishWindowEncoder.v1 {#MishWindowEncoder_v1} +### spacy.MishWindowEncoder.v1 {id="MishWindowEncoder_v1"} The `spacy.MishWindowEncoder.v1` architecture was producing a model of type `Model[Floats2D, Floats2D]`. Since `spacy.MishWindowEncoder.v2`, this has been @@ -103,24 +103,24 @@ and residual connections. | `depth` | The number of convolutional layers. Recommended value is `4`. ~~int~~ | | **CREATES** | The model using the architecture. ~~Model[Floats2d, Floats2d]~~ | -### spacy.HashEmbedCNN.v1 {#HashEmbedCNN_v1} +### spacy.HashEmbedCNN.v1 {id="HashEmbedCNN_v1"} Identical to [`spacy.HashEmbedCNN.v2`](/api/architectures#HashEmbedCNN) except using [`spacy.StaticVectors.v1`](#StaticVectors_v1) if vectors are included. -### spacy.MultiHashEmbed.v1 {#MultiHashEmbed_v1} +### spacy.MultiHashEmbed.v1 {id="MultiHashEmbed_v1"} Identical to [`spacy.MultiHashEmbed.v2`](/api/architectures#MultiHashEmbed) except with [`spacy.StaticVectors.v1`](#StaticVectors_v1) if vectors are included. -### spacy.CharacterEmbed.v1 {#CharacterEmbed_v1} +### spacy.CharacterEmbed.v1 {id="CharacterEmbed_v1"} Identical to [`spacy.CharacterEmbed.v2`](/api/architectures#CharacterEmbed) except using [`spacy.StaticVectors.v1`](#StaticVectors_v1) if vectors are included. -### spacy.TextCatEnsemble.v1 {#TextCatEnsemble_v1} +### spacy.TextCatEnsemble.v1 {id="TextCatEnsemble_v1"} The `spacy.TextCatEnsemble.v1` architecture built an internal `tok2vec` and `linear_model`. Since `spacy.TextCatEnsemble.v2`, this has been refactored so @@ -158,7 +158,7 @@ network has an internal CNN Tok2Vec layer and uses attention. | `nO` | Output dimension, determined by the number of different labels. If not set, the [`TextCategorizer`](/api/textcategorizer) component will set it when `initialize` is called. ~~Optional[int]~~ | | **CREATES** | The model using the architecture. ~~Model[List[Doc], Floats2d]~~ | -### spacy.TextCatCNN.v1 {#TextCatCNN_v1} +### spacy.TextCatCNN.v1 {id="TextCatCNN_v1"} Since `spacy.TextCatCNN.v2`, this architecture has become resizable, which means that you can add labels to a previously trained textcat. `TextCatCNN` v1 did not @@ -194,7 +194,7 @@ architecture is usually less accurate than the ensemble, but runs faster. | `nO` | Output dimension, determined by the number of different labels. If not set, the [`TextCategorizer`](/api/textcategorizer) component will set it when `initialize` is called. ~~Optional[int]~~ | | **CREATES** | The model using the architecture. ~~Model[List[Doc], Floats2d]~~ | -### spacy.TextCatBOW.v1 {#TextCatBOW_v1} +### spacy.TextCatBOW.v1 {id="TextCatBOW_v1"} Since `spacy.TextCatBOW.v2`, this architecture has become resizable, which means that you can add labels to a previously trained textcat. `TextCatBOW` v1 did not @@ -222,17 +222,17 @@ the others, but may not be as accurate, especially if texts are short. | `nO` | Output dimension, determined by the number of different labels. If not set, the [`TextCategorizer`](/api/textcategorizer) component will set it when `initialize` is called. ~~Optional[int]~~ | | **CREATES** | The model using the architecture. ~~Model[List[Doc], Floats2d]~~ | -### spacy.TransitionBasedParser.v1 {#TransitionBasedParser_v1} +### spacy.TransitionBasedParser.v1 {id="TransitionBasedParser_v1"} Identical to [`spacy.TransitionBasedParser.v2`](/api/architectures#TransitionBasedParser) except the `use_upper` was set to `True` by default. -## Layers {#layers} +## Layers {id="layers"} These functions are available from `@spacy.registry.layers`. -### spacy.StaticVectors.v1 {#StaticVectors_v1} +### spacy.StaticVectors.v1 {id="StaticVectors_v1"} Identical to [`spacy.StaticVectors.v2`](/api/architectures#StaticVectors) except for the handling of tokens without vectors. @@ -246,11 +246,11 @@ added to an existing vectors table. See more details in -## Loggers {#loggers} +## Loggers {id="loggers"} These functions are available from `@spacy.registry.loggers`. -### spacy.ConsoleLogger.v1 {#ConsoleLogger_v1} +### spacy.ConsoleLogger.v1 {id="ConsoleLogger_v1"} > #### Example config > @@ -264,7 +264,7 @@ Writes the results of a training step to the console in a tabular format. -```cli +```bash $ python -m spacy train config.cfg ``` diff --git a/website/docs/api/lemmatizer.md b/website/docs/api/lemmatizer.mdx similarity index 95% rename from website/docs/api/lemmatizer.md rename to website/docs/api/lemmatizer.mdx index 905096338..f6657dbf4 100644 --- a/website/docs/api/lemmatizer.md +++ b/website/docs/api/lemmatizer.mdx @@ -2,7 +2,7 @@ title: Lemmatizer tag: class source: spacy/pipeline/lemmatizer.py -new: 3 +version: 3 teaser: 'Pipeline component for lemmatization' api_string_name: lemmatizer api_trainable: false @@ -32,7 +32,7 @@ available in the pipeline and runs _before_ the lemmatizer. -## Assigned Attributes {#assigned-attributes} +## Assigned Attributes {id="assigned-attributes"} Lemmas generated by rules or predicted will be saved to `Token.lemma`. @@ -94,7 +94,7 @@ libraries (`pymorphy3`). %%GITHUB_SPACY/spacy/pipeline/lemmatizer.py ``` -## Lemmatizer.\_\_init\_\_ {#init tag="method"} +## Lemmatizer.\_\_init\_\_ {id="init",tag="method"} > #### Example > @@ -120,7 +120,7 @@ shortcut for this and instantiate the component using its string name and | mode | The lemmatizer mode, e.g. `"lookup"` or `"rule"`. Defaults to `"lookup"`. ~~str~~ | | overwrite | Whether to overwrite existing lemmas. ~~bool~~ | -## Lemmatizer.\_\_call\_\_ {#call tag="method"} +## Lemmatizer.\_\_call\_\_ {id="call",tag="method"} Apply the pipe to one document. The document is modified in place, and returned. This usually happens under the hood when the `nlp` object is called on a text @@ -140,7 +140,7 @@ and all pipeline components are applied to the `Doc` in order. | `doc` | The document to process. ~~Doc~~ | | **RETURNS** | The processed document. ~~Doc~~ | -## Lemmatizer.pipe {#pipe tag="method"} +## Lemmatizer.pipe {id="pipe",tag="method"} Apply the pipe to a stream of documents. This usually happens under the hood when the `nlp` object is called on a text and all pipeline components are @@ -161,7 +161,7 @@ applied to the `Doc` in order. | `batch_size` | The number of documents to buffer. Defaults to `128`. ~~int~~ | | **YIELDS** | The processed documents in order. ~~Doc~~ | -## Lemmatizer.initialize {#initialize tag="method"} +## Lemmatizer.initialize {id="initialize",tag="method"} Initialize the lemmatizer and load any data resources. This method is typically called by [`Language.initialize`](/api/language#initialize) and lets you @@ -192,7 +192,7 @@ training. At runtime, all data is loaded from disk. | `nlp` | The current `nlp` object. Defaults to `None`. ~~Optional[Language]~~ | | `lookups` | The lookups object containing the tables such as `"lemma_rules"`, `"lemma_index"`, `"lemma_exc"` and `"lemma_lookup"`. If `None`, default tables are loaded from [`spacy-lookups-data`](https://github.com/explosion/spacy-lookups-data). Defaults to `None`. ~~Optional[Lookups]~~ | -## Lemmatizer.lookup_lemmatize {#lookup_lemmatize tag="method"} +## Lemmatizer.lookup_lemmatize {id="lookup_lemmatize",tag="method"} Lemmatize a token using a lookup-based approach. If no lemma is found, the original string is returned. @@ -202,7 +202,7 @@ original string is returned. | `token` | The token to lemmatize. ~~Token~~ | | **RETURNS** | A list containing one or more lemmas. ~~List[str]~~ | -## Lemmatizer.rule_lemmatize {#rule_lemmatize tag="method"} +## Lemmatizer.rule_lemmatize {id="rule_lemmatize",tag="method"} Lemmatize a token using a rule-based approach. Typically relies on POS tags. @@ -211,7 +211,7 @@ Lemmatize a token using a rule-based approach. Typically relies on POS tags. | `token` | The token to lemmatize. ~~Token~~ | | **RETURNS** | A list containing one or more lemmas. ~~List[str]~~ | -## Lemmatizer.is_base_form {#is_base_form tag="method"} +## Lemmatizer.is_base_form {id="is_base_form",tag="method"} Check whether we're dealing with an uninflected paradigm, so we can avoid lemmatization entirely. @@ -221,7 +221,7 @@ lemmatization entirely. | `token` | The token to analyze. ~~Token~~ | | **RETURNS** | Whether the token's attributes (e.g., part-of-speech tag, morphological features) describe a base form. ~~bool~~ | -## Lemmatizer.get_lookups_config {#get_lookups_config tag="classmethod"} +## Lemmatizer.get_lookups_config {id="get_lookups_config",tag="classmethod"} Returns the lookups configuration settings for a given mode for use in [`Lemmatizer.load_lookups`](/api/lemmatizer#load_lookups). @@ -231,7 +231,7 @@ Returns the lookups configuration settings for a given mode for use in | `mode` | The lemmatizer mode. ~~str~~ | | **RETURNS** | The required table names and the optional table names. ~~Tuple[List[str], List[str]]~~ | -## Lemmatizer.to_disk {#to_disk tag="method"} +## Lemmatizer.to_disk {id="to_disk",tag="method"} Serialize the pipe to disk. @@ -248,7 +248,7 @@ Serialize the pipe to disk. | _keyword-only_ | | | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | -## Lemmatizer.from_disk {#from_disk tag="method"} +## Lemmatizer.from_disk {id="from_disk",tag="method"} Load the pipe from disk. Modifies the object in place and returns it. @@ -266,7 +266,7 @@ Load the pipe from disk. Modifies the object in place and returns it. | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | | **RETURNS** | The modified `Lemmatizer` object. ~~Lemmatizer~~ | -## Lemmatizer.to_bytes {#to_bytes tag="method"} +## Lemmatizer.to_bytes {id="to_bytes",tag="method"} > #### Example > @@ -283,7 +283,7 @@ Serialize the pipe to a bytestring. | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | | **RETURNS** | The serialized form of the `Lemmatizer` object. ~~bytes~~ | -## Lemmatizer.from_bytes {#from_bytes tag="method"} +## Lemmatizer.from_bytes {id="from_bytes",tag="method"} Load the pipe from a bytestring. Modifies the object in place and returns it. @@ -302,7 +302,7 @@ Load the pipe from a bytestring. Modifies the object in place and returns it. | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | | **RETURNS** | The `Lemmatizer` object. ~~Lemmatizer~~ | -## Attributes {#attributes} +## Attributes {id="attributes"} | Name | Description | | --------- | ------------------------------------------- | @@ -310,7 +310,7 @@ Load the pipe from a bytestring. Modifies the object in place and returns it. | `lookups` | The lookups object. ~~Lookups~~ | | `mode` | The lemmatizer mode. ~~str~~ | -## Serialization fields {#serialization-fields} +## Serialization fields {id="serialization-fields"} During serialization, spaCy will export several data fields used to restore different aspects of the object. If needed, you can exclude them from diff --git a/website/docs/api/lexeme.md b/website/docs/api/lexeme.mdx similarity index 96% rename from website/docs/api/lexeme.md rename to website/docs/api/lexeme.mdx index eb76afa90..539f502f0 100644 --- a/website/docs/api/lexeme.md +++ b/website/docs/api/lexeme.mdx @@ -9,7 +9,7 @@ A `Lexeme` has no string context – it's a word type, as opposed to a word toke It therefore has no part-of-speech tag, dependency parse, or lemma (if lemmatization depends on the part-of-speech tag). -## Lexeme.\_\_init\_\_ {#init tag="method"} +## Lexeme.\_\_init\_\_ {id="init",tag="method"} Create a `Lexeme` object. @@ -18,7 +18,7 @@ Create a `Lexeme` object. | `vocab` | The parent vocabulary. ~~Vocab~~ | | `orth` | The orth id of the lexeme. ~~int~~ | -## Lexeme.set_flag {#set_flag tag="method"} +## Lexeme.set_flag {id="set_flag",tag="method"} Change the value of a boolean flag. @@ -34,7 +34,7 @@ Change the value of a boolean flag. | `flag_id` | The attribute ID of the flag to set. ~~int~~ | | `value` | The new value of the flag. ~~bool~~ | -## Lexeme.check_flag {#check_flag tag="method"} +## Lexeme.check_flag {id="check_flag",tag="method"} Check the value of a boolean flag. @@ -51,7 +51,7 @@ Check the value of a boolean flag. | `flag_id` | The attribute ID of the flag to query. ~~int~~ | | **RETURNS** | The value of the flag. ~~bool~~ | -## Lexeme.similarity {#similarity tag="method" model="vectors"} +## Lexeme.similarity {id="similarity",tag="method",model="vectors"} Compute a semantic similarity estimate. Defaults to cosine over vectors. @@ -70,7 +70,7 @@ Compute a semantic similarity estimate. Defaults to cosine over vectors. | other | The object to compare with. By default, accepts `Doc`, `Span`, `Token` and `Lexeme` objects. ~~Union[Doc, Span, Token, Lexeme]~~ | | **RETURNS** | A scalar similarity score. Higher is more similar. ~~float~~ | -## Lexeme.has_vector {#has_vector tag="property" model="vectors"} +## Lexeme.has_vector {id="has_vector",tag="property",model="vectors"} A boolean value indicating whether a word vector is associated with the lexeme. @@ -85,7 +85,7 @@ A boolean value indicating whether a word vector is associated with the lexeme. | ----------- | ------------------------------------------------------- | | **RETURNS** | Whether the lexeme has a vector data attached. ~~bool~~ | -## Lexeme.vector {#vector tag="property" model="vectors"} +## Lexeme.vector {id="vector",tag="property",model="vectors"} A real-valued meaning representation. @@ -101,7 +101,7 @@ A real-valued meaning representation. | ----------- | ------------------------------------------------------------------------------------------------ | | **RETURNS** | A 1-dimensional array representing the lexeme's vector. ~~numpy.ndarray[ndim=1, dtype=float32]~~ | -## Lexeme.vector_norm {#vector_norm tag="property" model="vectors"} +## Lexeme.vector_norm {id="vector_norm",tag="property",model="vectors"} The L2 norm of the lexeme's vector representation. @@ -119,7 +119,7 @@ The L2 norm of the lexeme's vector representation. | ----------- | --------------------------------------------------- | | **RETURNS** | The L2 norm of the vector representation. ~~float~~ | -## Attributes {#attributes} +## Attributes {id="attributes"} | Name | Description | | ---------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | @@ -138,7 +138,7 @@ The L2 norm of the lexeme's vector representation. | `prefix` | Length-N substring from the start of the word. Defaults to `N=1`. ~~int~~ | | `prefix_` | Length-N substring from the start of the word. Defaults to `N=1`. ~~str~~ | | `suffix` | Length-N substring from the end of the word. Defaults to `N=3`. ~~int~~ | -| `suffix_` | Length-N substring from the start of the word. Defaults to `N=3`. ~~str~~ | +| `suffix_` | Length-N substring from the end of the word. Defaults to `N=3`. ~~str~~ | | `is_alpha` | Does the lexeme consist of alphabetic characters? Equivalent to `lexeme.text.isalpha()`. ~~bool~~ | | `is_ascii` | Does the lexeme consist of ASCII characters? Equivalent to `[any(ord(c) >= 128 for c in lexeme.text)]`. ~~bool~~ | | `is_digit` | Does the lexeme consist of digits? Equivalent to `lexeme.text.isdigit()`. ~~bool~~ | diff --git a/website/docs/api/lookups.md b/website/docs/api/lookups.mdx similarity index 89% rename from website/docs/api/lookups.md rename to website/docs/api/lookups.mdx index 9565e478f..71a857c60 100644 --- a/website/docs/api/lookups.md +++ b/website/docs/api/lookups.mdx @@ -3,7 +3,7 @@ title: Lookups teaser: A container for large lookup tables and dictionaries tag: class source: spacy/lookups.py -new: 2.2 +version: 2.2 --- This class allows convenient access to large lookup tables and dictionaries, @@ -13,7 +13,7 @@ can be accessed before the pipeline components are applied (e.g. in the tokenizer and lemmatizer), as well as within the pipeline components via `doc.vocab.lookups`. -## Lookups.\_\_init\_\_ {#init tag="method"} +## Lookups.\_\_init\_\_ {id="init",tag="method"} Create a `Lookups` object. @@ -24,7 +24,7 @@ Create a `Lookups` object. > lookups = Lookups() > ``` -## Lookups.\_\_len\_\_ {#len tag="method"} +## Lookups.\_\_len\_\_ {id="len",tag="method"} Get the current number of tables in the lookups. @@ -39,7 +39,7 @@ Get the current number of tables in the lookups. | ----------- | -------------------------------------------- | | **RETURNS** | The number of tables in the lookups. ~~int~~ | -## Lookups.\_\contains\_\_ {#contains tag="method"} +## Lookups.\_\_contains\_\_ {id="contains",tag="method"} Check if the lookups contain a table of a given name. Delegates to [`Lookups.has_table`](/api/lookups#has_table). @@ -57,7 +57,7 @@ Check if the lookups contain a table of a given name. Delegates to | `name` | Name of the table. ~~str~~ | | **RETURNS** | Whether a table of that name is in the lookups. ~~bool~~ | -## Lookups.tables {#tables tag="property"} +## Lookups.tables {id="tables",tag="property"} Get the names of all tables in the lookups. @@ -73,7 +73,7 @@ Get the names of all tables in the lookups. | ----------- | ------------------------------------------------- | | **RETURNS** | Names of the tables in the lookups. ~~List[str]~~ | -## Lookups.add_table {#add_table tag="method"} +## Lookups.add_table {id="add_table",tag="method"} Add a new table with optional data to the lookups. Raises an error if the table exists. @@ -91,7 +91,7 @@ exists. | `data` | Optional data to add to the table. ~~dict~~ | | **RETURNS** | The newly added table. ~~Table~~ | -## Lookups.get_table {#get_table tag="method"} +## Lookups.get_table {id="get_table",tag="method"} Get a table from the lookups. Raises an error if the table doesn't exist. @@ -109,7 +109,7 @@ Get a table from the lookups. Raises an error if the table doesn't exist. | `name` | Name of the table. ~~str~~ | | **RETURNS** | The table. ~~Table~~ | -## Lookups.remove_table {#remove_table tag="method"} +## Lookups.remove_table {id="remove_table",tag="method"} Remove a table from the lookups. Raises an error if the table doesn't exist. @@ -127,7 +127,7 @@ Remove a table from the lookups. Raises an error if the table doesn't exist. | `name` | Name of the table to remove. ~~str~~ | | **RETURNS** | The removed table. ~~Table~~ | -## Lookups.has_table {#has_table tag="method"} +## Lookups.has_table {id="has_table",tag="method"} Check if the lookups contain a table of a given name. Equivalent to [`Lookups.__contains__`](/api/lookups#contains). @@ -145,7 +145,7 @@ Check if the lookups contain a table of a given name. Equivalent to | `name` | Name of the table. ~~str~~ | | **RETURNS** | Whether a table of that name is in the lookups. ~~bool~~ | -## Lookups.to_bytes {#to_bytes tag="method"} +## Lookups.to_bytes {id="to_bytes",tag="method"} Serialize the lookups to a bytestring. @@ -159,7 +159,7 @@ Serialize the lookups to a bytestring. | ----------- | --------------------------------- | | **RETURNS** | The serialized lookups. ~~bytes~~ | -## Lookups.from_bytes {#from_bytes tag="method"} +## Lookups.from_bytes {id="from_bytes",tag="method"} Load the lookups from a bytestring. @@ -176,7 +176,7 @@ Load the lookups from a bytestring. | `bytes_data` | The data to load from. ~~bytes~~ | | **RETURNS** | The loaded lookups. ~~Lookups~~ | -## Lookups.to_disk {#to_disk tag="method"} +## Lookups.to_disk {id="to_disk",tag="method"} Save the lookups to a directory as `lookups.bin`. Expects a path to a directory, which will be created if it doesn't exist. @@ -191,7 +191,7 @@ which will be created if it doesn't exist. | ------ | ------------------------------------------------------------------------------------------------------------------------------------------ | | `path` | A path to a directory, which will be created if it doesn't exist. Paths may be either strings or `Path`-like objects. ~~Union[str, Path]~~ | -## Lookups.from_disk {#from_disk tag="method"} +## Lookups.from_disk {id="from_disk",tag="method"} Load lookups from a directory containing a `lookups.bin`. Will skip loading if the file doesn't exist. @@ -209,7 +209,7 @@ the file doesn't exist. | `path` | A path to a directory. Paths may be either strings or `Path`-like objects. ~~Union[str, Path]~~ | | **RETURNS** | The loaded lookups. ~~Lookups~~ | -## Table {#table tag="class, ordererddict"} +## Table {id="table",tag="class, ordererddict"} A table in the lookups. Subclass of `OrderedDict` that implements a slightly more consistent and unified API and includes a Bloom filter to speed up missed @@ -218,7 +218,7 @@ lookups. Supports **all other methods and attributes** of `OrderedDict` / accept both integers and strings (which will be hashed before being added to the table). -### Table.\_\_init\_\_ {#table.init tag="method"} +### Table.\_\_init\_\_ {id="table.init",tag="method"} Initialize a new table. @@ -236,7 +236,7 @@ Initialize a new table. | ------ | ------------------------------------------ | | `name` | Optional table name for reference. ~~str~~ | -### Table.from_dict {#table.from_dict tag="classmethod"} +### Table.from_dict {id="table.from_dict",tag="classmethod"} Initialize a new table from a dict. @@ -254,7 +254,7 @@ Initialize a new table from a dict. | `name` | Optional table name for reference. ~~str~~ | | **RETURNS** | The newly constructed object. ~~Table~~ | -### Table.set {#table.set tag="method"} +### Table.set {id="table.set",tag="method"} Set a new key / value pair. String keys will be hashed. Same as `table[key] = value`. @@ -273,7 +273,7 @@ Set a new key / value pair. String keys will be hashed. Same as | `key` | The key. ~~Union[str, int]~~ | | `value` | The value. | -### Table.to_bytes {#table.to_bytes tag="method"} +### Table.to_bytes {id="table.to_bytes",tag="method"} Serialize the table to a bytestring. @@ -287,7 +287,7 @@ Serialize the table to a bytestring. | ----------- | ------------------------------- | | **RETURNS** | The serialized table. ~~bytes~~ | -### Table.from_bytes {#table.from_bytes tag="method"} +### Table.from_bytes {id="table.from_bytes",tag="method"} Load a table from a bytestring. @@ -304,7 +304,7 @@ Load a table from a bytestring. | `bytes_data` | The data to load. ~~bytes~~ | | **RETURNS** | The loaded table. ~~Table~~ | -### Attributes {#table-attributes} +### Attributes {id="table-attributes"} | Name | Description | | -------------- | ------------------------------------------------------------- | diff --git a/website/docs/api/matcher.md b/website/docs/api/matcher.mdx similarity index 79% rename from website/docs/api/matcher.md rename to website/docs/api/matcher.mdx index cd7bfa070..c66579da8 100644 --- a/website/docs/api/matcher.md +++ b/website/docs/api/matcher.mdx @@ -13,7 +13,7 @@ tokens in context. For in-depth examples and workflows for combining rules and statistical models, see the [usage guide](/usage/rule-based-matching) on rule-based matching. -## Pattern format {#patterns} +## Pattern format {id="patterns"} > ```json > ### Example @@ -86,16 +86,22 @@ it compares to another value. > ] > ``` -| Attribute | Description | -| -------------------------- | -------------------------------------------------------------------------------------------------------- | -| `IN` | Attribute value is member of a list. ~~Any~~ | -| `NOT_IN` | Attribute value is _not_ member of a list. ~~Any~~ | -| `IS_SUBSET` | Attribute value (for `MORPH` or custom list attributes) is a subset of a list. ~~Any~~ | -| `IS_SUPERSET` | Attribute value (for `MORPH` or custom list attributes) is a superset of a list. ~~Any~~ | -| `INTERSECTS` | Attribute value (for `MORPH` or custom list attribute) has a non-empty intersection with a list. ~~Any~~ | -| `==`, `>=`, `<=`, `>`, `<` | Attribute value is equal, greater or equal, smaller or equal, greater or smaller. ~~Union[int, float]~~ | +| Attribute | Description | +| -------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `REGEX` | Attribute value matches the regular expression at any position in the string. ~~Any~~ | +| `FUZZY` | Attribute value matches if the `fuzzy_compare` method matches for `(value, pattern, -1)`. The default method allows a Levenshtein edit distance of at least 2 and up to 30% of the pattern string length. ~~Any~~ | +| `FUZZY1`, `FUZZY2`, ... `FUZZY9` | Attribute value matches if the `fuzzy_compare` method matches for `(value, pattern, N)`. The default method allows a Levenshtein edit distance of at most N (1-9). ~~Any~~ | +| `IN` | Attribute value is member of a list. ~~Any~~ | +| `NOT_IN` | Attribute value is _not_ member of a list. ~~Any~~ | +| `IS_SUBSET` | Attribute value (for `MORPH` or custom list attributes) is a subset of a list. ~~Any~~ | +| `IS_SUPERSET` | Attribute value (for `MORPH` or custom list attributes) is a superset of a list. ~~Any~~ | +| `INTERSECTS` | Attribute value (for `MORPH` or custom list attribute) has a non-empty intersection with a list. ~~Any~~ | +| `==`, `>=`, `<=`, `>`, `<` | Attribute value is equal, greater or equal, smaller or equal, greater or smaller. ~~Union[int, float]~~ | -## Matcher.\_\_init\_\_ {#init tag="method"} +As of spaCy v3.5, `REGEX` and `FUZZY` can be used in combination with `IN` and +`NOT_IN`. + +## Matcher.\_\_init\_\_ {id="init",tag="method"} Create the rule-based `Matcher`. If `validate=True` is set, all patterns added to the matcher will be validated against a JSON schema and a `MatchPatternError` @@ -109,12 +115,13 @@ string where an integer is expected) or unexpected property names. > matcher = Matcher(nlp.vocab) > ``` -| Name | Description | -| ---------- | ----------------------------------------------------------------------------------------------------- | -| `vocab` | The vocabulary object, which must be shared with the documents the matcher will operate on. ~~Vocab~~ | -| `validate` | Validate all patterns added to this matcher. ~~bool~~ | +| Name | Description | +| --------------- | ----------------------------------------------------------------------------------------------------- | +| `vocab` | The vocabulary object, which must be shared with the documents the matcher will operate on. ~~Vocab~~ | +| `validate` | Validate all patterns added to this matcher. ~~bool~~ | +| `fuzzy_compare` | The comparison method used for the `FUZZY` operators. ~~Callable[[str, str, int], bool]~~ | -## Matcher.\_\_call\_\_ {#call tag="method"} +## Matcher.\_\_call\_\_ {id="call",tag="method"} Find all token sequences matching the supplied patterns on the `Doc` or `Span`. @@ -143,7 +150,7 @@ the match. | `with_alignments` 3.0.6 | Return match alignment information as part of the match tuple as `List[int]` with the same length as the matched span. Each entry denotes the corresponding index of the token in the pattern. If `as_spans` is set to `True`, this setting is ignored. Defaults to `False`. ~~bool~~ | | **RETURNS** | A list of `(match_id, start, end)` tuples, describing the matches. A match tuple describes a span `doc[start:end`]. The `match_id` is the ID of the added match pattern. If `as_spans` is set to `True`, a list of `Span` objects is returned instead. ~~Union[List[Tuple[int, int, int]], List[Span]]~~ | -## Matcher.\_\_len\_\_ {#len tag="method" new="2"} +## Matcher.\_\_len\_\_ {id="len",tag="method",version="2"} Get the number of rules added to the matcher. Note that this only returns the number of rules (identical with the number of IDs), not the number of individual @@ -162,7 +169,7 @@ patterns. | ----------- | ---------------------------- | | **RETURNS** | The number of rules. ~~int~~ | -## Matcher.\_\_contains\_\_ {#contains tag="method" new="2"} +## Matcher.\_\_contains\_\_ {id="contains",tag="method",version="2"} Check whether the matcher contains rules for a match ID. @@ -180,7 +187,7 @@ Check whether the matcher contains rules for a match ID. | `key` | The match ID. ~~str~~ | | **RETURNS** | Whether the matcher contains rules for this match ID. ~~bool~~ | -## Matcher.add {#add tag="method" new="2"} +## Matcher.add {id="add",tag="method",version="2"} Add a rule to the matcher, consisting of an ID key, one or more patterns, and an optional callback function to act on the matches. The callback function will @@ -226,7 +233,7 @@ patterns = [[{"TEXT": "Google"}, {"TEXT": "Now"}], [{"TEXT": "GoogleNow"}]] | `on_match` | Callback function to act on matches. Takes the arguments `matcher`, `doc`, `i` and `matches`. ~~Optional[Callable[[Matcher, Doc, int, List[tuple], Any]]~~ | | `greedy` 3 | Optional filter for greedy matches. Can either be `"FIRST"` or `"LONGEST"`. ~~Optional[str]~~ | -## Matcher.remove {#remove tag="method" new="2"} +## Matcher.remove {id="remove",tag="method",version="2"} Remove a rule from the matcher. A `KeyError` is raised if the match ID does not exist. @@ -244,7 +251,7 @@ exist. | ----- | --------------------------------- | | `key` | The ID of the match rule. ~~str~~ | -## Matcher.get {#get tag="method" new="2"} +## Matcher.get {id="get",tag="method",version="2"} Retrieve the pattern stored for a key. Returns the rule as an `(on_match, patterns)` tuple containing the callback and available patterns. diff --git a/website/docs/api/morphologizer.md b/website/docs/api/morphologizer.mdx similarity index 88% rename from website/docs/api/morphologizer.md rename to website/docs/api/morphologizer.mdx index f874e8bea..8f189d129 100644 --- a/website/docs/api/morphologizer.md +++ b/website/docs/api/morphologizer.mdx @@ -2,7 +2,7 @@ title: Morphologizer tag: class source: spacy/pipeline/morphologizer.pyx -new: 3 +version: 3 teaser: 'Pipeline component for predicting morphological features' api_base_class: /api/tagger api_string_name: morphologizer @@ -15,7 +15,7 @@ coarse-grained POS tags following the Universal Dependencies [FEATS](https://universaldependencies.org/format.html#morphological-annotation) annotation guidelines. -## Assigned Attributes {#assigned-attributes} +## Assigned Attributes {id="assigned-attributes"} Predictions are saved to `Token.morph` and `Token.pos`. @@ -25,7 +25,7 @@ Predictions are saved to `Token.morph` and `Token.pos`. | `Token.pos_` | The UPOS part of speech. ~~str~~ | | `Token.morph` | Morphological features. ~~MorphAnalysis~~ | -## Config and implementation {#config} +## Config and implementation {id="config"} The default config is defined by the pipeline component factory and describes how the component should be configured. You can override its settings via the @@ -42,18 +42,19 @@ architectures and their arguments and hyperparameters. > nlp.add_pipe("morphologizer", config=config) > ``` -| Setting | Description | -| ---------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `model` | The model to use. Defaults to [Tagger](/api/architectures#Tagger). ~~Model[List[Doc], List[Floats2d]]~~ | -| `overwrite` 3.2 | Whether the values of existing features are overwritten. Defaults to `True`. ~~bool~~ | -| `extend` 3.2 | Whether existing feature types (whose values may or may not be overwritten depending on `overwrite`) are preserved. Defaults to `False`. ~~bool~~ | -| `scorer` 3.2 | The scoring method. Defaults to [`Scorer.score_token_attr`](/api/scorer#score_token_attr) for the attributes `"pos"` and `"morph"` and [`Scorer.score_token_attr_per_feat`](/api/scorer#score_token_attr_per_feat) for the attribute `"morph"`. ~~Optional[Callable]~~ | +| Setting | Description | +| ---------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `model` | The model to use. Defaults to [Tagger](/api/architectures#Tagger). ~~Model[List[Doc], List[Floats2d]]~~ | +| `overwrite` 3.2 | Whether the values of existing features are overwritten. Defaults to `True`. ~~bool~~ | +| `extend` 3.2 | Whether existing feature types (whose values may or may not be overwritten depending on `overwrite`) are preserved. Defaults to `False`. ~~bool~~ | +| `scorer` 3.2 | The scoring method. Defaults to [`Scorer.score_token_attr`](/api/scorer#score_token_attr) for the attributes `"pos"` and `"morph"` and [`Scorer.score_token_attr_per_feat`](/api/scorer#score_token_attr_per_feat) for the attribute `"morph"`. ~~Optional[Callable]~~ | +| `label_smoothing` 3.6 | [Label smoothing](https://arxiv.org/abs/1906.02629) factor. Defaults to `0.0`. ~~float~~ | ```python %%GITHUB_SPACY/spacy/pipeline/morphologizer.pyx ``` -## Morphologizer.\_\_init\_\_ {#init tag="method"} +## Morphologizer.\_\_init\_\_ {id="init",tag="method"} Create a new pipeline instance. In your application, you would normally use a shortcut for this and instantiate the component using its string name and @@ -97,7 +98,7 @@ annotation `C=E|X=Y`): | `extend` 3.2 | Whether existing feature types (whose values may or may not be overwritten depending on `overwrite`) are preserved. Defaults to `False`. ~~bool~~ | | `scorer` 3.2 | The scoring method. Defaults to [`Scorer.score_token_attr`](/api/scorer#score_token_attr) for the attributes `"pos"` and `"morph"` and [`Scorer.score_token_attr_per_feat`](/api/scorer#score_token_attr_per_feat) for the attribute `"morph"`. ~~Optional[Callable]~~ | -## Morphologizer.\_\_call\_\_ {#call tag="method"} +## Morphologizer.\_\_call\_\_ {id="call",tag="method"} Apply the pipe to one document. The document is modified in place, and returned. This usually happens under the hood when the `nlp` object is called on a text @@ -120,7 +121,7 @@ delegate to the [`predict`](/api/morphologizer#predict) and | `doc` | The document to process. ~~Doc~~ | | **RETURNS** | The processed document. ~~Doc~~ | -## Morphologizer.pipe {#pipe tag="method"} +## Morphologizer.pipe {id="pipe",tag="method"} Apply the pipe to a stream of documents. This usually happens under the hood when the `nlp` object is called on a text and all pipeline components are @@ -144,7 +145,7 @@ applied to the `Doc` in order. Both [`__call__`](/api/morphologizer#call) and | `batch_size` | The number of documents to buffer. Defaults to `128`. ~~int~~ | | **YIELDS** | The processed documents in order. ~~Doc~~ | -## Morphologizer.initialize {#initialize tag="method"} +## Morphologizer.initialize {id="initialize",tag="method"} Initialize the component for training. `get_examples` should be a function that returns an iterable of [`Example`](/api/example) objects. **At least one example @@ -181,7 +182,7 @@ config. | `nlp` | The current `nlp` object. Defaults to `None`. ~~Optional[Language]~~ | | `labels` | The label information to add to the component, as provided by the [`label_data`](#label_data) property after initialization. To generate a reusable JSON file from your data, you should run the [`init labels`](/api/cli#init-labels) command. If no labels are provided, the `get_examples` callback is used to extract the labels from the data, which may be a lot slower. ~~Optional[dict]~~ | -## Morphologizer.predict {#predict tag="method"} +## Morphologizer.predict {id="predict",tag="method"} Apply the component's model to a batch of [`Doc`](/api/doc) objects, without modifying them. @@ -198,7 +199,7 @@ modifying them. | `docs` | The documents to predict. ~~Iterable[Doc]~~ | | **RETURNS** | The model's prediction for each document. | -## Morphologizer.set_annotations {#set_annotations tag="method"} +## Morphologizer.set_annotations {id="set_annotations",tag="method"} Modify a batch of [`Doc`](/api/doc) objects, using pre-computed scores. @@ -215,7 +216,7 @@ Modify a batch of [`Doc`](/api/doc) objects, using pre-computed scores. | `docs` | The documents to modify. ~~Iterable[Doc]~~ | | `scores` | The scores to set, produced by `Morphologizer.predict`. | -## Morphologizer.update {#update tag="method"} +## Morphologizer.update {id="update",tag="method"} Learn from a batch of [`Example`](/api/example) objects containing the predictions and gold-standard annotations, and update the component's model. @@ -239,7 +240,7 @@ Delegates to [`predict`](/api/morphologizer#predict) and | `losses` | Optional record of the loss during training. Updated using the component name as the key. ~~Optional[Dict[str, float]]~~ | | **RETURNS** | The updated `losses` dictionary. ~~Dict[str, float]~~ | -## Morphologizer.get_loss {#get_loss tag="method"} +## Morphologizer.get_loss {id="get_loss",tag="method"} Find the loss and gradient of loss for the batch of documents and their predicted scores. @@ -258,7 +259,7 @@ predicted scores. | `scores` | Scores representing the model's predictions. | | **RETURNS** | The loss and the gradient, i.e. `(loss, gradient)`. ~~Tuple[float, float]~~ | -## Morphologizer.create_optimizer {#create_optimizer tag="method"} +## Morphologizer.create_optimizer {id="create_optimizer",tag="method"} Create an optimizer for the pipeline component. @@ -273,7 +274,7 @@ Create an optimizer for the pipeline component. | ----------- | ---------------------------- | | **RETURNS** | The optimizer. ~~Optimizer~~ | -## Morphologizer.use_params {#use_params tag="method, contextmanager"} +## Morphologizer.use_params {id="use_params",tag="method, contextmanager"} Modify the pipe's model, to use the given parameter values. At the end of the context, the original parameters are restored. @@ -290,7 +291,7 @@ context, the original parameters are restored. | -------- | -------------------------------------------------- | | `params` | The parameter values to use in the model. ~~dict~~ | -## Morphologizer.add_label {#add_label tag="method"} +## Morphologizer.add_label {id="add_label",tag="method"} Add a new label to the pipe. If the `Morphologizer` should set annotations for both `pos` and `morph`, the label should include the UPOS as the feature `POS`. @@ -313,7 +314,7 @@ will be automatically added to the model, and the output dimension will be | `label` | The label to add. ~~str~~ | | **RETURNS** | `0` if the label is already present, otherwise `1`. ~~int~~ | -## Morphologizer.to_disk {#to_disk tag="method"} +## Morphologizer.to_disk {id="to_disk",tag="method"} Serialize the pipe to disk. @@ -330,7 +331,7 @@ Serialize the pipe to disk. | _keyword-only_ | | | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | -## Morphologizer.from_disk {#from_disk tag="method"} +## Morphologizer.from_disk {id="from_disk",tag="method"} Load the pipe from disk. Modifies the object in place and returns it. @@ -348,7 +349,7 @@ Load the pipe from disk. Modifies the object in place and returns it. | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | | **RETURNS** | The modified `Morphologizer` object. ~~Morphologizer~~ | -## Morphologizer.to_bytes {#to_bytes tag="method"} +## Morphologizer.to_bytes {id="to_bytes",tag="method"} > #### Example > @@ -365,7 +366,7 @@ Serialize the pipe to a bytestring. | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | | **RETURNS** | The serialized form of the `Morphologizer` object. ~~bytes~~ | -## Morphologizer.from_bytes {#from_bytes tag="method"} +## Morphologizer.from_bytes {id="from_bytes",tag="method"} Load the pipe from a bytestring. Modifies the object in place and returns it. @@ -384,7 +385,7 @@ Load the pipe from a bytestring. Modifies the object in place and returns it. | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | | **RETURNS** | The `Morphologizer` object. ~~Morphologizer~~ | -## Morphologizer.labels {#labels tag="property"} +## Morphologizer.labels {id="labels",tag="property"} The labels currently added to the component in the Universal Dependencies [FEATS](https://universaldependencies.org/format.html#morphological-annotation) @@ -403,7 +404,7 @@ coarse-grained POS as the feature `POS`. | ----------- | ------------------------------------------------------ | | **RETURNS** | The labels added to the component. ~~Tuple[str, ...]~~ | -## Morphologizer.label_data {#label_data tag="property" new="3"} +## Morphologizer.label_data {id="label_data",tag="property",version="3"} The labels currently added to the component and their internal meta information. This is the data generated by [`init labels`](/api/cli#init-labels) and used by @@ -421,7 +422,7 @@ model with a pre-defined label set. | ----------- | ----------------------------------------------- | | **RETURNS** | The label data added to the component. ~~dict~~ | -## Serialization fields {#serialization-fields} +## Serialization fields {id="serialization-fields"} During serialization, spaCy will export several data fields used to restore different aspects of the object. If needed, you can exclude them from diff --git a/website/docs/api/morphology.md b/website/docs/api/morphology.mdx similarity index 89% rename from website/docs/api/morphology.md rename to website/docs/api/morphology.mdx index 20fcd1a40..68d80b814 100644 --- a/website/docs/api/morphology.md +++ b/website/docs/api/morphology.mdx @@ -10,7 +10,7 @@ morphological analysis, so queries of morphological attributes are delegated to this class. See [`MorphAnalysis`](/api/morphology#morphanalysis) for the container storing a single morphological analysis. -## Morphology.\_\_init\_\_ {#init tag="method"} +## Morphology.\_\_init\_\_ {id="init",tag="method"} Create a `Morphology` object. @@ -26,7 +26,7 @@ Create a `Morphology` object. | --------- | --------------------------------- | | `strings` | The string store. ~~StringStore~~ | -## Morphology.add {#add tag="method"} +## Morphology.add {id="add",tag="method"} Insert a morphological analysis in the morphology table, if not already present. The morphological analysis may be provided in the Universal Dependencies @@ -46,7 +46,7 @@ new analysis. | ---------- | ------------------------------------------------ | | `features` | The morphological features. ~~Union[Dict, str]~~ | -## Morphology.get {#get tag="method"} +## Morphology.get {id="get",tag="method"} > #### Example > @@ -64,7 +64,7 @@ string for the hash of the morphological analysis. | ------- | ----------------------------------------------- | | `morph` | The hash of the morphological analysis. ~~int~~ | -## Morphology.feats_to_dict {#feats_to_dict tag="staticmethod"} +## Morphology.feats_to_dict {id="feats_to_dict",tag="staticmethod"} Convert a string [FEATS](https://universaldependencies.org/format.html#morphological-annotation) @@ -84,7 +84,7 @@ tag map. | `feats` | The morphological features in Universal Dependencies [FEATS](https://universaldependencies.org/format.html#morphological-annotation) format. ~~str~~ | | **RETURNS** | The morphological features as a dictionary. ~~Dict[str, str]~~ | -## Morphology.dict_to_feats {#dict_to_feats tag="staticmethod"} +## Morphology.dict_to_feats {id="dict_to_feats",tag="staticmethod"} Convert a dictionary of features and values to a string [FEATS](https://universaldependencies.org/format.html#morphological-annotation) @@ -103,19 +103,19 @@ representation. | `feats_dict` | The morphological features as a dictionary. ~~Dict[str, str]~~ | | **RETURNS** | The morphological features in Universal Dependencies [FEATS](https://universaldependencies.org/format.html#morphological-annotation) format. ~~str~~ | -## Attributes {#attributes} +## Attributes {id="attributes"} -| Name | Description | -| ------------- | ------------------------------------------------------------------------------------------------------------------------------ | -| `FEATURE_SEP` | The [FEATS](https://universaldependencies.org/format.html#morphological-annotation) feature separator. Default is `|`. ~~str~~ | -| `FIELD_SEP` | The [FEATS](https://universaldependencies.org/format.html#morphological-annotation) field separator. Default is `=`. ~~str~~ | -| `VALUE_SEP` | The [FEATS](https://universaldependencies.org/format.html#morphological-annotation) value separator. Default is `,`. ~~str~~ | +| Name | Description | +| ------------- | ------------------------------------------------------------------------------------------------------------------------------- | +| `FEATURE_SEP` | The [FEATS](https://universaldependencies.org/format.html#morphological-annotation) feature separator. Default is `\|`. ~~str~~ | +| `FIELD_SEP` | The [FEATS](https://universaldependencies.org/format.html#morphological-annotation) field separator. Default is `=`. ~~str~~ | +| `VALUE_SEP` | The [FEATS](https://universaldependencies.org/format.html#morphological-annotation) value separator. Default is `,`. ~~str~~ | -## MorphAnalysis {#morphanalysis tag="class" source="spacy/tokens/morphanalysis.pyx"} +## MorphAnalysis {id="morphanalysis",tag="class",source="spacy/tokens/morphanalysis.pyx"} Stores a single morphological analysis. -### MorphAnalysis.\_\_init\_\_ {#morphanalysis-init tag="method"} +### MorphAnalysis.\_\_init\_\_ {id="morphanalysis-init",tag="method"} Initialize a MorphAnalysis object from a Universal Dependencies [FEATS](https://universaldependencies.org/format.html#morphological-annotation) @@ -135,7 +135,7 @@ string or a dictionary of morphological features. | `vocab` | The vocab. ~~Vocab~~ | | `features` | The morphological features. ~~Union[Dict[str, str], str]~~ | -### MorphAnalysis.\_\_contains\_\_ {#morphanalysis-contains tag="method"} +### MorphAnalysis.\_\_contains\_\_ {id="morphanalysis-contains",tag="method"} Whether a feature/value pair is in the analysis. @@ -151,7 +151,7 @@ Whether a feature/value pair is in the analysis. | ----------- | --------------------------------------------- | | **RETURNS** | A feature/value pair in the analysis. ~~str~~ | -### MorphAnalysis.\_\_iter\_\_ {#morphanalysis-iter tag="method"} +### MorphAnalysis.\_\_iter\_\_ {id="morphanalysis-iter",tag="method"} Iterate over the feature/value pairs in the analysis. @@ -167,7 +167,7 @@ Iterate over the feature/value pairs in the analysis. | ---------- | --------------------------------------------- | | **YIELDS** | A feature/value pair in the analysis. ~~str~~ | -### MorphAnalysis.\_\_len\_\_ {#morphanalysis-len tag="method"} +### MorphAnalysis.\_\_len\_\_ {id="morphanalysis-len",tag="method"} Returns the number of features in the analysis. @@ -183,7 +183,7 @@ Returns the number of features in the analysis. | ----------- | ----------------------------------------------- | | **RETURNS** | The number of features in the analysis. ~~int~~ | -### MorphAnalysis.\_\_str\_\_ {#morphanalysis-str tag="method"} +### MorphAnalysis.\_\_str\_\_ {id="morphanalysis-str",tag="method"} Returns the morphological analysis in the Universal Dependencies [FEATS](https://universaldependencies.org/format.html#morphological-annotation) @@ -201,7 +201,7 @@ string format. | ----------- | ------------------------------------------------------------------------------------------------------------------------------------------ | | **RETURNS** | The analysis in the Universal Dependencies [FEATS](https://universaldependencies.org/format.html#morphological-annotation) format. ~~str~~ | -### MorphAnalysis.get {#morphanalysis-get tag="method"} +### MorphAnalysis.get {id="morphanalysis-get",tag="method"} Retrieve values for a feature by field. @@ -218,7 +218,7 @@ Retrieve values for a feature by field. | `field` | The field to retrieve. ~~str~~ | | **RETURNS** | A list of the individual features. ~~List[str]~~ | -### MorphAnalysis.to_dict {#morphanalysis-to_dict tag="method"} +### MorphAnalysis.to_dict {id="morphanalysis-to_dict",tag="method"} Produce a dict representation of the analysis, in the same format as the tag map. @@ -235,7 +235,7 @@ map. | ----------- | ----------------------------------------------------------- | | **RETURNS** | The dict representation of the analysis. ~~Dict[str, str]~~ | -### MorphAnalysis.from_id {#morphanalysis-from_id tag="classmethod"} +### MorphAnalysis.from_id {id="morphanalysis-from_id",tag="classmethod"} Create a morphological analysis from a given hash ID. diff --git a/website/docs/api/phrasematcher.md b/website/docs/api/phrasematcher.mdx similarity index 96% rename from website/docs/api/phrasematcher.md rename to website/docs/api/phrasematcher.mdx index cd419ae5c..14ccefb77 100644 --- a/website/docs/api/phrasematcher.md +++ b/website/docs/api/phrasematcher.mdx @@ -3,7 +3,7 @@ title: PhraseMatcher teaser: Match sequences of tokens, based on documents tag: class source: spacy/matcher/phrasematcher.pyx -new: 2 +version: 2 --- The `PhraseMatcher` lets you efficiently match large terminology lists. While @@ -12,7 +12,7 @@ descriptions, the `PhraseMatcher` accepts match patterns in the form of `Doc` objects. See the [usage guide](/usage/rule-based-matching#phrasematcher) for examples. -## PhraseMatcher.\_\_init\_\_ {#init tag="method"} +## PhraseMatcher.\_\_init\_\_ {id="init",tag="method"} Create the rule-based `PhraseMatcher`. Setting a different `attr` to match on will change the token attributes that will be compared to determine a match. By @@ -42,7 +42,7 @@ be shown. | `attr` | The token attribute to match on. Defaults to `ORTH`, i.e. the verbatim token text. ~~Union[int, str]~~ | | `validate` | Validate patterns added to the matcher. ~~bool~~ | -## PhraseMatcher.\_\_call\_\_ {#call tag="method"} +## PhraseMatcher.\_\_call\_\_ {id="call",tag="method"} Find all token sequences matching the supplied patterns on the `Doc` or `Span`. @@ -76,7 +76,7 @@ match_id_string = nlp.vocab.strings[match_id] -## PhraseMatcher.\_\_len\_\_ {#len tag="method"} +## PhraseMatcher.\_\_len\_\_ {id="len",tag="method"} Get the number of rules added to the matcher. Note that this only returns the number of rules (identical with the number of IDs), not the number of individual @@ -95,7 +95,7 @@ patterns. | ----------- | ---------------------------- | | **RETURNS** | The number of rules. ~~int~~ | -## PhraseMatcher.\_\_contains\_\_ {#contains tag="method"} +## PhraseMatcher.\_\_contains\_\_ {id="contains",tag="method"} Check whether the matcher contains rules for a match ID. @@ -113,7 +113,7 @@ Check whether the matcher contains rules for a match ID. | `key` | The match ID. ~~str~~ | | **RETURNS** | Whether the matcher contains rules for this match ID. ~~bool~~ | -## PhraseMatcher.add {#add tag="method"} +## PhraseMatcher.add {id="add",tag="method"} Add a rule to the matcher, consisting of an ID key, one or more patterns, and a callback function to act on the matches. The callback function will receive the @@ -155,7 +155,7 @@ patterns = [nlp("health care reform"), nlp("healthcare reform")] | _keyword-only_ | | | `on_match` | Callback function to act on matches. Takes the arguments `matcher`, `doc`, `i` and `matches`. ~~Optional[Callable[[Matcher, Doc, int, List[tuple], Any]]~~ | -## PhraseMatcher.remove {#remove tag="method" new="2.2"} +## PhraseMatcher.remove {id="remove",tag="method",version="2.2"} Remove a rule from the matcher by match ID. A `KeyError` is raised if the key does not exist. diff --git a/website/docs/api/pipe.md b/website/docs/api/pipe.mdx similarity index 93% rename from website/docs/api/pipe.md rename to website/docs/api/pipe.mdx index 263942e3e..c2777edf0 100644 --- a/website/docs/api/pipe.md +++ b/website/docs/api/pipe.mdx @@ -12,7 +12,7 @@ spaCy pipeline. See the docs on [writing trainable components](/usage/processing-pipelines#trainable-components) for how to use the `TrainablePipe` base class to implement custom components. - +{/* TODO: Pipe vs TrainablePipe, check methods below (all renamed to TrainablePipe for now) */} > #### Why is it implemented in Cython? > @@ -27,7 +27,7 @@ for how to use the `TrainablePipe` base class to implement custom components. %%GITHUB_SPACY/spacy/pipeline/trainable_pipe.pyx ``` -## TrainablePipe.\_\_init\_\_ {#init tag="method"} +## TrainablePipe.\_\_init\_\_ {id="init",tag="method"} > #### Example > @@ -54,7 +54,7 @@ shortcut for this and instantiate the component using its string name and | `name` | String name of the component instance. Used to add entries to the `losses` during training. ~~str~~ | | `**cfg` | Additional config parameters and settings. Will be available as the dictionary `cfg` and is serialized with the component. | -## TrainablePipe.\_\_call\_\_ {#call tag="method"} +## TrainablePipe.\_\_call\_\_ {id="call",tag="method"} Apply the pipe to one document. The document is modified in place, and returned. This usually happens under the hood when the `nlp` object is called on a text @@ -77,7 +77,7 @@ and all pipeline components are applied to the `Doc` in order. Both | `doc` | The document to process. ~~Doc~~ | | **RETURNS** | The processed document. ~~Doc~~ | -## TrainablePipe.pipe {#pipe tag="method"} +## TrainablePipe.pipe {id="pipe",tag="method"} Apply the pipe to a stream of documents. This usually happens under the hood when the `nlp` object is called on a text and all pipeline components are @@ -100,7 +100,7 @@ applied to the `Doc` in order. Both [`__call__`](/api/pipe#call) and | `batch_size` | The number of documents to buffer. Defaults to `128`. ~~int~~ | | **YIELDS** | The processed documents in order. ~~Doc~~ | -## TrainablePipe.set_error_handler {#set_error_handler tag="method" new="3"} +## TrainablePipe.set_error_handler {id="set_error_handler",tag="method",version="3"} Define a callback that will be invoked when an error is thrown during processing of one or more documents with either [`__call__`](/api/pipe#call) or @@ -122,7 +122,7 @@ processed, and the original error. | --------------- | -------------------------------------------------------------------------------------------------------------- | | `error_handler` | A function that performs custom error handling. ~~Callable[[str, Callable[[Doc], Doc], List[Doc], Exception]~~ | -## TrainablePipe.get_error_handler {#get_error_handler tag="method" new="3"} +## TrainablePipe.get_error_handler {id="get_error_handler",tag="method",version="3"} Retrieve the callback that performs error handling for this component's [`__call__`](/api/pipe#call) and [`pipe`](/api/pipe#pipe) methods. If no custom @@ -141,7 +141,7 @@ returned that simply reraises the exception. | ----------- | ---------------------------------------------------------------------------------------------------------------- | | **RETURNS** | The function that performs custom error handling. ~~Callable[[str, Callable[[Doc], Doc], List[Doc], Exception]~~ | -## TrainablePipe.initialize {#initialize tag="method" new="3"} +## TrainablePipe.initialize {id="initialize",tag="method",version="3"} Initialize the component for training. `get_examples` should be a function that returns an iterable of [`Example`](/api/example) objects. The data examples are @@ -171,7 +171,7 @@ This method was previously called `begin_training`. | _keyword-only_ | | | `nlp` | The current `nlp` object. Defaults to `None`. ~~Optional[Language]~~ | -## TrainablePipe.predict {#predict tag="method"} +## TrainablePipe.predict {id="predict",tag="method"} Apply the component's model to a batch of [`Doc`](/api/doc) objects, without modifying them. @@ -194,7 +194,7 @@ This method needs to be overwritten with your own custom `predict` method. | `docs` | The documents to predict. ~~Iterable[Doc]~~ | | **RETURNS** | The model's prediction for each document. | -## TrainablePipe.set_annotations {#set_annotations tag="method"} +## TrainablePipe.set_annotations {id="set_annotations",tag="method"} Modify a batch of [`Doc`](/api/doc) objects, using pre-computed scores. @@ -218,7 +218,7 @@ method. | `docs` | The documents to modify. ~~Iterable[Doc]~~ | | `scores` | The scores to set, produced by `Tagger.predict`. | -## TrainablePipe.update {#update tag="method"} +## TrainablePipe.update {id="update",tag="method"} Learn from a batch of [`Example`](/api/example) objects containing the predictions and gold-standard annotations, and update the component's model. @@ -240,7 +240,7 @@ predictions and gold-standard annotations, and update the component's model. | `losses` | Optional record of the loss during training. Updated using the component name as the key. ~~Optional[Dict[str, float]]~~ | | **RETURNS** | The updated `losses` dictionary. ~~Dict[str, float]~~ | -## TrainablePipe.rehearse {#rehearse tag="method,experimental" new="3"} +## TrainablePipe.rehearse {id="rehearse",tag="method,experimental",version="3"} Perform a "rehearsal" update from a batch of data. Rehearsal updates teach the current model to make predictions similar to an initial model, to try to address @@ -262,7 +262,7 @@ the "catastrophic forgetting" problem. This feature is experimental. | `losses` | Optional record of the loss during training. Updated using the component name as the key. ~~Optional[Dict[str, float]]~~ | | **RETURNS** | The updated `losses` dictionary. ~~Dict[str, float]~~ | -## TrainablePipe.get_loss {#get_loss tag="method"} +## TrainablePipe.get_loss {id="get_loss",tag="method"} Find the loss and gradient of loss for the batch of documents and their predicted scores. @@ -287,7 +287,7 @@ This method needs to be overwritten with your own custom `get_loss` method. | `scores` | Scores representing the model's predictions. | | **RETURNS** | The loss and the gradient, i.e. `(loss, gradient)`. ~~Tuple[float, float]~~ | -## TrainablePipe.score {#score tag="method" new="3"} +## TrainablePipe.score {id="score",tag="method",version="3"} Score a batch of examples. @@ -304,7 +304,7 @@ Score a batch of examples. | `\*\*kwargs` | Any additional settings to pass on to the scorer. ~~Any~~ | | **RETURNS** | The scores, e.g. produced by the [`Scorer`](/api/scorer). ~~Dict[str, Union[float, Dict[str, float]]]~~ | -## TrainablePipe.create_optimizer {#create_optimizer tag="method"} +## TrainablePipe.create_optimizer {id="create_optimizer",tag="method"} Create an optimizer for the pipeline component. Defaults to [`Adam`](https://thinc.ai/docs/api-optimizers#adam) with default settings. @@ -320,7 +320,7 @@ Create an optimizer for the pipeline component. Defaults to | ----------- | ---------------------------- | | **RETURNS** | The optimizer. ~~Optimizer~~ | -## TrainablePipe.use_params {#use_params tag="method, contextmanager"} +## TrainablePipe.use_params {id="use_params",tag="method, contextmanager"} Modify the pipe's model, to use the given parameter values. At the end of the context, the original parameters are restored. @@ -337,7 +337,7 @@ context, the original parameters are restored. | -------- | -------------------------------------------------- | | `params` | The parameter values to use in the model. ~~dict~~ | -## TrainablePipe.finish_update {#finish_update tag="method"} +## TrainablePipe.finish_update {id="finish_update",tag="method"} Update parameters using the current parameter gradients. Defaults to calling [`self.model.finish_update`](https://thinc.ai/docs/api-model#finish_update). @@ -355,7 +355,7 @@ Update parameters using the current parameter gradients. Defaults to calling | ----- | ------------------------------------- | | `sgd` | An optimizer. ~~Optional[Optimizer]~~ | -## TrainablePipe.add_label {#add_label tag="method"} +## TrainablePipe.add_label {id="add_label",tag="method"} > #### Example > @@ -390,7 +390,7 @@ case, all labels found in the sample will be automatically added to the model, and the output dimension will be [inferred](/usage/layers-architectures#thinc-shape-inference) automatically. -## TrainablePipe.is_resizable {#is_resizable tag="property"} +## TrainablePipe.is_resizable {id="is_resizable",tag="property"} > #### Example > @@ -421,7 +421,7 @@ as an attribute to the component's model. | ----------- | ---------------------------------------------------------------------------------------------- | | **RETURNS** | Whether or not the output dimension of the model can be changed after initialization. ~~bool~~ | -## TrainablePipe.set_output {#set_output tag="method"} +## TrainablePipe.set_output {id="set_output",tag="method"} Change the output dimension of the component's model. If the component is not [resizable](#is_resizable), this method will raise a `NotImplementedError`. If a @@ -441,7 +441,7 @@ care should be taken to avoid the "catastrophic forgetting" problem. | ---- | --------------------------------- | | `nO` | The new output dimension. ~~int~~ | -## TrainablePipe.to_disk {#to_disk tag="method"} +## TrainablePipe.to_disk {id="to_disk",tag="method"} Serialize the pipe to disk. @@ -458,7 +458,7 @@ Serialize the pipe to disk. | _keyword-only_ | | | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | -## TrainablePipe.from_disk {#from_disk tag="method"} +## TrainablePipe.from_disk {id="from_disk",tag="method"} Load the pipe from disk. Modifies the object in place and returns it. @@ -476,7 +476,7 @@ Load the pipe from disk. Modifies the object in place and returns it. | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | | **RETURNS** | The modified pipe. ~~TrainablePipe~~ | -## TrainablePipe.to_bytes {#to_bytes tag="method"} +## TrainablePipe.to_bytes {id="to_bytes",tag="method"} > #### Example > @@ -493,7 +493,7 @@ Serialize the pipe to a bytestring. | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | | **RETURNS** | The serialized form of the pipe. ~~bytes~~ | -## TrainablePipe.from_bytes {#from_bytes tag="method"} +## TrainablePipe.from_bytes {id="from_bytes",tag="method"} Load the pipe from a bytestring. Modifies the object in place and returns it. @@ -512,7 +512,7 @@ Load the pipe from a bytestring. Modifies the object in place and returns it. | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | | **RETURNS** | The pipe. ~~TrainablePipe~~ | -## Attributes {#attributes} +## Attributes {id="attributes"} | Name | Description | | ------- | --------------------------------------------------------------------------------------------------------------------------------- | @@ -521,7 +521,7 @@ Load the pipe from a bytestring. Modifies the object in place and returns it. | `name` | The name of the component instance in the pipeline. Can be used in the losses. ~~str~~ | | `cfg` | Keyword arguments passed to [`TrainablePipe.__init__`](/api/pipe#init). Will be serialized with the component. ~~Dict[str, Any]~~ | -## Serialization fields {#serialization-fields} +## Serialization fields {id="serialization-fields"} During serialization, spaCy will export several data fields used to restore different aspects of the object. If needed, you can exclude them from diff --git a/website/docs/api/pipeline-functions.md b/website/docs/api/pipeline-functions.mdx similarity index 95% rename from website/docs/api/pipeline-functions.md rename to website/docs/api/pipeline-functions.mdx index 070292782..545ace2f2 100644 --- a/website/docs/api/pipeline-functions.md +++ b/website/docs/api/pipeline-functions.mdx @@ -10,7 +10,7 @@ menu: - ['doc_cleaner', 'doc_cleaner'] --- -## merge_noun_chunks {#merge_noun_chunks tag="function"} +## merge_noun_chunks {id="merge_noun_chunks",tag="function"} Merge noun chunks into a single token. Also available via the string name `"merge_noun_chunks"`. @@ -40,7 +40,7 @@ all other components. | `doc` | The `Doc` object to process, e.g. the `Doc` in the pipeline. ~~Doc~~ | | **RETURNS** | The modified `Doc` with merged noun chunks. ~~Doc~~ | -## merge_entities {#merge_entities tag="function"} +## merge_entities {id="merge_entities",tag="function"} Merge named entities into a single token. Also available via the string name `"merge_entities"`. @@ -70,7 +70,7 @@ components to the end of the pipeline and after all other components. | `doc` | The `Doc` object to process, e.g. the `Doc` in the pipeline. ~~Doc~~ | | **RETURNS** | The modified `Doc` with merged entities. ~~Doc~~ | -## merge_subtokens {#merge_subtokens tag="function" new="2.1"} +## merge_subtokens {id="merge_subtokens",tag="function",version="2.1"} Merge subtokens into a single token. Also available via the string name `"merge_subtokens"`. As of v2.1, the parser is able to predict "subtokens" that @@ -110,7 +110,7 @@ end of the pipeline and after all other components. | `label` | The subtoken dependency label. Defaults to `"subtok"`. ~~str~~ | | **RETURNS** | The modified `Doc` with merged subtokens. ~~Doc~~ | -## token_splitter {#token_splitter tag="function" new="3.0"} +## token_splitter {id="token_splitter",tag="function",version="3.0"} Split tokens longer than a minimum length into shorter tokens. Intended for use with transformer pipelines where long spaCy tokens lead to input text that @@ -132,7 +132,7 @@ exceed the transformer model max length. | `split_length` | The length of the split tokens. Defaults to `5`. ~~int~~ | | **RETURNS** | The modified `Doc` with the split tokens. ~~Doc~~ | -## doc_cleaner {#doc_cleaner tag="function" new="3.2.1"} +## doc_cleaner {id="doc_cleaner",tag="function",version="3.2.1"} Clean up `Doc` attributes. Intended for use at the end of pipelines with `tok2vec` or `transformer` pipeline components that store tensors and other @@ -154,7 +154,7 @@ whole pipeline has run. | `silent` | If `False`, show warnings if attributes aren't found or can't be set. Defaults to `True`. ~~bool~~ | | **RETURNS** | The modified `Doc` with the modified attributes. ~~Doc~~ | -## span_cleaner {#span_cleaner tag="function,experimental"} +## span_cleaner {id="span_cleaner",tag="function,experimental"} Remove `SpanGroup`s from `doc.spans` based on a key prefix. This is used to clean up after the [`CoreferenceResolver`](/api/coref) when it's paired with a diff --git a/website/docs/api/scorer.md b/website/docs/api/scorer.mdx similarity index 95% rename from website/docs/api/scorer.md rename to website/docs/api/scorer.mdx index 9ef36e6fc..6f0c95f6f 100644 --- a/website/docs/api/scorer.md +++ b/website/docs/api/scorer.mdx @@ -10,7 +10,7 @@ The `Scorer` computes evaluation scores. It's typically created by provides a number of evaluation methods for evaluating [`Token`](/api/token) and [`Doc`](/api/doc) attributes. -## Scorer.\_\_init\_\_ {#init tag="method"} +## Scorer.\_\_init\_\_ {id="init",tag="method"} Create a new `Scorer`. @@ -35,7 +35,7 @@ Create a new `Scorer`. | _keyword-only_ | | | `\*\*kwargs` | Any additional settings to pass on to the individual scoring methods. ~~Any~~ | -## Scorer.score {#score tag="method"} +## Scorer.score {id="score",tag="method"} Calculate the scores for a list of [`Example`](/api/example) objects using the scoring methods provided by the components in the pipeline. @@ -72,11 +72,11 @@ core pipeline components, the individual score names start with the `Token` or | `examples` | The `Example` objects holding both the predictions and the correct gold-standard annotations. ~~Iterable[Example]~~ | | **RETURNS** | A dictionary of scores. ~~Dict[str, Union[float, Dict[str, float]]]~~ | -## Scorer.score_tokenization {#score_tokenization tag="staticmethod" new="3"} +## Scorer.score_tokenization {id="score_tokenization",tag="staticmethod",version="3"} Scores the tokenization: -- `token_acc`: number of correct tokens / number of gold tokens +- `token_acc`: number of correct tokens / number of predicted tokens - `token_p`, `token_r`, `token_f`: precision, recall and F-score for token character spans @@ -93,7 +93,7 @@ Docs with `has_unknown_spaces` are skipped during scoring. | `examples` | The `Example` objects holding both the predictions and the correct gold-standard annotations. ~~Iterable[Example]~~ | | **RETURNS** | `Dict` | A dictionary containing the scores `token_acc`, `token_p`, `token_r`, `token_f`. ~~Dict[str, float]]~~ | -## Scorer.score_token_attr {#score_token_attr tag="staticmethod" new="3"} +## Scorer.score_token_attr {id="score_token_attr",tag="staticmethod",version="3"} Scores a single token attribute. Tokens with missing values in the reference doc are skipped during scoring. @@ -114,7 +114,7 @@ are skipped during scoring. | `missing_values` | Attribute values to treat as missing annotation in the reference annotation. Defaults to `{0, None, ""}`. ~~Set[Any]~~ | | **RETURNS** | A dictionary containing the score `{attr}_acc`. ~~Dict[str, float]~~ | -## Scorer.score_token_attr_per_feat {#score_token_attr_per_feat tag="staticmethod" new="3"} +## Scorer.score_token_attr_per_feat {id="score_token_attr_per_feat",tag="staticmethod",version="3"} Scores a single token attribute per feature for a token attribute in the Universal Dependencies @@ -138,7 +138,7 @@ scoring. | `missing_values` | Attribute values to treat as missing annotation in the reference annotation. Defaults to `{0, None, ""}`. ~~Set[Any]~~ | | **RETURNS** | A dictionary containing the micro PRF scores under the key `{attr}_micro_p/r/f` and the per-feature PRF scores under `{attr}_per_feat`. ~~Dict[str, Dict[str, float]]~~ | -## Scorer.score_spans {#score_spans tag="staticmethod" new="3"} +## Scorer.score_spans {id="score_spans",tag="staticmethod",version="3"} Returns PRF scores for labeled or unlabeled spans. @@ -160,7 +160,7 @@ Returns PRF scores for labeled or unlabeled spans. | `allow_overlap` | Defaults to `False`. Whether or not to allow overlapping spans. If set to `False`, the alignment will automatically resolve conflicts. ~~bool~~ | | **RETURNS** | A dictionary containing the PRF scores under the keys `{attr}_p`, `{attr}_r`, `{attr}_f` and the per-type PRF scores under `{attr}_per_type`. ~~Dict[str, Union[float, Dict[str, float]]]~~ | -## Scorer.score_deps {#score_deps tag="staticmethod" new="3"} +## Scorer.score_deps {id="score_deps",tag="staticmethod",version="3"} Calculate the UAS, LAS, and LAS per type scores for dependency parses. Tokens with missing values for the `attr` (typically `dep`) are skipped during scoring. @@ -194,7 +194,7 @@ with missing values for the `attr` (typically `dep`) are skipped during scoring. | `missing_values` | Attribute values to treat as missing annotation in the reference annotation. Defaults to `{0, None, ""}`. ~~Set[Any]~~ | | **RETURNS** | A dictionary containing the scores: `{attr}_uas`, `{attr}_las`, and `{attr}_las_per_type`. ~~Dict[str, Union[float, Dict[str, float]]]~~ | -## Scorer.score_cats {#score_cats tag="staticmethod" new="3"} +## Scorer.score_cats {id="score_cats",tag="staticmethod",version="3"} Calculate PRF and ROC AUC scores for a doc-level attribute that is a dict containing scores for each label like `Doc.cats`. The returned dictionary @@ -241,7 +241,7 @@ The reported `{attr}_score` depends on the classification properties: | `threshold` | Cutoff to consider a prediction "positive". Defaults to `0.5` for multi-label, and `0.0` (i.e. whatever's highest scoring) otherwise. ~~float~~ | | **RETURNS** | A dictionary containing the scores, with inapplicable scores as `None`. ~~Dict[str, Optional[float]]~~ | -## Scorer.score_links {#score_links tag="staticmethod" new="3"} +## Scorer.score_links {id="score_links",tag="staticmethod",version="3"} Returns PRF for predicted links on the entity level. To disentangle the performance of the NEL from the NER, this method only evaluates NEL links for @@ -264,7 +264,7 @@ entities that overlap between the gold reference and the predictions. | `negative_labels` | The string values that refer to no annotation (e.g. "NIL"). ~~Iterable[str]~~ | | **RETURNS** | A dictionary containing the scores. ~~Dict[str, Optional[float]]~~ | -## get_ner_prf {#get_ner_prf new="3"} +## get_ner_prf {id="get_ner_prf",version="3"} Compute micro-PRF and per-entity PRF scores. @@ -272,7 +272,7 @@ Compute micro-PRF and per-entity PRF scores. | ---------- | ------------------------------------------------------------------------------------------------------------------- | | `examples` | The `Example` objects holding both the predictions and the correct gold-standard annotations. ~~Iterable[Example]~~ | -## score_coref_clusters {#score_coref_clusters tag="experimental"} +## score_coref_clusters {id="score_coref_clusters",tag="experimental"} Returns LEA ([Moosavi and Strube, 2016](https://aclanthology.org/P16-1060/)) PRF scores for coreference clusters. @@ -301,7 +301,7 @@ the [CoreferenceResolver](/api/coref) docs. | `span_cluster_prefix` | The prefix used for spans representing coreference clusters. ~~str~~ | | **RETURNS** | A dictionary containing the scores. ~~Dict[str, Optional[float]]~~ | -## score_span_predictions {#score_span_predictions tag="experimental"} +## score_span_predictions {id="score_span_predictions",tag="experimental"} Return accuracy for reconstructions of spans from single tokens. Only exactly correct predictions are counted as correct, there is no partial credit for near diff --git a/website/docs/api/sentencerecognizer.md b/website/docs/api/sentencerecognizer.mdx similarity index 94% rename from website/docs/api/sentencerecognizer.md rename to website/docs/api/sentencerecognizer.mdx index 2f50350ae..5435399f9 100644 --- a/website/docs/api/sentencerecognizer.md +++ b/website/docs/api/sentencerecognizer.mdx @@ -2,7 +2,7 @@ title: SentenceRecognizer tag: class source: spacy/pipeline/senter.pyx -new: 3 +version: 3 teaser: 'Pipeline component for sentence segmentation' api_base_class: /api/tagger api_string_name: senter @@ -12,7 +12,7 @@ api_trainable: true A trainable pipeline component for sentence segmentation. For a simpler, rule-based strategy, see the [`Sentencizer`](/api/sentencizer). -## Assigned Attributes {#assigned-attributes} +## Assigned Attributes {id="assigned-attributes"} Predicted values will be assigned to `Token.is_sent_start`. The resulting sentences can be accessed using `Doc.sents`. @@ -22,7 +22,7 @@ sentences can be accessed using `Doc.sents`. | `Token.is_sent_start` | A boolean value indicating whether the token starts a sentence. This will be either `True` or `False` for all tokens. ~~bool~~ | | `Doc.sents` | An iterator over sentences in the `Doc`, determined by `Token.is_sent_start` values. ~~Iterator[Span]~~ | -## Config and implementation {#config} +## Config and implementation {id="config"} The default config is defined by the pipeline component factory and describes how the component should be configured. You can override its settings via the @@ -49,7 +49,7 @@ architectures and their arguments and hyperparameters. %%GITHUB_SPACY/spacy/pipeline/senter.pyx ``` -## SentenceRecognizer.\_\_init\_\_ {#init tag="method"} +## SentenceRecognizer.\_\_init\_\_ {id="init",tag="method"} Initialize the sentence recognizer. @@ -81,7 +81,7 @@ shortcut for this and instantiate the component using its string name and | `overwrite` 3.2 | Whether existing annotation is overwritten. Defaults to `False`. ~~bool~~ | | `scorer` 3.2 | The scoring method. Defaults to [`Scorer.score_spans`](/api/scorer#score_spans) for the attribute `"sents"`. ~~Optional[Callable]~~ | -## SentenceRecognizer.\_\_call\_\_ {#call tag="method"} +## SentenceRecognizer.\_\_call\_\_ {id="call",tag="method"} Apply the pipe to one document. The document is modified in place, and returned. This usually happens under the hood when the `nlp` object is called on a text @@ -105,7 +105,7 @@ and all pipeline components are applied to the `Doc` in order. Both | `doc` | The document to process. ~~Doc~~ | | **RETURNS** | The processed document. ~~Doc~~ | -## SentenceRecognizer.pipe {#pipe tag="method"} +## SentenceRecognizer.pipe {id="pipe",tag="method"} Apply the pipe to a stream of documents. This usually happens under the hood when the `nlp` object is called on a text and all pipeline components are @@ -129,7 +129,7 @@ and [`pipe`](/api/sentencerecognizer#pipe) delegate to the | `batch_size` | The number of documents to buffer. Defaults to `128`. ~~int~~ | | **YIELDS** | The processed documents in order. ~~Doc~~ | -## SentenceRecognizer.initialize {#initialize tag="method"} +## SentenceRecognizer.initialize {id="initialize",tag="method"} Initialize the component for training. `get_examples` should be a function that returns an iterable of [`Example`](/api/example) objects. **At least one example @@ -153,7 +153,7 @@ by [`Language.initialize`](/api/language#initialize). | _keyword-only_ | | | `nlp` | The current `nlp` object. Defaults to `None`. ~~Optional[Language]~~ | -## SentenceRecognizer.predict {#predict tag="method"} +## SentenceRecognizer.predict {id="predict",tag="method"} Apply the component's model to a batch of [`Doc`](/api/doc) objects, without modifying them. @@ -170,7 +170,7 @@ modifying them. | `docs` | The documents to predict. ~~Iterable[Doc]~~ | | **RETURNS** | The model's prediction for each document. | -## SentenceRecognizer.set_annotations {#set_annotations tag="method"} +## SentenceRecognizer.set_annotations {id="set_annotations",tag="method"} Modify a batch of [`Doc`](/api/doc) objects, using pre-computed scores. @@ -187,7 +187,7 @@ Modify a batch of [`Doc`](/api/doc) objects, using pre-computed scores. | `docs` | The documents to modify. ~~Iterable[Doc]~~ | | `scores` | The scores to set, produced by `SentenceRecognizer.predict`. | -## SentenceRecognizer.update {#update tag="method"} +## SentenceRecognizer.update {id="update",tag="method"} Learn from a batch of [`Example`](/api/example) objects containing the predictions and gold-standard annotations, and update the component's model. @@ -211,7 +211,7 @@ Delegates to [`predict`](/api/sentencerecognizer#predict) and | `losses` | Optional record of the loss during training. Updated using the component name as the key. ~~Optional[Dict[str, float]]~~ | | **RETURNS** | The updated `losses` dictionary. ~~Dict[str, float]~~ | -## SentenceRecognizer.rehearse {#rehearse tag="method,experimental" new="3"} +## SentenceRecognizer.rehearse {id="rehearse",tag="method,experimental",version="3"} Perform a "rehearsal" update from a batch of data. Rehearsal updates teach the current model to make predictions similar to an initial model to try to address @@ -234,7 +234,7 @@ the "catastrophic forgetting" problem. This feature is experimental. | `losses` | Optional record of the loss during training. Updated using the component name as the key. ~~Optional[Dict[str, float]]~~ | | **RETURNS** | The updated `losses` dictionary. ~~Dict[str, float]~~ | -## SentenceRecognizer.get_loss {#get_loss tag="method"} +## SentenceRecognizer.get_loss {id="get_loss",tag="method"} Find the loss and gradient of loss for the batch of documents and their predicted scores. @@ -253,7 +253,7 @@ predicted scores. | `scores` | Scores representing the model's predictions. | | **RETURNS** | The loss and the gradient, i.e. `(loss, gradient)`. ~~Tuple[float, float]~~ | -## SentenceRecognizer.create_optimizer {#create_optimizer tag="method"} +## SentenceRecognizer.create_optimizer {id="create_optimizer",tag="method"} Create an optimizer for the pipeline component. @@ -268,7 +268,7 @@ Create an optimizer for the pipeline component. | ----------- | ---------------------------- | | **RETURNS** | The optimizer. ~~Optimizer~~ | -## SentenceRecognizer.use_params {#use_params tag="method, contextmanager"} +## SentenceRecognizer.use_params {id="use_params",tag="method, contextmanager"} Modify the pipe's model, to use the given parameter values. At the end of the context, the original parameters are restored. @@ -285,7 +285,7 @@ context, the original parameters are restored. | -------- | -------------------------------------------------- | | `params` | The parameter values to use in the model. ~~dict~~ | -## SentenceRecognizer.to_disk {#to_disk tag="method"} +## SentenceRecognizer.to_disk {id="to_disk",tag="method"} Serialize the pipe to disk. @@ -302,7 +302,7 @@ Serialize the pipe to disk. | _keyword-only_ | | | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | -## SentenceRecognizer.from_disk {#from_disk tag="method"} +## SentenceRecognizer.from_disk {id="from_disk",tag="method"} Load the pipe from disk. Modifies the object in place and returns it. @@ -320,7 +320,7 @@ Load the pipe from disk. Modifies the object in place and returns it. | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | | **RETURNS** | The modified `SentenceRecognizer` object. ~~SentenceRecognizer~~ | -## SentenceRecognizer.to_bytes {#to_bytes tag="method"} +## SentenceRecognizer.to_bytes {id="to_bytes",tag="method"} > #### Example > @@ -337,7 +337,7 @@ Serialize the pipe to a bytestring. | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | | **RETURNS** | The serialized form of the `SentenceRecognizer` object. ~~bytes~~ | -## SentenceRecognizer.from_bytes {#from_bytes tag="method"} +## SentenceRecognizer.from_bytes {id="from_bytes",tag="method"} Load the pipe from a bytestring. Modifies the object in place and returns it. @@ -356,7 +356,7 @@ Load the pipe from a bytestring. Modifies the object in place and returns it. | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | | **RETURNS** | The `SentenceRecognizer` object. ~~SentenceRecognizer~~ | -## Serialization fields {#serialization-fields} +## Serialization fields {id="serialization-fields"} During serialization, spaCy will export several data fields used to restore different aspects of the object. If needed, you can exclude them from diff --git a/website/docs/api/sentencizer.md b/website/docs/api/sentencizer.mdx similarity index 94% rename from website/docs/api/sentencizer.md rename to website/docs/api/sentencizer.mdx index b75c7a2f1..9fb5ea71f 100644 --- a/website/docs/api/sentencizer.md +++ b/website/docs/api/sentencizer.mdx @@ -13,7 +13,7 @@ performed by the [`DependencyParser`](/api/dependencyparser), so the `Sentencizer` lets you implement a simpler, rule-based strategy that doesn't require a statistical model to be loaded. -## Assigned Attributes {#assigned-attributes} +## Assigned Attributes {id="assigned-attributes"} Calculated values will be assigned to `Token.is_sent_start`. The resulting sentences can be accessed using `Doc.sents`. @@ -23,7 +23,7 @@ sentences can be accessed using `Doc.sents`. | `Token.is_sent_start` | A boolean value indicating whether the token starts a sentence. This will be either `True` or `False` for all tokens. ~~bool~~ | | `Doc.sents` | An iterator over sentences in the `Doc`, determined by `Token.is_sent_start` values. ~~Iterator[Span]~~ | -## Config and implementation {#config} +## Config and implementation {id="config"} The default config is defined by the pipeline component factory and describes how the component should be configured. You can override its settings via the @@ -39,7 +39,7 @@ how the component should be configured. You can override its settings via the | Setting | Description | | ---------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------ | -| `punct_chars` | Optional custom list of punctuation characters that mark sentence ends. See below for defaults if not set. Defaults to `None`. ~~Optional[List[str]]~~ | `None` | +| `punct_chars` | Optional custom list of punctuation characters that mark sentence ends. See below for defaults if not set. Defaults to `None`. ~~Optional[List[str]]~~ | | `overwrite` 3.2 | Whether existing annotation is overwritten. Defaults to `False`. ~~bool~~ | | `scorer` 3.2 | The scoring method. Defaults to [`Scorer.score_spans`](/api/scorer#score_spans) for the attribute `"sents"` ~~Optional[Callable]~~ | @@ -47,7 +47,7 @@ how the component should be configured. You can override its settings via the %%GITHUB_SPACY/spacy/pipeline/sentencizer.pyx ``` -## Sentencizer.\_\_init\_\_ {#init tag="method"} +## Sentencizer.\_\_init\_\_ {id="init",tag="method"} Initialize the sentencizer. @@ -69,8 +69,7 @@ Initialize the sentencizer. | `overwrite` 3.2 | Whether existing annotation is overwritten. Defaults to `False`. ~~bool~~ | | `scorer` 3.2 | The scoring method. Defaults to [`Scorer.score_spans`](/api/scorer#score_spans) for the attribute `"sents"` ~~Optional[Callable]~~ | -```python -### punct_chars defaults +```python {title="punct_chars defaults"} ['!', '.', '?', '։', '؟', '۔', '܀', '܁', '܂', '߹', '।', '॥', '၊', '။', '።', '፧', '፨', '᙮', '᜵', '᜶', '᠃', '᠉', '᥄', '᥅', '᪨', '᪩', '᪪', '᪫', '᭚', '᭛', '᭞', '᭟', '᰻', '᰼', '᱾', '᱿', '‼', '‽', '⁇', '⁈', '⁉', @@ -83,7 +82,7 @@ Initialize the sentencizer. '𑪜', '𑱁', '𑱂', '𖩮', '𖩯', '𖫵', '𖬷', '𖬸', '𖭄', '𛲟', '𝪈', '。', '。'] ``` -## Sentencizer.\_\_call\_\_ {#call tag="method"} +## Sentencizer.\_\_call\_\_ {id="call",tag="method"} Apply the sentencizer on a `Doc`. Typically, this happens automatically after the component has been added to the pipeline using @@ -105,7 +104,7 @@ the component has been added to the pipeline using | `doc` | The `Doc` object to process, e.g. the `Doc` in the pipeline. ~~Doc~~ | | **RETURNS** | The modified `Doc` with added sentence boundaries. ~~Doc~~ | -## Sentencizer.pipe {#pipe tag="method"} +## Sentencizer.pipe {id="pipe",tag="method"} Apply the pipe to a stream of documents. This usually happens under the hood when the `nlp` object is called on a text and all pipeline components are @@ -126,7 +125,7 @@ applied to the `Doc` in order. | `batch_size` | The number of documents to buffer. Defaults to `128`. ~~int~~ | | **YIELDS** | The processed documents in order. ~~Doc~~ | -## Sentencizer.to_disk {#to_disk tag="method"} +## Sentencizer.to_disk {id="to_disk",tag="method"} Save the sentencizer settings (punctuation characters) to a directory. Will create a file `sentencizer.json`. This also happens automatically when you save @@ -144,7 +143,7 @@ an `nlp` object with a sentencizer added to its pipeline. | ------ | ------------------------------------------------------------------------------------------------------------------------------------------ | | `path` | A path to a JSON file, which will be created if it doesn't exist. Paths may be either strings or `Path`-like objects. ~~Union[str, Path]~~ | -## Sentencizer.from_disk {#from_disk tag="method"} +## Sentencizer.from_disk {id="from_disk",tag="method"} Load the sentencizer settings from a file. Expects a JSON file. This also happens automatically when you load an `nlp` object or model with a sentencizer @@ -162,7 +161,7 @@ added to its pipeline. | `path` | A path to a JSON file. Paths may be either strings or `Path`-like objects. ~~Union[str, Path]~~ | | **RETURNS** | The modified `Sentencizer` object. ~~Sentencizer~~ | -## Sentencizer.to_bytes {#to_bytes tag="method"} +## Sentencizer.to_bytes {id="to_bytes",tag="method"} Serialize the sentencizer settings to a bytestring. @@ -178,7 +177,7 @@ Serialize the sentencizer settings to a bytestring. | ----------- | ------------------------------ | | **RETURNS** | The serialized data. ~~bytes~~ | -## Sentencizer.from_bytes {#from_bytes tag="method"} +## Sentencizer.from_bytes {id="from_bytes",tag="method"} Load the pipe from a bytestring. Modifies the object in place and returns it. diff --git a/website/docs/api/span-resolver.md b/website/docs/api/span-resolver.mdx similarity index 94% rename from website/docs/api/span-resolver.md rename to website/docs/api/span-resolver.mdx index 3e992cd03..f061d8df3 100644 --- a/website/docs/api/span-resolver.md +++ b/website/docs/api/span-resolver.mdx @@ -33,7 +33,7 @@ use case is as a post-processing step on word-level [coreference resolution](/api/coref). The input and output keys used to store `Span` objects are configurable. -## Assigned Attributes {#assigned-attributes} +## Assigned Attributes {id="assigned-attributes"} Predictions will be saved to `Doc.spans` as [`SpanGroup`s](/api/spangroup). @@ -46,7 +46,7 @@ prefixes are configurable. | ------------------------------------------------- | ------------------------------------------------------------------------- | | `Doc.spans[output_prefix + "_" + cluster_number]` | One group of predicted spans. Cluster number starts from 1. ~~SpanGroup~~ | -## Config and implementation {#config} +## Config and implementation {id="config"} The default config is defined by the pipeline component factory and describes how the component should be configured. You can override its settings via the @@ -74,7 +74,7 @@ details on the architectures and their arguments and hyperparameters. | `input_prefix` | The prefix to use for input `SpanGroup`s. Defaults to `coref_head_clusters`. ~~str~~ | | `output_prefix` | The prefix for predicted `SpanGroup`s. Defaults to `coref_clusters`. ~~str~~ | -## SpanResolver.\_\_init\_\_ {#init tag="method"} +## SpanResolver.\_\_init\_\_ {id="init",tag="method"} > #### Example > @@ -104,7 +104,7 @@ shortcut for this and instantiate the component using its string name and | `input_prefix` | The prefix to use for input `SpanGroup`s. Defaults to `coref_head_clusters`. ~~str~~ | | `output_prefix` | The prefix for predicted `SpanGroup`s. Defaults to `coref_clusters`. ~~str~~ | -## SpanResolver.\_\_call\_\_ {#call tag="method"} +## SpanResolver.\_\_call\_\_ {id="call",tag="method"} Apply the pipe to one document. The document is modified in place and returned. This usually happens under the hood when the `nlp` object is called on a text @@ -126,7 +126,7 @@ and [`set_annotations`](#set_annotations) methods. | `doc` | The document to process. ~~Doc~~ | | **RETURNS** | The processed document. ~~Doc~~ | -## SpanResolver.pipe {#pipe tag="method"} +## SpanResolver.pipe {id="pipe",tag="method"} Apply the pipe to a stream of documents. This usually happens under the hood when the `nlp` object is called on a text and all pipeline components are @@ -150,7 +150,7 @@ applied to the `Doc` in order. Both [`__call__`](/api/span-resolver#call) and | `batch_size` | The number of documents to buffer. Defaults to `128`. ~~int~~ | | **YIELDS** | The processed documents in order. ~~Doc~~ | -## SpanResolver.initialize {#initialize tag="method"} +## SpanResolver.initialize {id="initialize",tag="method"} Initialize the component for training. `get_examples` should be a function that returns an iterable of [`Example`](/api/example) objects. **At least one example @@ -174,7 +174,7 @@ by [`Language.initialize`](/api/language#initialize). | _keyword-only_ | | | `nlp` | The current `nlp` object. Defaults to `None`. ~~Optional[Language]~~ | -## SpanResolver.predict {#predict tag="method"} +## SpanResolver.predict {id="predict",tag="method"} Apply the component's model to a batch of [`Doc`](/api/doc) objects, without modifying them. Predictions are returned as a list of `MentionClusters`, one for @@ -194,7 +194,7 @@ correspond to token indices. | `docs` | The documents to predict. ~~Iterable[Doc]~~ | | **RETURNS** | The predicted spans for the `Doc`s. ~~List[MentionClusters]~~ | -## SpanResolver.set_annotations {#set_annotations tag="method"} +## SpanResolver.set_annotations {id="set_annotations",tag="method"} Modify a batch of documents, saving predictions using the output prefix in `Doc.spans`. @@ -212,7 +212,7 @@ Modify a batch of documents, saving predictions using the output prefix in | `docs` | The documents to modify. ~~Iterable[Doc]~~ | | `spans` | The predicted spans for the `docs`. ~~List[MentionClusters]~~ | -## SpanResolver.update {#update tag="method"} +## SpanResolver.update {id="update",tag="method"} Learn from a batch of [`Example`](/api/example) objects. Delegates to [`predict`](/api/span-resolver#predict). @@ -234,7 +234,7 @@ Learn from a batch of [`Example`](/api/example) objects. Delegates to | `losses` | Optional record of the loss during training. Updated using the component name as the key. ~~Optional[Dict[str, float]]~~ | | **RETURNS** | The updated `losses` dictionary. ~~Dict[str, float]~~ | -## SpanResolver.create_optimizer {#create_optimizer tag="method"} +## SpanResolver.create_optimizer {id="create_optimizer",tag="method"} Create an optimizer for the pipeline component. @@ -249,7 +249,7 @@ Create an optimizer for the pipeline component. | ----------- | ---------------------------- | | **RETURNS** | The optimizer. ~~Optimizer~~ | -## SpanResolver.use_params {#use_params tag="method, contextmanager"} +## SpanResolver.use_params {id="use_params",tag="method, contextmanager"} Modify the pipe's model, to use the given parameter values. At the end of the context, the original parameters are restored. @@ -266,7 +266,7 @@ context, the original parameters are restored. | -------- | -------------------------------------------------- | | `params` | The parameter values to use in the model. ~~dict~~ | -## SpanResolver.to_disk {#to_disk tag="method"} +## SpanResolver.to_disk {id="to_disk",tag="method"} Serialize the pipe to disk. @@ -283,7 +283,7 @@ Serialize the pipe to disk. | _keyword-only_ | | | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | -## SpanResolver.from_disk {#from_disk tag="method"} +## SpanResolver.from_disk {id="from_disk",tag="method"} Load the pipe from disk. Modifies the object in place and returns it. @@ -301,7 +301,7 @@ Load the pipe from disk. Modifies the object in place and returns it. | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | | **RETURNS** | The modified `SpanResolver` object. ~~SpanResolver~~ | -## SpanResolver.to_bytes {#to_bytes tag="method"} +## SpanResolver.to_bytes {id="to_bytes",tag="method"} > #### Example > @@ -318,7 +318,7 @@ Serialize the pipe to a bytestring. | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | | **RETURNS** | The serialized form of the `SpanResolver` object. ~~bytes~~ | -## SpanResolver.from_bytes {#from_bytes tag="method"} +## SpanResolver.from_bytes {id="from_bytes",tag="method"} Load the pipe from a bytestring. Modifies the object in place and returns it. @@ -337,7 +337,7 @@ Load the pipe from a bytestring. Modifies the object in place and returns it. | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | | **RETURNS** | The `SpanResolver` object. ~~SpanResolver~~ | -## Serialization fields {#serialization-fields} +## Serialization fields {id="serialization-fields"} During serialization, spaCy will export several data fields used to restore different aspects of the object. If needed, you can exclude them from diff --git a/website/docs/api/span.md b/website/docs/api/span.mdx similarity index 81% rename from website/docs/api/span.md rename to website/docs/api/span.mdx index 69bbe8db1..41422a5b4 100644 --- a/website/docs/api/span.md +++ b/website/docs/api/span.mdx @@ -6,7 +6,7 @@ source: spacy/tokens/span.pyx A slice from a [`Doc`](/api/doc) object. -## Span.\_\_init\_\_ {#init tag="method"} +## Span.\_\_init\_\_ {id="init",tag="method"} Create a `Span` object from the slice `doc[start : end]`. @@ -29,7 +29,7 @@ Create a `Span` object from the slice `doc[start : end]`. | `kb_id` | A knowledge base ID to attach to the span, e.g. for named entities. ~~Union[str, int]~~ | | `span_id` | An ID to associate with the span. ~~Union[str, int]~~ | -## Span.\_\_getitem\_\_ {#getitem tag="method"} +## Span.\_\_getitem\_\_ {id="getitem",tag="method"} Get a `Token` object. @@ -61,7 +61,7 @@ Get a `Span` object. | `start_end` | The slice of the span to get. ~~Tuple[int, int]~~ | | **RETURNS** | The span at `span[start : end]`. ~~Span~~ | -## Span.\_\_iter\_\_ {#iter tag="method"} +## Span.\_\_iter\_\_ {id="iter",tag="method"} Iterate over `Token` objects. @@ -77,7 +77,7 @@ Iterate over `Token` objects. | ---------- | --------------------------- | | **YIELDS** | A `Token` object. ~~Token~~ | -## Span.\_\_len\_\_ {#len tag="method"} +## Span.\_\_len\_\_ {id="len",tag="method"} Get the number of tokens in the span. @@ -93,7 +93,7 @@ Get the number of tokens in the span. | ----------- | ----------------------------------------- | | **RETURNS** | The number of tokens in the span. ~~int~~ | -## Span.set_extension {#set_extension tag="classmethod" new="2"} +## Span.set_extension {id="set_extension",tag="classmethod",version="2"} Define a custom attribute on the `Span` which becomes available via `Span._`. For details, see the documentation on @@ -118,7 +118,7 @@ For details, see the documentation on | `setter` | Setter function that takes the `Span` and a value, and modifies the object. Is called when the user writes to the `Span._` attribute. ~~Optional[Callable[[Span, Any], None]]~~ | | `force` | Force overwriting existing attribute. ~~bool~~ | -## Span.get_extension {#get_extension tag="classmethod" new="2"} +## Span.get_extension {id="get_extension",tag="classmethod",version="2"} Look up a previously registered extension by name. Returns a 4-tuple `(default, method, getter, setter)` if the extension is registered. Raises a @@ -138,7 +138,7 @@ Look up a previously registered extension by name. Returns a 4-tuple | `name` | Name of the extension. ~~str~~ | | **RETURNS** | A `(default, method, getter, setter)` tuple of the extension. ~~Tuple[Optional[Any], Optional[Callable], Optional[Callable], Optional[Callable]]~~ | -## Span.has_extension {#has_extension tag="classmethod" new="2"} +## Span.has_extension {id="has_extension",tag="classmethod",version="2"} Check whether an extension has been registered on the `Span` class. @@ -155,7 +155,7 @@ Check whether an extension has been registered on the `Span` class. | `name` | Name of the extension to check. ~~str~~ | | **RETURNS** | Whether the extension has been registered. ~~bool~~ | -## Span.remove_extension {#remove_extension tag="classmethod" new="2.0.12"} +## Span.remove_extension {id="remove_extension",tag="classmethod",version="2.0.12"} Remove a previously registered extension. @@ -173,7 +173,7 @@ Remove a previously registered extension. | `name` | Name of the extension. ~~str~~ | | **RETURNS** | A `(default, method, getter, setter)` tuple of the removed extension. ~~Tuple[Optional[Any], Optional[Callable], Optional[Callable], Optional[Callable]]~~ | -## Span.char_span {#char_span tag="method" new="2.2.4"} +## Span.char_span {id="char_span",tag="method",version="2.2.4"} Create a `Span` object from the slice `span.text[start:end]`. Returns `None` if the character indices don't map to a valid span. @@ -186,16 +186,19 @@ the character indices don't map to a valid span. > assert span.text == "New York" > ``` -| Name | Description | -| ----------- | ----------------------------------------------------------------------------------------- | -| `start` | The index of the first character of the span. ~~int~~ | -| `end` | The index of the last character after the span. ~~int~~ | -| `label` | A label to attach to the span, e.g. for named entities. ~~Union[int, str]~~ | -| `kb_id` | An ID from a knowledge base to capture the meaning of a named entity. ~~Union[int, str]~~ | -| `vector` | A meaning representation of the span. ~~numpy.ndarray[ndim=1, dtype=float32]~~ | -| **RETURNS** | The newly constructed object or `None`. ~~Optional[Span]~~ | +| Name | Description | +| ----------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `start` | The index of the first character of the span. ~~int~~ | +| `end` | The index of the last character after the span. ~~int~~ | +| `label` | A label to attach to the span, e.g. for named entities. ~~Union[int, str]~~ | +| `kb_id` | An ID from a knowledge base to capture the meaning of a named entity. ~~Union[int, str]~~ | +| `vector` | A meaning representation of the span. ~~numpy.ndarray[ndim=1, dtype=float32]~~ | +| `id` | Unused. ~~Union[int, str]~~ | +| `alignment_mode` 3.5.1 | How character indices snap to token boundaries. Options: `"strict"` (no snapping), `"contract"` (span of all tokens completely within the character span), `"expand"` (span of all tokens at least partially covered by the character span). Defaults to `"strict"`. ~~str~~ | +| `span_id` 3.5.1 | An identifier to associate with the span. ~~Union[int, str]~~ | +| **RETURNS** | The newly constructed object or `None`. ~~Optional[Span]~~ | -## Span.similarity {#similarity tag="method" model="vectors"} +## Span.similarity {id="similarity",tag="method",model="vectors"} Make a semantic similarity estimate. The default estimate is cosine similarity using an average of word vectors. @@ -216,7 +219,7 @@ using an average of word vectors. | `other` | The object to compare with. By default, accepts `Doc`, `Span`, `Token` and `Lexeme` objects. ~~Union[Doc, Span, Token, Lexeme]~~ | | **RETURNS** | A scalar similarity score. Higher is more similar. ~~float~~ | -## Span.get_lca_matrix {#get_lca_matrix tag="method"} +## Span.get_lca_matrix {id="get_lca_matrix",tag="method"} Calculates the lowest common ancestor matrix for a given `Span`. Returns LCA matrix containing the integer index of the ancestor, or `-1` if no common @@ -235,7 +238,7 @@ ancestor is found, e.g. if span excludes a necessary ancestor. | ----------- | --------------------------------------------------------------------------------------- | | **RETURNS** | The lowest common ancestor matrix of the `Span`. ~~numpy.ndarray[ndim=2, dtype=int32]~~ | -## Span.to_array {#to_array tag="method" new="2"} +## Span.to_array {id="to_array",tag="method",version="2"} Given a list of `M` attribute IDs, export the tokens to a numpy `ndarray` of shape `(N, M)`, where `N` is the length of the document. The values will be @@ -256,7 +259,7 @@ shape `(N, M)`, where `N` is the length of the document. The values will be | `attr_ids` | A list of attributes (int IDs or string names) or a single attribute (int ID or string name). ~~Union[int, str, List[Union[int, str]]]~~ | | **RETURNS** | The exported attributes as a numpy array. ~~Union[numpy.ndarray[ndim=2, dtype=uint64], numpy.ndarray[ndim=1, dtype=uint64]]~~ | -## Span.ents {#ents tag="property" new="2.0.13" model="ner"} +## Span.ents {id="ents",tag="property",version="2.0.13",model="ner"} The named entities that fall completely within the span. Returns a tuple of `Span` objects. @@ -276,7 +279,7 @@ The named entities that fall completely within the span. Returns a tuple of | ----------- | ----------------------------------------------------------------- | | **RETURNS** | Entities in the span, one `Span` per entity. ~~Tuple[Span, ...]~~ | -## Span.noun_chunks {#noun_chunks tag="property" model="parser"} +## Span.noun_chunks {id="noun_chunks",tag="property",model="parser"} Iterate over the base noun phrases in the span. Yields base noun-phrase `Span` objects, if the document has been syntactically parsed. A base noun phrase, or @@ -302,7 +305,7 @@ raised. | ---------- | --------------------------------- | | **YIELDS** | Noun chunks in the span. ~~Span~~ | -## Span.as_doc {#as_doc tag="method"} +## Span.as_doc {id="as_doc",tag="method"} Create a new `Doc` object corresponding to the `Span`, with a copy of the data. @@ -326,7 +329,7 @@ time. | `array` | Precomputed array version of the original doc as generated by [`Doc.to_array`](/api/doc#to_array). ~~numpy.ndarray~~ | | **RETURNS** | A `Doc` object of the `Span`'s content. ~~Doc~~ | -## Span.root {#root tag="property" model="parser"} +## Span.root {id="root",tag="property",model="parser"} The token with the shortest path to the root of the sentence (or the root itself). If multiple tokens are equally high in the tree, the first token is @@ -347,7 +350,7 @@ taken. | ----------- | ------------------------- | | **RETURNS** | The root token. ~~Token~~ | -## Span.conjuncts {#conjuncts tag="property" model="parser"} +## Span.conjuncts {id="conjuncts",tag="property",model="parser"} A tuple of tokens coordinated to `span.root`. @@ -363,7 +366,7 @@ A tuple of tokens coordinated to `span.root`. | ----------- | --------------------------------------------- | | **RETURNS** | The coordinated tokens. ~~Tuple[Token, ...]~~ | -## Span.lefts {#lefts tag="property" model="parser"} +## Span.lefts {id="lefts",tag="property",model="parser"} Tokens that are to the left of the span, whose heads are within the span. @@ -379,7 +382,7 @@ Tokens that are to the left of the span, whose heads are within the span. | ---------- | ---------------------------------------------- | | **YIELDS** | A left-child of a token of the span. ~~Token~~ | -## Span.rights {#rights tag="property" model="parser"} +## Span.rights {id="rights",tag="property",model="parser"} Tokens that are to the right of the span, whose heads are within the span. @@ -395,7 +398,7 @@ Tokens that are to the right of the span, whose heads are within the span. | ---------- | ----------------------------------------------- | | **YIELDS** | A right-child of a token of the span. ~~Token~~ | -## Span.n_lefts {#n_lefts tag="property" model="parser"} +## Span.n_lefts {id="n_lefts",tag="property",model="parser"} The number of tokens that are to the left of the span, whose heads are within the span. @@ -411,7 +414,7 @@ the span. | ----------- | ---------------------------------------- | | **RETURNS** | The number of left-child tokens. ~~int~~ | -## Span.n_rights {#n_rights tag="property" model="parser"} +## Span.n_rights {id="n_rights",tag="property",model="parser"} The number of tokens that are to the right of the span, whose heads are within the span. @@ -427,7 +430,7 @@ the span. | ----------- | ----------------------------------------- | | **RETURNS** | The number of right-child tokens. ~~int~~ | -## Span.subtree {#subtree tag="property" model="parser"} +## Span.subtree {id="subtree",tag="property",model="parser"} Tokens within the span and tokens which descend from them. @@ -443,7 +446,7 @@ Tokens within the span and tokens which descend from them. | ---------- | ----------------------------------------------------------- | | **YIELDS** | A token within the span, or a descendant from it. ~~Token~~ | -## Span.has_vector {#has_vector tag="property" model="vectors"} +## Span.has_vector {id="has_vector",tag="property",model="vectors"} A boolean value indicating whether a word vector is associated with the object. @@ -458,7 +461,7 @@ A boolean value indicating whether a word vector is associated with the object. | ----------- | ----------------------------------------------------- | | **RETURNS** | Whether the span has a vector data attached. ~~bool~~ | -## Span.vector {#vector tag="property" model="vectors"} +## Span.vector {id="vector",tag="property",model="vectors"} A real-valued meaning representation. Defaults to an average of the token vectors. @@ -475,7 +478,7 @@ vectors. | ----------- | ----------------------------------------------------------------------------------------------- | | **RETURNS** | A 1-dimensional array representing the span's vector. ~~`numpy.ndarray[ndim=1, dtype=float32]~~ | -## Span.vector_norm {#vector_norm tag="property" model="vectors"} +## Span.vector_norm {id="vector_norm",tag="property",model="vectors"} The L2 norm of the span's vector representation. @@ -492,7 +495,7 @@ The L2 norm of the span's vector representation. | ----------- | --------------------------------------------------- | | **RETURNS** | The L2 norm of the vector representation. ~~float~~ | -## Span.sent {#sent tag="property" model="sentences"} +## Span.sent {id="sent",tag="property",model="sentences"} The sentence span that this span is a part of. This property is only available when [sentence boundaries](/usage/linguistic-features#sbd) have been set on the @@ -520,7 +523,7 @@ sent = doc[sent.start : max(sent.end, span.end)] | ----------- | ------------------------------------------------------- | | **RETURNS** | The sentence span that this span is a part of. ~~Span~~ | -## Span.sents {#sents tag="property" model="sentences" new="3.2.1"} +## Span.sents {id="sents",tag="property",model="sentences",version="3.2.1"} Returns a generator over the sentences the span belongs to. This property is only available when [sentence boundaries](/usage/linguistic-features#sbd) have @@ -542,7 +545,7 @@ overlaps with will be returned. | ----------- | -------------------------------------------------------------------------- | | **RETURNS** | A generator yielding sentences this `Span` is a part of ~~Iterable[Span]~~ | -## Attributes {#attributes} +## Attributes {id="attributes"} | Name | Description | | -------------- | ----------------------------------------------------------------------------------------------------------------------------- | diff --git a/website/docs/api/spancategorizer.md b/website/docs/api/spancategorizer.mdx similarity index 68% rename from website/docs/api/spancategorizer.md rename to website/docs/api/spancategorizer.mdx index 58a06bcf5..f54a8687b 100644 --- a/website/docs/api/spancategorizer.md +++ b/website/docs/api/spancategorizer.mdx @@ -2,7 +2,7 @@ title: SpanCategorizer tag: class,experimental source: spacy/pipeline/spancat.py -new: 3.1 +version: 3.1 teaser: 'Pipeline component for labeling potentially overlapping spans of text' api_base_class: /api/pipe api_string_name: spancat @@ -13,23 +13,33 @@ A span categorizer consists of two parts: a [suggester function](#suggesters) that proposes candidate spans, which may or may not overlap, and a labeler model that predicts zero or more labels for each candidate. -Predicted spans will be saved in a [`SpanGroup`](/api/spangroup) on the doc. -Individual span scores can be found in `spangroup.attrs["scores"]`. +This component comes in two forms: `spancat` and `spancat_singlelabel` (added in +spaCy v3.5.1). When you need to perform multi-label classification on your +spans, use `spancat`. The `spancat` component uses a `Logistic` layer where the +output class probabilities are independent for each class. However, if you need +to predict at most one true class for a span, then use `spancat_singlelabel`. It +uses a `Softmax` layer and treats the task as a multi-class problem. -## Assigned Attributes {#assigned-attributes} +Predicted spans will be saved in a [`SpanGroup`](/api/spangroup) on the doc +under `doc.spans[spans_key]`, where `spans_key` is a component config setting. +Individual span scores are stored in `doc.spans[spans_key].attrs["scores"]`. + +## Assigned Attributes {id="assigned-attributes"} Predictions will be saved to `Doc.spans[spans_key]` as a [`SpanGroup`](/api/spangroup). The scores for the spans in the `SpanGroup` will be saved in `SpanGroup.attrs["scores"]`. -`spans_key` defaults to `"sc"`, but can be passed as a parameter. +`spans_key` defaults to `"sc"`, but can be passed as a parameter. The `spancat` +component will overwrite any existing spans under the spans key +`doc.spans[spans_key]`. | Location | Value | | -------------------------------------- | -------------------------------------------------------- | | `Doc.spans[spans_key]` | The annotated spans. ~~SpanGroup~~ | | `Doc.spans[spans_key].attrs["scores"]` | The score for each span in the `SpanGroup`. ~~Floats1d~~ | -## Config and implementation {#config} +## Config and implementation {id="config"} The default config is defined by the pipeline component factory and describes how the component should be configured. You can override its settings via the @@ -38,7 +48,7 @@ how the component should be configured. You can override its settings via the [model architectures](/api/architectures) documentation for details on the architectures and their arguments and hyperparameters. -> #### Example +> #### Example (spancat) > > ```python > from spacy.pipeline.spancat import DEFAULT_SPANCAT_MODEL @@ -52,25 +62,45 @@ architectures and their arguments and hyperparameters. > nlp.add_pipe("spancat", config=config) > ``` -| Setting | Description | -| -------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `suggester` | A function that [suggests spans](#suggesters). Spans are returned as a ragged array with two integer columns, for the start and end positions. Defaults to [`ngram_suggester`](#ngram_suggester). ~~Callable[[Iterable[Doc], Optional[Ops]], Ragged]~~ | -| `model` | A model instance that is given a a list of documents and `(start, end)` indices representing candidate span offsets. The model predicts a probability for each category for each span. Defaults to [SpanCategorizer](/api/architectures#SpanCategorizer). ~~Model[Tuple[List[Doc], Ragged], Floats2d]~~ | -| `spans_key` | Key of the [`Doc.spans`](/api/doc#spans) dict to save the spans under. During initialization and training, the component will look for spans on the reference document under the same key. Defaults to `"sc"`. ~~str~~ | -| `threshold` | Minimum probability to consider a prediction positive. Spans with a positive prediction will be saved on the Doc. Defaults to `0.5`. ~~float~~ | -| `max_positive` | Maximum number of labels to consider positive per span. Defaults to `None`, indicating no limit. ~~Optional[int]~~ | -| `scorer` | The scoring method. Defaults to [`Scorer.score_spans`](/api/scorer#score_spans) for `Doc.spans[spans_key]` with overlapping spans allowed. ~~Optional[Callable]~~ | +> #### Example (spancat_singlelabel) +> +> ```python +> from spacy.pipeline.spancat import DEFAULT_SPANCAT_SINGLELABEL_MODEL +> config = { +> "threshold": 0.5, +> "spans_key": "labeled_spans", +> "model": DEFAULT_SPANCAT_SINGLELABEL_MODEL, +> "suggester": {"@misc": "spacy.ngram_suggester.v1", "sizes": [1, 2, 3]}, +> # Additional spancat_singlelabel parameters +> "negative_weight": 0.8, +> "allow_overlap": True, +> } +> nlp.add_pipe("spancat_singlelabel", config=config) +> ``` + +| Setting | Description | +| --------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `suggester` | A function that [suggests spans](#suggesters). Spans are returned as a ragged array with two integer columns, for the start and end positions. Defaults to [`ngram_suggester`](#ngram_suggester). ~~Callable[[Iterable[Doc], Optional[Ops]], Ragged]~~ | +| `model` | A model instance that is given a a list of documents and `(start, end)` indices representing candidate span offsets. The model predicts a probability for each category for each span. Defaults to [SpanCategorizer](/api/architectures#SpanCategorizer). ~~Model[Tuple[List[Doc], Ragged], Floats2d]~~ | +| `spans_key` | Key of the [`Doc.spans`](/api/doc#spans) dict to save the spans under. During initialization and training, the component will look for spans on the reference document under the same key. Defaults to `"sc"`. ~~str~~ | +| `threshold` | Minimum probability to consider a prediction positive. Spans with a positive prediction will be saved on the Doc. Meant to be used in combination with the multi-class `spancat` component with a `Logistic` scoring layer. Defaults to `0.5`. ~~float~~ | +| `max_positive` | Maximum number of labels to consider positive per span. Defaults to `None`, indicating no limit. Meant to be used together with the `spancat` component and defaults to 0 with `spancat_singlelabel`. ~~Optional[int]~~ | +| `scorer` | The scoring method. Defaults to [`Scorer.score_spans`](/api/scorer#score_spans) for `Doc.spans[spans_key]` with overlapping spans allowed. ~~Optional[Callable]~~ | +| `add_negative_label` 3.5.1 | Whether to learn to predict a special negative label for each unannotated `Span` . This should be `True` when using a `Softmax` classifier layer and so its `True` by default for `spancat_singlelabel`. Spans with negative labels and their scores are not stored as annotations. ~~bool~~ | +| `negative_weight` 3.5.1 | Multiplier for the loss terms. It can be used to downweight the negative samples if there are too many. It is only used when `add_negative_label` is `True`. Defaults to `1.0`. ~~float~~ | +| `allow_overlap` 3.5.1 | If `True`, the data is assumed to contain overlapping spans. It is only available when `max_positive` is exactly 1. Defaults to `True`. ~~bool~~ | ```python %%GITHUB_SPACY/spacy/pipeline/spancat.py ``` -## SpanCategorizer.\_\_init\_\_ {#init tag="method"} +## SpanCategorizer.\_\_init\_\_ {id="init",tag="method"} > #### Example > > ```python > # Construction via add_pipe with default model +> # Replace 'spancat' with 'spancat_singlelabel' for exclusive classes > spancat = nlp.add_pipe("spancat") > > # Construction via add_pipe with custom model @@ -86,18 +116,21 @@ Create a new pipeline instance. In your application, you would normally use a shortcut for this and instantiate the component using its string name and [`nlp.add_pipe`](/api/language#create_pipe). -| Name | Description | -| -------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -| `vocab` | The shared vocabulary. ~~Vocab~~ | -| `model` | A model instance that is given a a list of documents and `(start, end)` indices representing candidate span offsets. The model predicts a probability for each category for each span. ~~Model[Tuple[List[Doc], Ragged], Floats2d]~~ | -| `suggester` | A function that [suggests spans](#suggesters). Spans are returned as a ragged array with two integer columns, for the start and end positions. ~~Callable[[Iterable[Doc], Optional[Ops]], Ragged]~~ | -| `name` | String name of the component instance. Used to add entries to the `losses` during training. ~~str~~ | -| _keyword-only_ | | -| `spans_key` | Key of the [`Doc.spans`](/api/doc#sans) dict to save the spans under. During initialization and training, the component will look for spans on the reference document under the same key. Defaults to `"sc"`. ~~str~~ | -| `threshold` | Minimum probability to consider a prediction positive. Spans with a positive prediction will be saved on the Doc. Defaults to `0.5`. ~~float~~ | -| `max_positive` | Maximum number of labels to consider positive per span. Defaults to `None`, indicating no limit. ~~Optional[int]~~ | +| Name | Description | +| --------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `vocab` | The shared vocabulary. ~~Vocab~~ | +| `model` | A model instance that is given a a list of documents and `(start, end)` indices representing candidate span offsets. The model predicts a probability for each category for each span. ~~Model[Tuple[List[Doc], Ragged], Floats2d]~~ | +| `suggester` | A function that [suggests spans](#suggesters). Spans are returned as a ragged array with two integer columns, for the start and end positions. ~~Callable[[Iterable[Doc], Optional[Ops]], Ragged]~~ | +| `name` | String name of the component instance. Used to add entries to the `losses` during training. ~~str~~ | +| _keyword-only_ | | +| `spans_key` | Key of the [`Doc.spans`](/api/doc#sans) dict to save the spans under. During initialization and training, the component will look for spans on the reference document under the same key. Defaults to `"sc"`. ~~str~~ | +| `threshold` | Minimum probability to consider a prediction positive. Spans with a positive prediction will be saved on the Doc. Defaults to `0.5`. ~~float~~ | +| `max_positive` | Maximum number of labels to consider positive per span. Defaults to `None`, indicating no limit. ~~Optional[int]~~ | +| `allow_overlap` 3.5.1 | If `True`, the data is assumed to contain overlapping spans. It is only available when `max_positive` is exactly 1. Defaults to `True`. ~~bool~~ | +| `add_negative_label` 3.5.1 | Whether to learn to predict a special negative label for each unannotated `Span`. This should be `True` when using a `Softmax` classifier layer and so its `True` by default for `spancat_singlelabel` . Spans with negative labels and their scores are not stored as annotations. ~~bool~~ | +| `negative_weight` 3.5.1 | Multiplier for the loss terms. It can be used to downweight the negative samples if there are too many . It is only used when `add_negative_label` is `True`. Defaults to `1.0`. ~~float~~ | -## SpanCategorizer.\_\_call\_\_ {#call tag="method"} +## SpanCategorizer.\_\_call\_\_ {id="call",tag="method"} Apply the pipe to one document. The document is modified in place, and returned. This usually happens under the hood when the `nlp` object is called on a text @@ -120,7 +153,7 @@ delegate to the [`predict`](/api/spancategorizer#predict) and | `doc` | The document to process. ~~Doc~~ | | **RETURNS** | The processed document. ~~Doc~~ | -## SpanCategorizer.pipe {#pipe tag="method"} +## SpanCategorizer.pipe {id="pipe",tag="method"} Apply the pipe to a stream of documents. This usually happens under the hood when the `nlp` object is called on a text and all pipeline components are @@ -144,7 +177,7 @@ applied to the `Doc` in order. Both [`__call__`](/api/spancategorizer#call) and | `batch_size` | The number of documents to buffer. Defaults to `128`. ~~int~~ | | **YIELDS** | The processed documents in order. ~~Doc~~ | -## SpanCategorizer.initialize {#initialize tag="method"} +## SpanCategorizer.initialize {id="initialize",tag="method"} Initialize the component for training. `get_examples` should be a function that returns an iterable of [`Example`](/api/example) objects. **At least one example @@ -181,7 +214,7 @@ config. | `nlp` | The current `nlp` object. Defaults to `None`. ~~Optional[Language]~~ | | `labels` | The label information to add to the component, as provided by the [`label_data`](#label_data) property after initialization. To generate a reusable JSON file from your data, you should run the [`init labels`](/api/cli#init-labels) command. If no labels are provided, the `get_examples` callback is used to extract the labels from the data, which may be a lot slower. ~~Optional[Iterable[str]]~~ | -## SpanCategorizer.predict {#predict tag="method"} +## SpanCategorizer.predict {id="predict",tag="method"} Apply the component's model to a batch of [`Doc`](/api/doc) objects without modifying them. @@ -198,7 +231,7 @@ modifying them. | `docs` | The documents to predict. ~~Iterable[Doc]~~ | | **RETURNS** | The model's prediction for each document. | -## SpanCategorizer.set_annotations {#set_annotations tag="method"} +## SpanCategorizer.set_annotations {id="set_annotations",tag="method"} Modify a batch of [`Doc`](/api/doc) objects using pre-computed scores. @@ -215,7 +248,7 @@ Modify a batch of [`Doc`](/api/doc) objects using pre-computed scores. | `docs` | The documents to modify. ~~Iterable[Doc]~~ | | `scores` | The scores to set, produced by `SpanCategorizer.predict`. | -## SpanCategorizer.update {#update tag="method"} +## SpanCategorizer.update {id="update",tag="method"} Learn from a batch of [`Example`](/api/example) objects containing the predictions and gold-standard annotations, and update the component's model. @@ -239,7 +272,7 @@ Delegates to [`predict`](/api/spancategorizer#predict) and | `losses` | Optional record of the loss during training. Updated using the component name as the key. ~~Optional[Dict[str, float]]~~ | | **RETURNS** | The updated `losses` dictionary. ~~Dict[str, float]~~ | -## SpanCategorizer.set_candidates {#set_candidates tag="method", new="3.3"} +## SpanCategorizer.set_candidates {id="set_candidates",tag="method", version="3.3"} Use the suggester to add a list of [`Span`](/api/span) candidates to a list of [`Doc`](/api/doc) objects. This method is intended to be used for debugging @@ -257,7 +290,7 @@ purposes. | `docs` | The documents to modify. ~~Iterable[Doc]~~ | | `candidates_key` | Key of the Doc.spans dict to save the candidate spans under. ~~str~~ | -## SpanCategorizer.get_loss {#get_loss tag="method"} +## SpanCategorizer.get_loss {id="get_loss",tag="method"} Find the loss and gradient of loss for the batch of documents and their predicted scores. @@ -276,7 +309,7 @@ predicted scores. | `spans_scores` | Scores representing the model's predictions. ~~Tuple[Ragged, Floats2d]~~ | | **RETURNS** | The loss and the gradient, i.e. `(loss, gradient)`. ~~Tuple[float, float]~~ | -## SpanCategorizer.create_optimizer {#create_optimizer tag="method"} +## SpanCategorizer.create_optimizer {id="create_optimizer",tag="method"} Create an optimizer for the pipeline component. @@ -291,7 +324,7 @@ Create an optimizer for the pipeline component. | ----------- | ---------------------------- | | **RETURNS** | The optimizer. ~~Optimizer~~ | -## SpanCategorizer.use_params {#use_params tag="method, contextmanager"} +## SpanCategorizer.use_params {id="use_params",tag="method, contextmanager"} Modify the pipe's model to use the given parameter values. @@ -307,7 +340,7 @@ Modify the pipe's model to use the given parameter values. | -------- | -------------------------------------------------- | | `params` | The parameter values to use in the model. ~~dict~~ | -## SpanCategorizer.add_label {#add_label tag="method"} +## SpanCategorizer.add_label {id="add_label",tag="method"} Add a new label to the pipe. Raises an error if the output dimension is already set, or if the model has already been fully [initialized](#initialize). Note @@ -329,7 +362,7 @@ automatically. | `label` | The label to add. ~~str~~ | | **RETURNS** | `0` if the label is already present, otherwise `1`. ~~int~~ | -## SpanCategorizer.to_disk {#to_disk tag="method"} +## SpanCategorizer.to_disk {id="to_disk",tag="method"} Serialize the pipe to disk. @@ -346,7 +379,7 @@ Serialize the pipe to disk. | _keyword-only_ | | | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | -## SpanCategorizer.from_disk {#from_disk tag="method"} +## SpanCategorizer.from_disk {id="from_disk",tag="method"} Load the pipe from disk. Modifies the object in place and returns it. @@ -364,7 +397,7 @@ Load the pipe from disk. Modifies the object in place and returns it. | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | | **RETURNS** | The modified `SpanCategorizer` object. ~~SpanCategorizer~~ | -## SpanCategorizer.to_bytes {#to_bytes tag="method"} +## SpanCategorizer.to_bytes {id="to_bytes",tag="method"} > #### Example > @@ -381,7 +414,7 @@ Serialize the pipe to a bytestring. | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | | **RETURNS** | The serialized form of the `SpanCategorizer` object. ~~bytes~~ | -## SpanCategorizer.from_bytes {#from_bytes tag="method"} +## SpanCategorizer.from_bytes {id="from_bytes",tag="method"} Load the pipe from a bytestring. Modifies the object in place and returns it. @@ -400,7 +433,7 @@ Load the pipe from a bytestring. Modifies the object in place and returns it. | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | | **RETURNS** | The `SpanCategorizer` object. ~~SpanCategorizer~~ | -## SpanCategorizer.labels {#labels tag="property"} +## SpanCategorizer.labels {id="labels",tag="property"} The labels currently added to the component. @@ -415,7 +448,7 @@ The labels currently added to the component. | ----------- | ------------------------------------------------------ | | **RETURNS** | The labels added to the component. ~~Tuple[str, ...]~~ | -## SpanCategorizer.label_data {#label_data tag="property"} +## SpanCategorizer.label_data {id="label_data",tag="property"} The labels currently added to the component and their internal meta information. This is the data generated by [`init labels`](/api/cli#init-labels) and used by @@ -433,7 +466,7 @@ the model with a pre-defined label set. | ----------- | ---------------------------------------------------------- | | **RETURNS** | The label data added to the component. ~~Tuple[str, ...]~~ | -## Serialization fields {#serialization-fields} +## Serialization fields {id="serialization-fields"} During serialization, spaCy will export several data fields used to restore different aspects of the object. If needed, you can exclude them from @@ -451,9 +484,9 @@ serialization by passing in the string names via the `exclude` argument. | `cfg` | The config file. You usually don't want to exclude this. | | `model` | The binary model data. You usually don't want to exclude this. | -## Suggesters {#suggesters tag="registered functions" source="spacy/pipeline/spancat.py"} +## Suggesters {id="suggesters",tag="registered functions",source="spacy/pipeline/spancat.py"} -### spacy.ngram_suggester.v1 {#ngram_suggester} +### spacy.ngram_suggester.v1 {id="ngram_suggester"} > #### Example Config > @@ -471,7 +504,7 @@ integers. The array has two columns, indicating the start and end position. | `sizes` | The phrase lengths to suggest. For example, `[1, 2]` will suggest phrases consisting of 1 or 2 tokens. ~~List[int]~~ | | **CREATES** | The suggester function. ~~Callable[[Iterable[Doc], Optional[Ops]], Ragged]~~ | -### spacy.ngram_range_suggester.v1 {#ngram_range_suggester} +### spacy.ngram_range_suggester.v1 {id="ngram_range_suggester"} > #### Example Config > diff --git a/website/docs/api/spangroup.md b/website/docs/api/spangroup.mdx similarity index 88% rename from website/docs/api/spangroup.md rename to website/docs/api/spangroup.mdx index 2d1cf73c4..cd0accb6a 100644 --- a/website/docs/api/spangroup.md +++ b/website/docs/api/spangroup.mdx @@ -2,7 +2,7 @@ title: SpanGroup tag: class source: spacy/tokens/span_group.pyx -new: 3 +version: 3 --- A group of arbitrary, potentially overlapping [`Span`](/api/span) objects that @@ -13,7 +13,7 @@ into a `SpanGroup` object for you automatically on assignment. `SpanGroup` objects behave similar to `list`s, so you can append `Span` objects to them or access a member at a given index. -## SpanGroup.\_\_init\_\_ {#init tag="method"} +## SpanGroup.\_\_init\_\_ {id="init",tag="method"} Create a `SpanGroup`. @@ -42,7 +42,7 @@ Create a `SpanGroup`. | `attrs` | Optional JSON-serializable attributes to attach to the span group. ~~Dict[str, Any]~~ | | `spans` | The spans to add to the span group. ~~Iterable[Span]~~ | -## SpanGroup.doc {#doc tag="property"} +## SpanGroup.doc {id="doc",tag="property"} The [`Doc`](/api/doc) object the span group is referring to. @@ -68,7 +68,7 @@ the scope of your function. | ----------- | ------------------------------- | | **RETURNS** | The reference document. ~~Doc~~ | -## SpanGroup.has_overlap {#has_overlap tag="property"} +## SpanGroup.has_overlap {id="has_overlap",tag="property"} Check whether the span group contains overlapping spans. @@ -86,7 +86,7 @@ Check whether the span group contains overlapping spans. | ----------- | -------------------------------------------------- | | **RETURNS** | Whether the span group contains overlaps. ~~bool~~ | -## SpanGroup.\_\_len\_\_ {#len tag="method"} +## SpanGroup.\_\_len\_\_ {id="len",tag="method"} Get the number of spans in the group. @@ -102,7 +102,7 @@ Get the number of spans in the group. | ----------- | ----------------------------------------- | | **RETURNS** | The number of spans in the group. ~~int~~ | -## SpanGroup.\_\_getitem\_\_ {#getitem tag="method"} +## SpanGroup.\_\_getitem\_\_ {id="getitem",tag="method"} Get a span from the group. Note that a copy of the span is returned, so if any changes are made to this span, they are not reflected in the corresponding @@ -125,7 +125,7 @@ changes to be reflected in the span group. | `i` | The item index. ~~int~~ | | **RETURNS** | The span at the given index. ~~Span~~ | -## SpanGroup.\_\_setitem\_\_ {#setitem tag="method", new="3.3"} +## SpanGroup.\_\_setitem\_\_ {id="setitem",tag="method", version="3.3"} Set a span in the span group. @@ -144,7 +144,7 @@ Set a span in the span group. | `i` | The item index. ~~int~~ | | `span` | The new value. ~~Span~~ | -## SpanGroup.\_\_delitem\_\_ {#delitem tag="method", new="3.3"} +## SpanGroup.\_\_delitem\_\_ {id="delitem",tag="method", version="3.3"} Delete a span from the span group. @@ -161,7 +161,7 @@ Delete a span from the span group. | ---- | ----------------------- | | `i` | The item index. ~~int~~ | -## SpanGroup.\_\_add\_\_ {#add tag="method", new="3.3"} +## SpanGroup.\_\_add\_\_ {id="add",tag="method", version="3.3"} Concatenate the current span group with another span group and return the result in a new span group. Any `attrs` from the first span group will have precedence @@ -182,7 +182,7 @@ over `attrs` in the second. | `other` | The span group or spans to concatenate. ~~Union[SpanGroup, Iterable[Span]]~~ | | **RETURNS** | The new span group. ~~SpanGroup~~ | -## SpanGroup.\_\_iadd\_\_ {#iadd tag="method", new="3.3"} +## SpanGroup.\_\_iadd\_\_ {id="iadd",tag="method", version="3.3"} Append an iterable of spans or the content of a span group to the current span group. Any `attrs` in the other span group will be added for keys that are not @@ -202,7 +202,25 @@ already present in the current span group. | `other` | The span group or spans to append. ~~Union[SpanGroup, Iterable[Span]]~~ | | **RETURNS** | The span group. ~~SpanGroup~~ | -## SpanGroup.append {#append tag="method"} +## SpanGroup.\_\_iter\_\_ {id="iter",tag="method",version="3.5"} + +Iterate over the spans in this span group. + +> #### Example +> +> ```python +> doc = nlp("Their goi ng home") +> doc.spans["errors"] = [doc[0:1], doc[1:3]] +> for error_span in doc.spans["errors"]: +> print(error_span) +> ``` + +| Name | Description | +| ---------- | ----------------------------------- | +| **YIELDS** | A span in this span group. ~~Span~~ | + + +## SpanGroup.append {id="append",tag="method"} Add a [`Span`](/api/span) object to the group. The span must refer to the same [`Doc`](/api/doc) object as the span group. @@ -220,7 +238,7 @@ Add a [`Span`](/api/span) object to the group. The span must refer to the same | ------ | ---------------------------- | | `span` | The span to append. ~~Span~~ | -## SpanGroup.extend {#extend tag="method"} +## SpanGroup.extend {id="extend",tag="method"} Add multiple [`Span`](/api/span) objects or contents of another `SpanGroup` to the group. All spans must refer to the same [`Doc`](/api/doc) object as the span @@ -241,7 +259,7 @@ group. | ------- | -------------------------------------------------------- | | `spans` | The spans to add. ~~Union[SpanGroup, Iterable["Span"]]~~ | -## SpanGroup.copy {#copy tag="method", new="3.3"} +## SpanGroup.copy {id="copy",tag="method", version="3.3"} Return a copy of the span group. @@ -260,7 +278,7 @@ Return a copy of the span group. | `doc` | The document to which the copy is bound. Defaults to `None` for the current doc. ~~Optional[Doc]~~ | | **RETURNS** | A copy of the `SpanGroup` object. ~~SpanGroup~~ | -## SpanGroup.to_bytes {#to_bytes tag="method"} +## SpanGroup.to_bytes {id="to_bytes",tag="method"} Serialize the span group to a bytestring. @@ -276,7 +294,7 @@ Serialize the span group to a bytestring. | ----------- | ------------------------------------- | | **RETURNS** | The serialized `SpanGroup`. ~~bytes~~ | -## SpanGroup.from_bytes {#from_bytes tag="method"} +## SpanGroup.from_bytes {id="from_bytes",tag="method"} Load the span group from a bytestring. Modifies the object in place and returns it. diff --git a/website/docs/api/spanruler.md b/website/docs/api/spanruler.mdx similarity index 64% rename from website/docs/api/spanruler.md rename to website/docs/api/spanruler.mdx index b573f7c58..d2d41f620 100644 --- a/website/docs/api/spanruler.md +++ b/website/docs/api/spanruler.mdx @@ -2,7 +2,7 @@ title: SpanRuler tag: class source: spacy/pipeline/span_ruler.py -new: 3.3 +version: 3.3 teaser: 'Pipeline component for rule-based span and named entity recognition' api_string_name: span_ruler api_trainable: false @@ -13,7 +13,7 @@ The span ruler lets you add spans to [`Doc.spans`](/api/doc#spans) and/or usage examples, see the docs on [rule-based span matching](/usage/rule-based-matching#spanruler). -## Assigned Attributes {#assigned-attributes} +## Assigned Attributes {id="assigned-attributes"} Matches will be saved to `Doc.spans[spans_key]` as a [`SpanGroup`](/api/spangroup) and/or to `Doc.ents`, where the annotation is @@ -28,7 +28,7 @@ saved in the `Token.ent_type` and `Token.ent_iob` fields. | `Token.ent_type` | The label part of the named entity tag (hash). ~~int~~ | | `Token.ent_type_` | The label part of the named entity tag. ~~str~~ | -## Config and implementation {#config} +## Config and implementation {id="config"} The default config is defined by the pipeline component factory and describes how the component should be configured. You can override its settings via the @@ -46,22 +46,23 @@ how the component should be configured. You can override its settings via the > nlp.add_pipe("span_ruler", config=config) > ``` -| Setting | Description | -| --------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `spans_key` | The spans key to save the spans under. If `None`, no spans are saved. Defaults to `"ruler"`. ~~Optional[str]~~ | -| `spans_filter` | The optional method to filter spans before they are assigned to doc.spans. Defaults to `None`. ~~Optional[Callable[[Iterable[Span], Iterable[Span]], List[Span]]]~~ | -| `annotate_ents` | Whether to save spans to doc.ents. Defaults to `False`. ~~bool~~ | -| `ents_filter` | The method to filter spans before they are assigned to doc.ents. Defaults to `util.filter_chain_spans`. ~~Callable[[Iterable[Span], Iterable[Span]], List[Span]]~~ | -| `phrase_matcher_attr` | Token attribute to match on, passed to the internal PhraseMatcher as `attr`. Defaults to `None`. ~~Optional[Union[int, str]]~~ | -| `validate` | Whether patterns should be validated, passed to Matcher and PhraseMatcher as `validate`. Defaults to `False`. ~~bool~~ | -| `overwrite` | Whether to remove any existing spans under `Doc.spans[spans key]` if `spans_key` is set, or to remove any ents under `Doc.ents` if `annotate_ents` is set. Defaults to `True`. ~~bool~~ | -| `scorer` | The scoring method. Defaults to [`Scorer.score_spans`](/api/scorer#score_spans) for `Doc.spans[spans_key]` with overlapping spans allowed. ~~Optional[Callable]~~ | +| Setting | Description | +| ---------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `spans_key` | The spans key to save the spans under. If `None`, no spans are saved. Defaults to `"ruler"`. ~~Optional[str]~~ | +| `spans_filter` | The optional method to filter spans before they are assigned to doc.spans. Defaults to `None`. ~~Optional[Callable[[Iterable[Span], Iterable[Span]], List[Span]]]~~ | +| `annotate_ents` | Whether to save spans to doc.ents. Defaults to `False`. ~~bool~~ | +| `ents_filter` | The method to filter spans before they are assigned to doc.ents. Defaults to `util.filter_chain_spans`. ~~Callable[[Iterable[Span], Iterable[Span]], List[Span]]~~ | +| `phrase_matcher_attr` | Token attribute to match on, passed to the internal `PhraseMatcher` as `attr`. Defaults to `None`. ~~Optional[Union[int, str]]~~ | +| `matcher_fuzzy_compare` 3.5 | The fuzzy comparison method, passed on to the internal `Matcher`. Defaults to `spacy.matcher.levenshtein.levenshtein_compare`. ~~Callable~~ | +| `validate` | Whether patterns should be validated, passed to `Matcher` and `PhraseMatcher` as `validate`. Defaults to `False`. ~~bool~~ | +| `overwrite` | Whether to remove any existing spans under `Doc.spans[spans key]` if `spans_key` is set, or to remove any ents under `Doc.ents` if `annotate_ents` is set. Defaults to `True`. ~~bool~~ | +| `scorer` | The scoring method. Defaults to [`Scorer.score_spans`](/api/scorer#score_spans) for `Doc.spans[spans_key]` with overlapping spans allowed. ~~Optional[Callable]~~ | ```python %%GITHUB_SPACY/spacy/pipeline/span_ruler.py ``` -## SpanRuler.\_\_init\_\_ {#init tag="method"} +## SpanRuler.\_\_init\_\_ {id="init",tag="method"} Initialize the span ruler. If patterns are supplied here, they need to be a list of dictionaries with a `"label"` and `"pattern"` key. A pattern can either be a @@ -79,21 +80,22 @@ token pattern (list) or a phrase pattern (string). For example: > ruler = SpanRuler(nlp, overwrite=True) > ``` -| Name | Description | -| --------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `nlp` | The shared nlp object to pass the vocab to the matchers and process phrase patterns. ~~Language~~ | -| `name` | Instance name of the current pipeline component. Typically passed in automatically from the factory when the component is added. Used to disable the current span ruler while creating phrase patterns with the nlp object. ~~str~~ | -| _keyword-only_ | | -| `spans_key` | The spans key to save the spans under. If `None`, no spans are saved. Defaults to `"ruler"`. ~~Optional[str]~~ | -| `spans_filter` | The optional method to filter spans before they are assigned to doc.spans. Defaults to `None`. ~~Optional[Callable[[Iterable[Span], Iterable[Span]], List[Span]]]~~ | -| `annotate_ents` | Whether to save spans to doc.ents. Defaults to `False`. ~~bool~~ | -| `ents_filter` | The method to filter spans before they are assigned to doc.ents. Defaults to `util.filter_chain_spans`. ~~Callable[[Iterable[Span], Iterable[Span]], List[Span]]~~ | -| `phrase_matcher_attr` | Token attribute to match on, passed to the internal PhraseMatcher as `attr`. Defaults to `None`. ~~Optional[Union[int, str]]~~ | -| `validate` | Whether patterns should be validated, passed to Matcher and PhraseMatcher as `validate`. Defaults to `False`. ~~bool~~ | -| `overwrite` | Whether to remove any existing spans under `Doc.spans[spans key]` if `spans_key` is set, or to remove any ents under `Doc.ents` if `annotate_ents` is set. Defaults to `True`. ~~bool~~ | -| `scorer` | The scoring method. Defaults to [`Scorer.score_spans`](/api/scorer#score_spans) for `Doc.spans[spans_key]` with overlapping spans allowed. ~~Optional[Callable]~~ | +| Name | Description | +| ---------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `nlp` | The shared nlp object to pass the vocab to the matchers and process phrase patterns. ~~Language~~ | +| `name` | Instance name of the current pipeline component. Typically passed in automatically from the factory when the component is added. Used to disable the current span ruler while creating phrase patterns with the nlp object. ~~str~~ | +| _keyword-only_ | | +| `spans_key` | The spans key to save the spans under. If `None`, no spans are saved. Defaults to `"ruler"`. ~~Optional[str]~~ | +| `spans_filter` | The optional method to filter spans before they are assigned to doc.spans. Defaults to `None`. ~~Optional[Callable[[Iterable[Span], Iterable[Span]], List[Span]]]~~ | +| `annotate_ents` | Whether to save spans to doc.ents. Defaults to `False`. ~~bool~~ | +| `ents_filter` | The method to filter spans before they are assigned to doc.ents. Defaults to `util.filter_chain_spans`. ~~Callable[[Iterable[Span], Iterable[Span]], List[Span]]~~ | +| `phrase_matcher_attr` | Token attribute to match on, passed to the internal PhraseMatcher as `attr`. Defaults to `None`. ~~Optional[Union[int, str]]~~ | +| `matcher_fuzzy_compare` 3.5 | The fuzzy comparison method, passed on to the internal `Matcher`. Defaults to `spacy.matcher.levenshtein.levenshtein_compare`. ~~Callable~~ | +| `validate` | Whether patterns should be validated, passed to Matcher and PhraseMatcher as `validate`. Defaults to `False`. ~~bool~~ | +| `overwrite` | Whether to remove any existing spans under `Doc.spans[spans key]` if `spans_key` is set, or to remove any ents under `Doc.ents` if `annotate_ents` is set. Defaults to `True`. ~~bool~~ | +| `scorer` | The scoring method. Defaults to [`Scorer.score_spans`](/api/scorer#score_spans) for `Doc.spans[spans_key]` with overlapping spans allowed. ~~Optional[Callable]~~ | -## SpanRuler.initialize {#initialize tag="method"} +## SpanRuler.initialize {id="initialize",tag="method"} Initialize the component with data and used before training to load in rules from a [pattern file](/usage/rule-based-matching/#spanruler-files). This method @@ -125,7 +127,7 @@ config. Any existing patterns are removed on initialization. | `nlp` | The current `nlp` object. Defaults to `None`. ~~Optional[Language]~~ | | `patterns` | The list of patterns. Defaults to `None`. ~~Optional[Sequence[Dict[str, Union[str, List[Dict[str, Any]]]]]]~~ | -## SpanRuler.\_\len\_\_ {#len tag="method"} +## SpanRuler.\_\_len\_\_ {id="len",tag="method"} The number of all patterns added to the span ruler. @@ -142,7 +144,7 @@ The number of all patterns added to the span ruler. | ----------- | ------------------------------- | | **RETURNS** | The number of patterns. ~~int~~ | -## SpanRuler.\_\_contains\_\_ {#contains tag="method"} +## SpanRuler.\_\_contains\_\_ {id="contains",tag="method"} Whether a label is present in the patterns. @@ -160,7 +162,7 @@ Whether a label is present in the patterns. | `label` | The label to check. ~~str~~ | | **RETURNS** | Whether the span ruler contains the label. ~~bool~~ | -## SpanRuler.\_\_call\_\_ {#call tag="method"} +## SpanRuler.\_\_call\_\_ {id="call",tag="method"} Find matches in the `Doc` and add them to `doc.spans[span_key]` and/or `doc.ents`. Typically, this happens automatically after the component has been @@ -184,7 +186,7 @@ will be removed. | `doc` | The `Doc` object to process, e.g. the `Doc` in the pipeline. ~~Doc~~ | | **RETURNS** | The modified `Doc` with added spans/entities. ~~Doc~~ | -## SpanRuler.add_patterns {#add_patterns tag="method"} +## SpanRuler.add_patterns {id="add_patterns",tag="method"} Add patterns to the span ruler. A pattern can either be a token pattern (list of dicts) or a phrase pattern (string). For more details, see the usage guide on @@ -205,7 +207,7 @@ dicts) or a phrase pattern (string). For more details, see the usage guide on | ---------- | ---------------------------------------------------------------- | | `patterns` | The patterns to add. ~~List[Dict[str, Union[str, List[dict]]]]~~ | -## SpanRuler.remove {#remove tag="method"} +## SpanRuler.remove {id="remove",tag="method"} Remove patterns by label from the span ruler. A `ValueError` is raised if the label does not exist in any patterns. @@ -223,7 +225,7 @@ label does not exist in any patterns. | ------- | -------------------------------------- | | `label` | The label of the pattern rule. ~~str~~ | -## SpanRuler.remove_by_id {#remove_by_id tag="method"} +## SpanRuler.remove_by_id {id="remove_by_id",tag="method"} Remove patterns by ID from the span ruler. A `ValueError` is raised if the ID does not exist in any patterns. @@ -241,7 +243,7 @@ does not exist in any patterns. | ------------ | ----------------------------------- | | `pattern_id` | The ID of the pattern rule. ~~str~~ | -## SpanRuler.clear {#clear tag="method"} +## SpanRuler.clear {id="clear",tag="method"} Remove all patterns the span ruler. @@ -254,7 +256,7 @@ Remove all patterns the span ruler. > ruler.clear() > ``` -## SpanRuler.to_disk {#to_disk tag="method"} +## SpanRuler.to_disk {id="to_disk",tag="method"} Save the span ruler patterns to a directory. The patterns will be saved as newline-delimited JSON (JSONL). @@ -270,7 +272,7 @@ newline-delimited JSON (JSONL). | ------ | ------------------------------------------------------------------------------------------------------------------------------------------ | | `path` | A path to a directory, which will be created if it doesn't exist. Paths may be either strings or `Path`-like objects. ~~Union[str, Path]~~ | -## SpanRuler.from_disk {#from_disk tag="method"} +## SpanRuler.from_disk {id="from_disk",tag="method"} Load the span ruler from a path. @@ -286,7 +288,7 @@ Load the span ruler from a path. | `path` | A path to a directory. Paths may be either strings or `Path`-like objects. ~~Union[str, Path]~~ | | **RETURNS** | The modified `SpanRuler` object. ~~SpanRuler~~ | -## SpanRuler.to_bytes {#to_bytes tag="method"} +## SpanRuler.to_bytes {id="to_bytes",tag="method"} Serialize the span ruler to a bytestring. @@ -301,7 +303,7 @@ Serialize the span ruler to a bytestring. | ----------- | ---------------------------------- | | **RETURNS** | The serialized patterns. ~~bytes~~ | -## SpanRuler.from_bytes {#from_bytes tag="method"} +## SpanRuler.from_bytes {id="from_bytes",tag="method"} Load the pipe from a bytestring. Modifies the object in place and returns it. @@ -318,7 +320,7 @@ Load the pipe from a bytestring. Modifies the object in place and returns it. | `bytes_data` | The bytestring to load. ~~bytes~~ | | **RETURNS** | The modified `SpanRuler` object. ~~SpanRuler~~ | -## SpanRuler.labels {#labels tag="property"} +## SpanRuler.labels {id="labels",tag="property"} All labels present in the match patterns. @@ -326,7 +328,7 @@ All labels present in the match patterns. | ----------- | -------------------------------------- | | **RETURNS** | The string labels. ~~Tuple[str, ...]~~ | -## SpanRuler.ids {#ids tag="property"} +## SpanRuler.ids {id="ids",tag="property"} All IDs present in the `id` property of the match patterns. @@ -334,7 +336,7 @@ All IDs present in the `id` property of the match patterns. | ----------- | ----------------------------------- | | **RETURNS** | The string IDs. ~~Tuple[str, ...]~~ | -## SpanRuler.patterns {#patterns tag="property"} +## SpanRuler.patterns {id="patterns",tag="property"} All patterns that were added to the span ruler. @@ -342,7 +344,7 @@ All patterns that were added to the span ruler. | ----------- | ---------------------------------------------------------------------------------------- | | **RETURNS** | The original patterns, one dictionary per pattern. ~~List[Dict[str, Union[str, dict]]]~~ | -## Attributes {#attributes} +## Attributes {id="attributes"} | Name | Description | | ---------------- | -------------------------------------------------------------------------------- | diff --git a/website/docs/api/stringstore.md b/website/docs/api/stringstore.mdx similarity index 86% rename from website/docs/api/stringstore.md rename to website/docs/api/stringstore.mdx index cd414b1f0..6a3e9d664 100644 --- a/website/docs/api/stringstore.md +++ b/website/docs/api/stringstore.mdx @@ -8,7 +8,14 @@ Look up strings by 64-bit hashes. As of v2.0, spaCy uses hash values instead of integer IDs. This ensures that strings always map to the same ID, even from different `StringStores`. -## StringStore.\_\_init\_\_ {#init tag="method"} + + +Note that a `StringStore` instance is not static. It increases in size as texts +with new tokens are processed. + + + +## StringStore.\_\_init\_\_ {id="init",tag="method"} Create the `StringStore`. @@ -23,7 +30,7 @@ Create the `StringStore`. | --------- | ---------------------------------------------------------------------- | | `strings` | A sequence of strings to add to the store. ~~Optional[Iterable[str]]~~ | -## StringStore.\_\_len\_\_ {#len tag="method"} +## StringStore.\_\_len\_\_ {id="len",tag="method"} Get the number of strings in the store. @@ -38,7 +45,7 @@ Get the number of strings in the store. | ----------- | ------------------------------------------- | | **RETURNS** | The number of strings in the store. ~~int~~ | -## StringStore.\_\_getitem\_\_ {#getitem tag="method"} +## StringStore.\_\_getitem\_\_ {id="getitem",tag="method"} Retrieve a string from a given hash, or vice versa. @@ -56,7 +63,7 @@ Retrieve a string from a given hash, or vice versa. | `string_or_id` | The value to encode. ~~Union[bytes, str, int]~~ | | **RETURNS** | The value to be retrieved. ~~Union[str, int]~~ | -## StringStore.\_\_contains\_\_ {#contains tag="method"} +## StringStore.\_\_contains\_\_ {id="contains",tag="method"} Check whether a string is in the store. @@ -73,7 +80,7 @@ Check whether a string is in the store. | `string` | The string to check. ~~str~~ | | **RETURNS** | Whether the store contains the string. ~~bool~~ | -## StringStore.\_\_iter\_\_ {#iter tag="method"} +## StringStore.\_\_iter\_\_ {id="iter",tag="method"} Iterate over the strings in the store, in order. Note that a newly initialized store will always include an empty string `""` at position `0`. @@ -90,7 +97,7 @@ store will always include an empty string `""` at position `0`. | ---------- | ------------------------------ | | **YIELDS** | A string in the store. ~~str~~ | -## StringStore.add {#add tag="method" new="2"} +## StringStore.add {id="add",tag="method",version="2"} Add a string to the `StringStore`. @@ -110,7 +117,7 @@ Add a string to the `StringStore`. | `string` | The string to add. ~~str~~ | | **RETURNS** | The string's hash value. ~~int~~ | -## StringStore.to_disk {#to_disk tag="method" new="2"} +## StringStore.to_disk {id="to_disk",tag="method",version="2"} Save the current state to a directory. @@ -124,7 +131,7 @@ Save the current state to a directory. | ------ | ------------------------------------------------------------------------------------------------------------------------------------------ | | `path` | A path to a directory, which will be created if it doesn't exist. Paths may be either strings or `Path`-like objects. ~~Union[str, Path]~~ | -## StringStore.from_disk {#from_disk tag="method" new="2"} +## StringStore.from_disk {id="from_disk",tag="method",version="2"} Loads state from a directory. Modifies the object in place and returns it. @@ -140,7 +147,7 @@ Loads state from a directory. Modifies the object in place and returns it. | `path` | A path to a directory. Paths may be either strings or `Path`-like objects. ~~Union[str, Path]~~ | | **RETURNS** | The modified `StringStore` object. ~~StringStore~~ | -## StringStore.to_bytes {#to_bytes tag="method"} +## StringStore.to_bytes {id="to_bytes",tag="method"} Serialize the current state to a binary string. @@ -154,7 +161,7 @@ Serialize the current state to a binary string. | ----------- | ---------------------------------------------------------- | | **RETURNS** | The serialized form of the `StringStore` object. ~~bytes~~ | -## StringStore.from_bytes {#from_bytes tag="method"} +## StringStore.from_bytes {id="from_bytes",tag="method"} Load state from a binary string. @@ -171,9 +178,9 @@ Load state from a binary string. | `bytes_data` | The data to load from. ~~bytes~~ | | **RETURNS** | The `StringStore` object. ~~StringStore~~ | -## Utilities {#util} +## Utilities {id="util"} -### strings.hash_string {#hash_string tag="function"} +### strings.hash_string {id="hash_string",tag="function"} Get a 64-bit hash for a given string. diff --git a/website/docs/api/tagger.md b/website/docs/api/tagger.mdx similarity index 89% rename from website/docs/api/tagger.md rename to website/docs/api/tagger.mdx index 90a49b197..d9b0506fb 100644 --- a/website/docs/api/tagger.md +++ b/website/docs/api/tagger.mdx @@ -14,7 +14,7 @@ part-of-speech tag set. In the pre-trained pipelines, the tag schemas vary by language; see the [individual model pages](/models) for details. -## Assigned Attributes {#assigned-attributes} +## Assigned Attributes {id="assigned-attributes"} Predictions are assigned to `Token.tag`. @@ -23,7 +23,7 @@ Predictions are assigned to `Token.tag`. | `Token.tag` | The part of speech (hash). ~~int~~ | | `Token.tag_` | The part of speech. ~~str~~ | -## Config and implementation {#config} +## Config and implementation {id="config"} The default config is defined by the pipeline component factory and describes how the component should be configured. You can override its settings via the @@ -40,18 +40,19 @@ architectures and their arguments and hyperparameters. > nlp.add_pipe("tagger", config=config) > ``` -| Setting | Description | -| ------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -| `model` | A model instance that predicts the tag probabilities. The output vectors should match the number of tags in size, and be normalized as probabilities (all scores between 0 and 1, with the rows summing to `1`). Defaults to [Tagger](/api/architectures#Tagger). ~~Model[List[Doc], List[Floats2d]]~~ | -| `overwrite` 3.2 | Whether existing annotation is overwritten. Defaults to `False`. ~~bool~~ | -| `scorer` 3.2 | The scoring method. Defaults to [`Scorer.score_token_attr`](/api/scorer#score_token_attr) for the attribute `"tag"`. ~~Optional[Callable]~~ | -| `neg_prefix` 3.2.1 | The prefix used to specify incorrect tags while training. The tagger will learn not to predict exactly this tag. Defaults to `!`. ~~str~~ | +| Setting | Description | +| ---------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `model` | A model instance that predicts the tag probabilities. The output vectors should match the number of tags in size, and be normalized as probabilities (all scores between 0 and 1, with the rows summing to `1`). Defaults to [Tagger](/api/architectures#Tagger). ~~Model[List[Doc], List[Floats2d]]~~ | +| `overwrite` 3.2 | Whether existing annotation is overwritten. Defaults to `False`. ~~bool~~ | +| `scorer` 3.2 | The scoring method. Defaults to [`Scorer.score_token_attr`](/api/scorer#score_token_attr) for the attribute `"tag"`. ~~Optional[Callable]~~ | +| `neg_prefix` 3.2.1 | The prefix used to specify incorrect tags while training. The tagger will learn not to predict exactly this tag. Defaults to `!`. ~~str~~ | +| `label_smoothing` 3.6 | [Label smoothing](https://arxiv.org/abs/1906.02629) factor. Defaults to `0.0`. ~~float~~ | ```python %%GITHUB_SPACY/spacy/pipeline/tagger.pyx ``` -## Tagger.\_\_init\_\_ {#init tag="method"} +## Tagger.\_\_init\_\_ {id="init",tag="method"} > #### Example > @@ -81,7 +82,7 @@ shortcut for this and instantiate the component using its string name and | `overwrite` 3.2 | Whether existing annotation is overwritten. Defaults to `False`. ~~bool~~ | | `scorer` 3.2 | The scoring method. Defaults to [`Scorer.score_token_attr`](/api/scorer#score_token_attr) for the attribute `"tag"`. ~~Optional[Callable]~~ | -## Tagger.\_\_call\_\_ {#call tag="method"} +## Tagger.\_\_call\_\_ {id="call",tag="method"} Apply the pipe to one document. The document is modified in place, and returned. This usually happens under the hood when the `nlp` object is called on a text @@ -104,7 +105,7 @@ and all pipeline components are applied to the `Doc` in order. Both | `doc` | The document to process. ~~Doc~~ | | **RETURNS** | The processed document. ~~Doc~~ | -## Tagger.pipe {#pipe tag="method"} +## Tagger.pipe {id="pipe",tag="method"} Apply the pipe to a stream of documents. This usually happens under the hood when the `nlp` object is called on a text and all pipeline components are @@ -127,7 +128,7 @@ applied to the `Doc` in order. Both [`__call__`](/api/tagger#call) and | `batch_size` | The number of documents to buffer. Defaults to `128`. ~~int~~ | | **YIELDS** | The processed documents in order. ~~Doc~~ | -## Tagger.initialize {#initialize tag="method" new="3"} +## Tagger.initialize {id="initialize",tag="method",version="3"} Initialize the component for training. `get_examples` should be a function that returns an iterable of [`Example`](/api/example) objects. **At least one example @@ -170,7 +171,7 @@ This method was previously called `begin_training`. | `nlp` | The current `nlp` object. Defaults to `None`. ~~Optional[Language]~~ | | `labels` | The label information to add to the component, as provided by the [`label_data`](#label_data) property after initialization. To generate a reusable JSON file from your data, you should run the [`init labels`](/api/cli#init-labels) command. If no labels are provided, the `get_examples` callback is used to extract the labels from the data, which may be a lot slower. ~~Optional[Iterable[str]]~~ | -## Tagger.predict {#predict tag="method"} +## Tagger.predict {id="predict",tag="method"} Apply the component's model to a batch of [`Doc`](/api/doc) objects, without modifying them. @@ -187,7 +188,7 @@ modifying them. | `docs` | The documents to predict. ~~Iterable[Doc]~~ | | **RETURNS** | The model's prediction for each document. | -## Tagger.set_annotations {#set_annotations tag="method"} +## Tagger.set_annotations {id="set_annotations",tag="method"} Modify a batch of [`Doc`](/api/doc) objects, using pre-computed scores. @@ -204,7 +205,7 @@ Modify a batch of [`Doc`](/api/doc) objects, using pre-computed scores. | `docs` | The documents to modify. ~~Iterable[Doc]~~ | | `scores` | The scores to set, produced by `Tagger.predict`. | -## Tagger.update {#update tag="method"} +## Tagger.update {id="update",tag="method"} Learn from a batch of [`Example`](/api/example) objects containing the predictions and gold-standard annotations, and update the component's model. @@ -228,7 +229,7 @@ Delegates to [`predict`](/api/tagger#predict) and | `losses` | Optional record of the loss during training. Updated using the component name as the key. ~~Optional[Dict[str, float]]~~ | | **RETURNS** | The updated `losses` dictionary. ~~Dict[str, float]~~ | -## Tagger.rehearse {#rehearse tag="method,experimental" new="3"} +## Tagger.rehearse {id="rehearse",tag="method,experimental",version="3"} Perform a "rehearsal" update from a batch of data. Rehearsal updates teach the current model to make predictions similar to an initial model, to try to address @@ -251,7 +252,7 @@ the "catastrophic forgetting" problem. This feature is experimental. | `losses` | Optional record of the loss during training. Updated using the component name as the key. ~~Optional[Dict[str, float]]~~ | | **RETURNS** | The updated `losses` dictionary. ~~Dict[str, float]~~ | -## Tagger.get_loss {#get_loss tag="method"} +## Tagger.get_loss {id="get_loss",tag="method"} Find the loss and gradient of loss for the batch of documents and their predicted scores. @@ -270,7 +271,7 @@ predicted scores. | `scores` | Scores representing the model's predictions. | | **RETURNS** | The loss and the gradient, i.e. `(loss, gradient)`. ~~Tuple[float, float]~~ | -## Tagger.create_optimizer {#create_optimizer tag="method"} +## Tagger.create_optimizer {id="create_optimizer",tag="method"} Create an optimizer for the pipeline component. @@ -285,7 +286,7 @@ Create an optimizer for the pipeline component. | ----------- | ---------------------------- | | **RETURNS** | The optimizer. ~~Optimizer~~ | -## Tagger.use_params {#use_params tag="method, contextmanager"} +## Tagger.use_params {id="use_params",tag="method, contextmanager"} Modify the pipe's model, to use the given parameter values. At the end of the context, the original parameters are restored. @@ -302,7 +303,7 @@ context, the original parameters are restored. | -------- | -------------------------------------------------- | | `params` | The parameter values to use in the model. ~~dict~~ | -## Tagger.add_label {#add_label tag="method"} +## Tagger.add_label {id="add_label",tag="method"} Add a new label to the pipe. Raises an error if the output dimension is already set, or if the model has already been fully [initialized](#initialize). Note @@ -324,7 +325,7 @@ automatically. | `label` | The label to add. ~~str~~ | | **RETURNS** | `0` if the label is already present, otherwise `1`. ~~int~~ | -## Tagger.to_disk {#to_disk tag="method"} +## Tagger.to_disk {id="to_disk",tag="method"} Serialize the pipe to disk. @@ -341,7 +342,7 @@ Serialize the pipe to disk. | _keyword-only_ | | | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | -## Tagger.from_disk {#from_disk tag="method"} +## Tagger.from_disk {id="from_disk",tag="method"} Load the pipe from disk. Modifies the object in place and returns it. @@ -359,7 +360,7 @@ Load the pipe from disk. Modifies the object in place and returns it. | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | | **RETURNS** | The modified `Tagger` object. ~~Tagger~~ | -## Tagger.to_bytes {#to_bytes tag="method"} +## Tagger.to_bytes {id="to_bytes",tag="method"} > #### Example > @@ -376,7 +377,7 @@ Serialize the pipe to a bytestring. | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | | **RETURNS** | The serialized form of the `Tagger` object. ~~bytes~~ | -## Tagger.from_bytes {#from_bytes tag="method"} +## Tagger.from_bytes {id="from_bytes",tag="method"} Load the pipe from a bytestring. Modifies the object in place and returns it. @@ -395,7 +396,7 @@ Load the pipe from a bytestring. Modifies the object in place and returns it. | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | | **RETURNS** | The `Tagger` object. ~~Tagger~~ | -## Tagger.labels {#labels tag="property"} +## Tagger.labels {id="labels",tag="property"} The labels currently added to the component. @@ -410,7 +411,7 @@ The labels currently added to the component. | ----------- | ------------------------------------------------------ | | **RETURNS** | The labels added to the component. ~~Tuple[str, ...]~~ | -## Tagger.label_data {#label_data tag="property" new="3"} +## Tagger.label_data {id="label_data",tag="property",version="3"} The labels currently added to the component and their internal meta information. This is the data generated by [`init labels`](/api/cli#init-labels) and used by @@ -428,7 +429,7 @@ pre-defined label set. | ----------- | ---------------------------------------------------------- | | **RETURNS** | The label data added to the component. ~~Tuple[str, ...]~~ | -## Serialization fields {#serialization-fields} +## Serialization fields {id="serialization-fields"} During serialization, spaCy will export several data fields used to restore different aspects of the object. If needed, you can exclude them from diff --git a/website/docs/api/textcategorizer.md b/website/docs/api/textcategorizer.mdx similarity index 94% rename from website/docs/api/textcategorizer.md rename to website/docs/api/textcategorizer.mdx index f5f8706ec..a259b7b3c 100644 --- a/website/docs/api/textcategorizer.md +++ b/website/docs/api/textcategorizer.mdx @@ -2,7 +2,7 @@ title: TextCategorizer tag: class source: spacy/pipeline/textcat.py -new: 2 +version: 2 teaser: 'Pipeline component for text classification' api_base_class: /api/pipe api_string_name: textcat @@ -29,7 +29,7 @@ only. -## Assigned Attributes {#assigned-attributes} +## Assigned Attributes {id="assigned-attributes"} Predictions will be saved to `doc.cats` as a dictionary, where the key is the name of the category and the value is a score between 0 and 1 (inclusive). For @@ -49,7 +49,7 @@ supported. | ---------- | ------------------------------------- | | `Doc.cats` | Category scores. ~~Dict[str, float]~~ | -## Config and implementation {#config} +## Config and implementation {id="config"} The default config is defined by the pipeline component factory and describes how the component should be configured. You can override its settings via the @@ -93,7 +93,7 @@ architectures and their arguments and hyperparameters. %%GITHUB_SPACY/spacy/pipeline/textcat_multilabel.py ``` -## TextCategorizer.\_\_init\_\_ {#init tag="method"} +## TextCategorizer.\_\_init\_\_ {id="init",tag="method"} > #### Example > @@ -125,7 +125,7 @@ shortcut for this and instantiate the component using its string name and | `threshold` | Cutoff to consider a prediction "positive", relevant for `textcat_multilabel` when calculating accuracy scores. ~~float~~ | | `scorer` | The scoring method. Defaults to [`Scorer.score_cats`](/api/scorer#score_cats) for the attribute `"cats"`. ~~Optional[Callable]~~ | -## TextCategorizer.\_\_call\_\_ {#call tag="method"} +## TextCategorizer.\_\_call\_\_ {id="call",tag="method"} Apply the pipe to one document. The document is modified in place, and returned. This usually happens under the hood when the `nlp` object is called on a text @@ -148,7 +148,7 @@ delegate to the [`predict`](/api/textcategorizer#predict) and | `doc` | The document to process. ~~Doc~~ | | **RETURNS** | The processed document. ~~Doc~~ | -## TextCategorizer.pipe {#pipe tag="method"} +## TextCategorizer.pipe {id="pipe",tag="method"} Apply the pipe to a stream of documents. This usually happens under the hood when the `nlp` object is called on a text and all pipeline components are @@ -172,7 +172,7 @@ applied to the `Doc` in order. Both [`__call__`](/api/textcategorizer#call) and | `batch_size` | The number of documents to buffer. Defaults to `128`. ~~int~~ | | **YIELDS** | The processed documents in order. ~~Doc~~ | -## TextCategorizer.initialize {#initialize tag="method" new="3"} +## TextCategorizer.initialize {id="initialize",tag="method",version="3"} Initialize the component for training. `get_examples` should be a function that returns an iterable of [`Example`](/api/example) objects. **At least one example @@ -217,7 +217,7 @@ This method was previously called `begin_training`. | `labels` | The label information to add to the component, as provided by the [`label_data`](#label_data) property after initialization. To generate a reusable JSON file from your data, you should run the [`init labels`](/api/cli#init-labels) command. If no labels are provided, the `get_examples` callback is used to extract the labels from the data, which may be a lot slower. ~~Optional[Iterable[str]]~~ | | `positive_label` | The positive label for a binary task with exclusive classes, `None` otherwise and by default. This parameter is only used during scoring. It is not available when using the `textcat_multilabel` component. ~~Optional[str]~~ | -## TextCategorizer.predict {#predict tag="method"} +## TextCategorizer.predict {id="predict",tag="method"} Apply the component's model to a batch of [`Doc`](/api/doc) objects without modifying them. @@ -234,7 +234,7 @@ modifying them. | `docs` | The documents to predict. ~~Iterable[Doc]~~ | | **RETURNS** | The model's prediction for each document. | -## TextCategorizer.set_annotations {#set_annotations tag="method"} +## TextCategorizer.set_annotations {id="set_annotations",tag="method"} Modify a batch of [`Doc`](/api/doc) objects using pre-computed scores. @@ -251,7 +251,7 @@ Modify a batch of [`Doc`](/api/doc) objects using pre-computed scores. | `docs` | The documents to modify. ~~Iterable[Doc]~~ | | `scores` | The scores to set, produced by `TextCategorizer.predict`. | -## TextCategorizer.update {#update tag="method"} +## TextCategorizer.update {id="update",tag="method"} Learn from a batch of [`Example`](/api/example) objects containing the predictions and gold-standard annotations, and update the component's model. @@ -275,7 +275,7 @@ Delegates to [`predict`](/api/textcategorizer#predict) and | `losses` | Optional record of the loss during training. Updated using the component name as the key. ~~Optional[Dict[str, float]]~~ | | **RETURNS** | The updated `losses` dictionary. ~~Dict[str, float]~~ | -## TextCategorizer.rehearse {#rehearse tag="method,experimental" new="3"} +## TextCategorizer.rehearse {id="rehearse",tag="method,experimental",version="3"} Perform a "rehearsal" update from a batch of data. Rehearsal updates teach the current model to make predictions similar to an initial model to try to address @@ -298,7 +298,7 @@ the "catastrophic forgetting" problem. This feature is experimental. | `losses` | Optional record of the loss during training. Updated using the component name as the key. ~~Optional[Dict[str, float]]~~ | | **RETURNS** | The updated `losses` dictionary. ~~Dict[str, float]~~ | -## TextCategorizer.get_loss {#get_loss tag="method"} +## TextCategorizer.get_loss {id="get_loss",tag="method"} Find the loss and gradient of loss for the batch of documents and their predicted scores. @@ -317,7 +317,7 @@ predicted scores. | `scores` | Scores representing the model's predictions. | | **RETURNS** | The loss and the gradient, i.e. `(loss, gradient)`. ~~Tuple[float, float]~~ | -## TextCategorizer.score {#score tag="method" new="3"} +## TextCategorizer.score {id="score",tag="method",version="3"} Score a batch of examples. @@ -333,7 +333,7 @@ Score a batch of examples. | _keyword-only_ | | | **RETURNS** | The scores, produced by [`Scorer.score_cats`](/api/scorer#score_cats). ~~Dict[str, Union[float, Dict[str, float]]]~~ | -## TextCategorizer.create_optimizer {#create_optimizer tag="method"} +## TextCategorizer.create_optimizer {id="create_optimizer",tag="method"} Create an optimizer for the pipeline component. @@ -348,7 +348,7 @@ Create an optimizer for the pipeline component. | ----------- | ---------------------------- | | **RETURNS** | The optimizer. ~~Optimizer~~ | -## TextCategorizer.use_params {#use_params tag="method, contextmanager"} +## TextCategorizer.use_params {id="use_params",tag="method, contextmanager"} Modify the pipe's model to use the given parameter values. @@ -364,7 +364,7 @@ Modify the pipe's model to use the given parameter values. | -------- | -------------------------------------------------- | | `params` | The parameter values to use in the model. ~~dict~~ | -## TextCategorizer.add_label {#add_label tag="method"} +## TextCategorizer.add_label {id="add_label",tag="method"} Add a new label to the pipe. Raises an error if the output dimension is already set, or if the model has already been fully [initialized](#initialize). Note @@ -386,7 +386,7 @@ automatically. | `label` | The label to add. ~~str~~ | | **RETURNS** | `0` if the label is already present, otherwise `1`. ~~int~~ | -## TextCategorizer.to_disk {#to_disk tag="method"} +## TextCategorizer.to_disk {id="to_disk",tag="method"} Serialize the pipe to disk. @@ -403,7 +403,7 @@ Serialize the pipe to disk. | _keyword-only_ | | | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | -## TextCategorizer.from_disk {#from_disk tag="method"} +## TextCategorizer.from_disk {id="from_disk",tag="method"} Load the pipe from disk. Modifies the object in place and returns it. @@ -421,7 +421,7 @@ Load the pipe from disk. Modifies the object in place and returns it. | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | | **RETURNS** | The modified `TextCategorizer` object. ~~TextCategorizer~~ | -## TextCategorizer.to_bytes {#to_bytes tag="method"} +## TextCategorizer.to_bytes {id="to_bytes",tag="method"} > #### Example > @@ -438,7 +438,7 @@ Serialize the pipe to a bytestring. | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | | **RETURNS** | The serialized form of the `TextCategorizer` object. ~~bytes~~ | -## TextCategorizer.from_bytes {#from_bytes tag="method"} +## TextCategorizer.from_bytes {id="from_bytes",tag="method"} Load the pipe from a bytestring. Modifies the object in place and returns it. @@ -457,7 +457,7 @@ Load the pipe from a bytestring. Modifies the object in place and returns it. | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | | **RETURNS** | The `TextCategorizer` object. ~~TextCategorizer~~ | -## TextCategorizer.labels {#labels tag="property"} +## TextCategorizer.labels {id="labels",tag="property"} The labels currently added to the component. @@ -472,7 +472,7 @@ The labels currently added to the component. | ----------- | ------------------------------------------------------ | | **RETURNS** | The labels added to the component. ~~Tuple[str, ...]~~ | -## TextCategorizer.label_data {#label_data tag="property" new="3"} +## TextCategorizer.label_data {id="label_data",tag="property",version="3"} The labels currently added to the component and their internal meta information. This is the data generated by [`init labels`](/api/cli#init-labels) and used by @@ -490,7 +490,7 @@ the model with a pre-defined label set. | ----------- | ---------------------------------------------------------- | | **RETURNS** | The label data added to the component. ~~Tuple[str, ...]~~ | -## Serialization fields {#serialization-fields} +## Serialization fields {id="serialization-fields"} During serialization, spaCy will export several data fields used to restore different aspects of the object. If needed, you can exclude them from diff --git a/website/docs/api/tok2vec.md b/website/docs/api/tok2vec.mdx similarity index 94% rename from website/docs/api/tok2vec.md rename to website/docs/api/tok2vec.mdx index 2dcb1a013..a1bb1265e 100644 --- a/website/docs/api/tok2vec.md +++ b/website/docs/api/tok2vec.mdx @@ -1,7 +1,7 @@ --- title: Tok2Vec source: spacy/pipeline/tok2vec.py -new: 3 +version: 3 teaser: null api_base_class: /api/pipe api_string_name: tok2vec @@ -23,7 +23,7 @@ components can backpropagate to the shared weights. This implementation is used because it allows us to avoid relying on object identity within the models to achieve the parameter sharing. -## Config and implementation {#config} +## Config and implementation {id="config"} The default config is defined by the pipeline component factory and describes how the component should be configured. You can override its settings via the @@ -48,7 +48,7 @@ architectures and their arguments and hyperparameters. %%GITHUB_SPACY/spacy/pipeline/tok2vec.py ``` -## Tok2Vec.\_\_init\_\_ {#init tag="method"} +## Tok2Vec.\_\_init\_\_ {id="init",tag="method"} > #### Example > @@ -75,7 +75,7 @@ shortcut for this and instantiate the component using its string name and | `model` | The Thinc [`Model`](https://thinc.ai/docs/api-model) powering the pipeline component. ~~Model[List[Doc], List[Floats2d]~~ | | `name` | String name of the component instance. Used to add entries to the `losses` during training. ~~str~~ | -## Tok2Vec.\_\_call\_\_ {#call tag="method"} +## Tok2Vec.\_\_call\_\_ {id="call",tag="method"} Apply the pipe to one document and add context-sensitive embeddings to the `Doc.tensor` attribute, allowing them to be used as features by downstream @@ -100,7 +100,7 @@ pipeline components are applied to the `Doc` in order. Both | `doc` | The document to process. ~~Doc~~ | | **RETURNS** | The processed document. ~~Doc~~ | -## Tok2Vec.pipe {#pipe tag="method"} +## Tok2Vec.pipe {id="pipe",tag="method"} Apply the pipe to a stream of documents. This usually happens under the hood when the `nlp` object is called on a text and all pipeline components are @@ -123,7 +123,7 @@ and [`set_annotations`](/api/tok2vec#set_annotations) methods. | `batch_size` | The number of documents to buffer. Defaults to `128`. ~~int~~ | | **YIELDS** | The processed documents in order. ~~Doc~~ | -## Tok2Vec.initialize {#initialize tag="method"} +## Tok2Vec.initialize {id="initialize",tag="method"} Initialize the component for training and return an [`Optimizer`](https://thinc.ai/docs/api-optimizers). `get_examples` should be a @@ -148,7 +148,7 @@ by [`Language.initialize`](/api/language#initialize). | _keyword-only_ | | | `nlp` | The current `nlp` object. Defaults to `None`. ~~Optional[Language]~~ | -## Tok2Vec.predict {#predict tag="method"} +## Tok2Vec.predict {id="predict",tag="method"} Apply the component's model to a batch of [`Doc`](/api/doc) objects without modifying them. @@ -165,7 +165,7 @@ modifying them. | `docs` | The documents to predict. ~~Iterable[Doc]~~ | | **RETURNS** | The model's prediction for each document. | -## Tok2Vec.set_annotations {#set_annotations tag="method"} +## Tok2Vec.set_annotations {id="set_annotations",tag="method"} Modify a batch of [`Doc`](/api/doc) objects, using pre-computed scores. @@ -182,7 +182,7 @@ Modify a batch of [`Doc`](/api/doc) objects, using pre-computed scores. | `docs` | The documents to modify. ~~Iterable[Doc]~~ | | `scores` | The scores to set, produced by `Tok2Vec.predict`. | -## Tok2Vec.update {#update tag="method"} +## Tok2Vec.update {id="update",tag="method"} Learn from a batch of [`Example`](/api/example) objects containing the predictions and gold-standard annotations, and update the component's model. @@ -205,7 +205,7 @@ Delegates to [`predict`](/api/tok2vec#predict). | `losses` | Optional record of the loss during training. Updated using the component name as the key. ~~Optional[Dict[str, float]]~~ | | **RETURNS** | The updated `losses` dictionary. ~~Dict[str, float]~~ | -## Tok2Vec.create_optimizer {#create_optimizer tag="method"} +## Tok2Vec.create_optimizer {id="create_optimizer",tag="method"} Create an optimizer for the pipeline component. @@ -220,7 +220,7 @@ Create an optimizer for the pipeline component. | ----------- | ---------------------------- | | **RETURNS** | The optimizer. ~~Optimizer~~ | -## Tok2Vec.use_params {#use_params tag="method, contextmanager"} +## Tok2Vec.use_params {id="use_params",tag="method, contextmanager"} Modify the pipe's model to use the given parameter values. At the end of the context, the original parameters are restored. @@ -237,7 +237,7 @@ context, the original parameters are restored. | -------- | -------------------------------------------------- | | `params` | The parameter values to use in the model. ~~dict~~ | -## Tok2Vec.to_disk {#to_disk tag="method"} +## Tok2Vec.to_disk {id="to_disk",tag="method"} Serialize the pipe to disk. @@ -254,7 +254,7 @@ Serialize the pipe to disk. | _keyword-only_ | | | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | -## Tok2Vec.from_disk {#from_disk tag="method"} +## Tok2Vec.from_disk {id="from_disk",tag="method"} Load the pipe from disk. Modifies the object in place and returns it. @@ -272,7 +272,7 @@ Load the pipe from disk. Modifies the object in place and returns it. | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | | **RETURNS** | The modified `Tok2Vec` object. ~~Tok2Vec~~ | -## Tok2Vec.to_bytes {#to_bytes tag="method"} +## Tok2Vec.to_bytes {id="to_bytes",tag="method"} > #### Example > @@ -289,7 +289,7 @@ Serialize the pipe to a bytestring. | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | | **RETURNS** | The serialized form of the `Tok2Vec` object. ~~bytes~~ | -## Tok2Vec.from_bytes {#from_bytes tag="method"} +## Tok2Vec.from_bytes {id="from_bytes",tag="method"} Load the pipe from a bytestring. Modifies the object in place and returns it. @@ -308,7 +308,7 @@ Load the pipe from a bytestring. Modifies the object in place and returns it. | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | | **RETURNS** | The `Tok2Vec` object. ~~Tok2Vec~~ | -## Serialization fields {#serialization-fields} +## Serialization fields {id="serialization-fields"} During serialization, spaCy will export several data fields used to restore different aspects of the object. If needed, you can exclude them from diff --git a/website/docs/api/token.md b/website/docs/api/token.mdx similarity index 96% rename from website/docs/api/token.md rename to website/docs/api/token.mdx index 89bd77447..63ee1080b 100644 --- a/website/docs/api/token.md +++ b/website/docs/api/token.mdx @@ -5,7 +5,7 @@ tag: class source: spacy/tokens/token.pyx --- -## Token.\_\_init\_\_ {#init tag="method"} +## Token.\_\_init\_\_ {id="init",tag="method"} Construct a `Token` object. @@ -23,7 +23,7 @@ Construct a `Token` object. | `doc` | The parent document. ~~Doc~~ | | `offset` | The index of the token within the document. ~~int~~ | -## Token.\_\_len\_\_ {#len tag="method"} +## Token.\_\_len\_\_ {id="len",tag="method"} The number of unicode characters in the token, i.e. `token.text`. @@ -39,7 +39,7 @@ The number of unicode characters in the token, i.e. `token.text`. | ----------- | ------------------------------------------------------ | | **RETURNS** | The number of unicode characters in the token. ~~int~~ | -## Token.set_extension {#set_extension tag="classmethod" new="2"} +## Token.set_extension {id="set_extension",tag="classmethod",version="2"} Define a custom attribute on the `Token` which becomes available via `Token._`. For details, see the documentation on @@ -64,7 +64,7 @@ For details, see the documentation on | `setter` | Setter function that takes the `Token` and a value, and modifies the object. Is called when the user writes to the `Token._` attribute. ~~Optional[Callable[[Token, Any], None]]~~ | | `force` | Force overwriting existing attribute. ~~bool~~ | -## Token.get_extension {#get_extension tag="classmethod" new="2"} +## Token.get_extension {id="get_extension",tag="classmethod",version="2"} Look up a previously registered extension by name. Returns a 4-tuple `(default, method, getter, setter)` if the extension is registered. Raises a @@ -84,7 +84,7 @@ Look up a previously registered extension by name. Returns a 4-tuple | `name` | Name of the extension. ~~str~~ | | **RETURNS** | A `(default, method, getter, setter)` tuple of the extension. ~~Tuple[Optional[Any], Optional[Callable], Optional[Callable], Optional[Callable]]~~ | -## Token.has_extension {#has_extension tag="classmethod" new="2"} +## Token.has_extension {id="has_extension",tag="classmethod",version="2"} Check whether an extension has been registered on the `Token` class. @@ -101,7 +101,7 @@ Check whether an extension has been registered on the `Token` class. | `name` | Name of the extension to check. ~~str~~ | | **RETURNS** | Whether the extension has been registered. ~~bool~~ | -## Token.remove_extension {#remove_extension tag="classmethod" new=""2.0.11""} +## Token.remove_extension {id="remove_extension",tag="classmethod",version="2.0.11"} Remove a previously registered extension. @@ -119,7 +119,7 @@ Remove a previously registered extension. | `name` | Name of the extension. ~~str~~ | | **RETURNS** | A `(default, method, getter, setter)` tuple of the removed extension. ~~Tuple[Optional[Any], Optional[Callable], Optional[Callable], Optional[Callable]]~~ | -## Token.check_flag {#check_flag tag="method"} +## Token.check_flag {id="check_flag",tag="method"} Check the value of a boolean flag. @@ -137,7 +137,7 @@ Check the value of a boolean flag. | `flag_id` | The attribute ID of the flag to check. ~~int~~ | | **RETURNS** | Whether the flag is set. ~~bool~~ | -## Token.similarity {#similarity tag="method" model="vectors"} +## Token.similarity {id="similarity",tag="method",model="vectors"} Compute a semantic similarity estimate. Defaults to cosine over vectors. @@ -155,7 +155,7 @@ Compute a semantic similarity estimate. Defaults to cosine over vectors. | other | The object to compare with. By default, accepts `Doc`, `Span`, `Token` and `Lexeme` objects. ~~Union[Doc, Span, Token, Lexeme]~~ | | **RETURNS** | A scalar similarity score. Higher is more similar. ~~float~~ | -## Token.nbor {#nbor tag="method"} +## Token.nbor {id="nbor",tag="method"} Get a neighboring token. @@ -172,7 +172,7 @@ Get a neighboring token. | `i` | The relative position of the token to get. Defaults to `1`. ~~int~~ | | **RETURNS** | The token at position `self.doc[self.i+i]`. ~~Token~~ | -## Token.set_morph {#set_morph tag="method"} +## Token.set_morph {id="set_morph",tag="method"} Set the morphological analysis from a UD FEATS string, hash value of a UD FEATS string, features dict or `MorphAnalysis`. The value `None` can be used to reset @@ -191,7 +191,7 @@ the morph to an unset state. | -------- | --------------------------------------------------------------------------------- | | features | The morphological features to set. ~~Union[int, dict, str, MorphAnalysis, None]~~ | -## Token.has_morph {#has_morph tag="method"} +## Token.has_morph {id="has_morph",tag="method"} Check whether the token has annotated morph information. Return `False` when the morph annotation is unset/missing. @@ -200,7 +200,7 @@ morph annotation is unset/missing. | ----------- | --------------------------------------------- | | **RETURNS** | Whether the morph annotation is set. ~~bool~~ | -## Token.is_ancestor {#is_ancestor tag="method" model="parser"} +## Token.is_ancestor {id="is_ancestor",tag="method",model="parser"} Check whether this token is a parent, grandparent, etc. of another in the dependency tree. @@ -219,7 +219,7 @@ dependency tree. | descendant | Another token. ~~Token~~ | | **RETURNS** | Whether this token is the ancestor of the descendant. ~~bool~~ | -## Token.ancestors {#ancestors tag="property" model="parser"} +## Token.ancestors {id="ancestors",tag="property",model="parser"} A sequence of the token's syntactic ancestors (parents, grandparents, etc). @@ -237,7 +237,7 @@ A sequence of the token's syntactic ancestors (parents, grandparents, etc). | ---------- | ------------------------------------------------------------------------------- | | **YIELDS** | A sequence of ancestor tokens such that `ancestor.is_ancestor(self)`. ~~Token~~ | -## Token.conjuncts {#conjuncts tag="property" model="parser"} +## Token.conjuncts {id="conjuncts",tag="property",model="parser"} A tuple of coordinated tokens, not including the token itself. @@ -253,7 +253,7 @@ A tuple of coordinated tokens, not including the token itself. | ----------- | --------------------------------------------- | | **RETURNS** | The coordinated tokens. ~~Tuple[Token, ...]~~ | -## Token.children {#children tag="property" model="parser"} +## Token.children {id="children",tag="property",model="parser"} A sequence of the token's immediate syntactic children. @@ -269,7 +269,7 @@ A sequence of the token's immediate syntactic children. | ---------- | ------------------------------------------------------- | | **YIELDS** | A child token such that `child.head == self`. ~~Token~~ | -## Token.lefts {#lefts tag="property" model="parser"} +## Token.lefts {id="lefts",tag="property",model="parser"} The leftward immediate children of the word in the syntactic dependency parse. @@ -285,7 +285,7 @@ The leftward immediate children of the word in the syntactic dependency parse. | ---------- | ------------------------------------ | | **YIELDS** | A left-child of the token. ~~Token~~ | -## Token.rights {#rights tag="property" model="parser"} +## Token.rights {id="rights",tag="property",model="parser"} The rightward immediate children of the word in the syntactic dependency parse. @@ -301,7 +301,7 @@ The rightward immediate children of the word in the syntactic dependency parse. | ---------- | ------------------------------------- | | **YIELDS** | A right-child of the token. ~~Token~~ | -## Token.n_lefts {#n_lefts tag="property" model="parser"} +## Token.n_lefts {id="n_lefts",tag="property",model="parser"} The number of leftward immediate children of the word in the syntactic dependency parse. @@ -317,7 +317,7 @@ dependency parse. | ----------- | ---------------------------------------- | | **RETURNS** | The number of left-child tokens. ~~int~~ | -## Token.n_rights {#n_rights tag="property" model="parser"} +## Token.n_rights {id="n_rights",tag="property",model="parser"} The number of rightward immediate children of the word in the syntactic dependency parse. @@ -333,7 +333,7 @@ dependency parse. | ----------- | ----------------------------------------- | | **RETURNS** | The number of right-child tokens. ~~int~~ | -## Token.subtree {#subtree tag="property" model="parser"} +## Token.subtree {id="subtree",tag="property",model="parser"} A sequence containing the token and all the token's syntactic descendants. @@ -349,7 +349,7 @@ A sequence containing the token and all the token's syntactic descendants. | ---------- | ------------------------------------------------------------------------------------ | | **YIELDS** | A descendant token such that `self.is_ancestor(token)` or `token == self`. ~~Token~~ | -## Token.has_vector {#has_vector tag="property" model="vectors"} +## Token.has_vector {id="has_vector",tag="property",model="vectors"} A boolean value indicating whether a word vector is associated with the token. @@ -365,7 +365,7 @@ A boolean value indicating whether a word vector is associated with the token. | ----------- | ------------------------------------------------------ | | **RETURNS** | Whether the token has a vector data attached. ~~bool~~ | -## Token.vector {#vector tag="property" model="vectors"} +## Token.vector {id="vector",tag="property",model="vectors"} A real-valued meaning representation. @@ -382,7 +382,7 @@ A real-valued meaning representation. | ----------- | ----------------------------------------------------------------------------------------------- | | **RETURNS** | A 1-dimensional array representing the token's vector. ~~numpy.ndarray[ndim=1, dtype=float32]~~ | -## Token.vector_norm {#vector_norm tag="property" model="vectors"} +## Token.vector_norm {id="vector_norm",tag="property",model="vectors"} The L2 norm of the token's vector representation. @@ -401,7 +401,7 @@ The L2 norm of the token's vector representation. | ----------- | --------------------------------------------------- | | **RETURNS** | The L2 norm of the vector representation. ~~float~~ | -## Attributes {#attributes} +## Attributes {id="attributes"} | Name | Description | | ---------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | diff --git a/website/docs/api/tokenizer.md b/website/docs/api/tokenizer.mdx similarity index 95% rename from website/docs/api/tokenizer.md rename to website/docs/api/tokenizer.mdx index 6eb7e8024..0a579ab4c 100644 --- a/website/docs/api/tokenizer.md +++ b/website/docs/api/tokenizer.mdx @@ -20,7 +20,7 @@ The tokenizer is typically created automatically when a like punctuation and special case rules from the [`Language.Defaults`](/api/language#defaults) provided by the language subclass. -## Tokenizer.\_\_init\_\_ {#init tag="method"} +## Tokenizer.\_\_init\_\_ {id="init",tag="method"} Create a `Tokenizer` to create `Doc` objects given unicode text. For examples of how to construct a custom tokenizer with different tokenization rules, see the @@ -55,7 +55,7 @@ how to construct a custom tokenizer with different tokenization rules, see the | `url_match` | A function matching the signature of `re.compile(string).match` to find token matches after considering prefixes and suffixes. ~~Optional[Callable[[str], Optional[Match]]]~~ | | `faster_heuristics` 3.3.0 | Whether to restrict the final `Matcher`-based pass for rules to those containing affixes or space. Defaults to `True`. ~~bool~~ | -## Tokenizer.\_\_call\_\_ {#call tag="method"} +## Tokenizer.\_\_call\_\_ {id="call",tag="method"} Tokenize a string. @@ -71,7 +71,7 @@ Tokenize a string. | `string` | The string to tokenize. ~~str~~ | | **RETURNS** | A container for linguistic annotations. ~~Doc~~ | -## Tokenizer.pipe {#pipe tag="method"} +## Tokenizer.pipe {id="pipe",tag="method"} Tokenize a stream of texts. @@ -89,7 +89,7 @@ Tokenize a stream of texts. | `batch_size` | The number of texts to accumulate in an internal buffer. Defaults to `1000`. ~~int~~ | | **YIELDS** | The tokenized `Doc` objects, in order. ~~Doc~~ | -## Tokenizer.find_infix {#find_infix tag="method"} +## Tokenizer.find_infix {id="find_infix",tag="method"} Find internal split points of the string. @@ -98,7 +98,7 @@ Find internal split points of the string. | `string` | The string to split. ~~str~~ | | **RETURNS** | A list of `re.MatchObject` objects that have `.start()` and `.end()` methods, denoting the placement of internal segment separators, e.g. hyphens. ~~List[Match]~~ | -## Tokenizer.find_prefix {#find_prefix tag="method"} +## Tokenizer.find_prefix {id="find_prefix",tag="method"} Find the length of a prefix that should be segmented from the string, or `None` if no prefix rules match. @@ -108,7 +108,7 @@ if no prefix rules match. | `string` | The string to segment. ~~str~~ | | **RETURNS** | The length of the prefix if present, otherwise `None`. ~~Optional[int]~~ | -## Tokenizer.find_suffix {#find_suffix tag="method"} +## Tokenizer.find_suffix {id="find_suffix",tag="method"} Find the length of a suffix that should be segmented from the string, or `None` if no suffix rules match. @@ -118,7 +118,7 @@ if no suffix rules match. | `string` | The string to segment. ~~str~~ | | **RETURNS** | The length of the suffix if present, otherwise `None`. ~~Optional[int]~~ | -## Tokenizer.add_special_case {#add_special_case tag="method"} +## Tokenizer.add_special_case {id="add_special_case",tag="method"} Add a special-case tokenization rule. This mechanism is also used to add custom tokenizer exceptions to the language data. See the usage guide on the @@ -139,7 +139,7 @@ details and examples. | `string` | The string to specially tokenize. ~~str~~ | | `token_attrs` | A sequence of dicts, where each dict describes a token and its attributes. The `ORTH` fields of the attributes must exactly match the string when they are concatenated. ~~Iterable[Dict[int, str]]~~ | -## Tokenizer.explain {#explain tag="method"} +## Tokenizer.explain {id="explain",tag="method"} Tokenize a string with a slow debugging tokenizer that provides information about which tokenizer rule or pattern was matched for each token. The tokens @@ -158,7 +158,7 @@ produced are identical to `Tokenizer.__call__` except for whitespace tokens. | `string` | The string to tokenize with the debugging tokenizer. ~~str~~ | | **RETURNS** | A list of `(pattern_string, token_string)` tuples. ~~List[Tuple[str, str]]~~ | -## Tokenizer.to_disk {#to_disk tag="method"} +## Tokenizer.to_disk {id="to_disk",tag="method"} Serialize the tokenizer to disk. @@ -175,7 +175,7 @@ Serialize the tokenizer to disk. | _keyword-only_ | | | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | -## Tokenizer.from_disk {#from_disk tag="method"} +## Tokenizer.from_disk {id="from_disk",tag="method"} Load the tokenizer from disk. Modifies the object in place and returns it. @@ -193,7 +193,7 @@ Load the tokenizer from disk. Modifies the object in place and returns it. | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | | **RETURNS** | The modified `Tokenizer` object. ~~Tokenizer~~ | -## Tokenizer.to_bytes {#to_bytes tag="method"} +## Tokenizer.to_bytes {id="to_bytes",tag="method"} > #### Example > @@ -210,7 +210,7 @@ Serialize the tokenizer to a bytestring. | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | | **RETURNS** | The serialized form of the `Tokenizer` object. ~~bytes~~ | -## Tokenizer.from_bytes {#from_bytes tag="method"} +## Tokenizer.from_bytes {id="from_bytes",tag="method"} Load the tokenizer from a bytestring. Modifies the object in place and returns it. @@ -230,7 +230,7 @@ it. | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | | **RETURNS** | The `Tokenizer` object. ~~Tokenizer~~ | -## Attributes {#attributes} +## Attributes {id="attributes"} | Name | Description | | ---------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | @@ -241,7 +241,7 @@ it. | `token_match` | A function matching the signature of `re.compile(string).match` to find token matches. Returns an `re.MatchObject` or `None`. ~~Optional[Callable[[str], Optional[Match]]]~~ | | `rules` | A dictionary of tokenizer exceptions and special cases. ~~Optional[Dict[str, List[Dict[int, str]]]]~~ | -## Serialization fields {#serialization-fields} +## Serialization fields {id="serialization-fields"} During serialization, spaCy will export several data fields used to restore different aspects of the object. If needed, you can exclude them from diff --git a/website/docs/api/top-level.md b/website/docs/api/top-level.mdx similarity index 86% rename from website/docs/api/top-level.md rename to website/docs/api/top-level.mdx index 26a5d42f4..6de1acdf0 100644 --- a/website/docs/api/top-level.md +++ b/website/docs/api/top-level.mdx @@ -13,9 +13,9 @@ menu: - ['Utility Functions', 'util'] --- -## spaCy {#spacy hidden="true"} +## spaCy {id="spacy",hidden="true"} -### spacy.load {#spacy.load tag="function"} +### spacy.load {id="spacy.load",tag="function"} Load a pipeline using the name of an installed [package](/usage/saving-loading#models), a string path or a `Path`-like object. @@ -25,7 +25,10 @@ and call the package's own `load()` method. If a pipeline is loaded from a path, spaCy will assume it's a data directory, load its [`config.cfg`](/api/data-formats#config) and use the language and pipeline information to construct the `Language` class. The data will be loaded in via -[`Language.from_disk`](/api/language#from_disk). +[`Language.from_disk`](/api/language#from_disk). Loading a pipeline from a +package will also import any custom code, if present, whereas loading from a +directory does not. For these cases, you need to manually import your custom +code. @@ -61,8 +64,7 @@ Essentially, `spacy.load()` is a convenience wrapper that reads the pipeline's information to construct a `Language` object, loads in the model data and weights, and returns it. -```python -### Abstract example +```python {title="Abstract example"} cls = spacy.util.get_lang_class(lang) # 1. Get Language class, e.g. English nlp = cls() # 2. Initialize it for name in pipeline: @@ -70,7 +72,7 @@ for name in pipeline: nlp.from_disk(data_path) # 4. Load in the binary data ``` -### spacy.blank {#spacy.blank tag="function" new="2"} +### spacy.blank {id="spacy.blank",tag="function",version="2"} Create a blank pipeline of a given language class. This function is the twin of `spacy.load()`. @@ -91,7 +93,7 @@ Create a blank pipeline of a given language class. This function is the twin of | `meta` | Optional meta overrides for [`nlp.meta`](/api/language#meta). ~~Dict[str, Any]~~ | | **RETURNS** | An empty `Language` object of the appropriate subclass. ~~Language~~ | -### spacy.info {#spacy.info tag="function"} +### spacy.info {id="spacy.info",tag="function"} The same as the [`info` command](/api/cli#info). Pretty-print information about your installation, installed pipelines and local setup from within spaCy. @@ -111,7 +113,7 @@ your installation, installed pipelines and local setup from within spaCy. | `markdown` | Print information as Markdown. ~~bool~~ | | `silent` | Don't print anything, just return. ~~bool~~ | -### spacy.explain {#spacy.explain tag="function"} +### spacy.explain {id="spacy.explain",tag="function"} Get a description for a given POS tag, dependency label or entity type. For a list of available terms, see [`glossary.py`](%%GITHUB_SPACY/spacy/glossary.py). @@ -134,7 +136,7 @@ list of available terms, see [`glossary.py`](%%GITHUB_SPACY/spacy/glossary.py). | `term` | Term to explain. ~~str~~ | | **RETURNS** | The explanation, or `None` if not found in the glossary. ~~Optional[str]~~ | -### spacy.prefer_gpu {#spacy.prefer_gpu tag="function" new="2.0.14"} +### spacy.prefer_gpu {id="spacy.prefer_gpu",tag="function",version="2.0.14"} Allocate data and perform operations on [GPU](/usage/#gpu), if available. If data has already been allocated on CPU, it will not be moved. Ideally, this @@ -162,7 +164,7 @@ ensure that the model is loaded on the correct device. See | `gpu_id` | Device index to select. Defaults to `0`. ~~int~~ | | **RETURNS** | Whether the GPU was activated. ~~bool~~ | -### spacy.require_gpu {#spacy.require_gpu tag="function" new="2.0.14"} +### spacy.require_gpu {id="spacy.require_gpu",tag="function",version="2.0.14"} Allocate data and perform operations on [GPU](/usage/#gpu). Will raise an error if no GPU is available. If data has already been allocated on CPU, it will not @@ -190,7 +192,7 @@ ensure that the model is loaded on the correct device. See | `gpu_id` | Device index to select. Defaults to `0`. ~~int~~ | | **RETURNS** | `True` ~~bool~~ | -### spacy.require_cpu {#spacy.require_cpu tag="function" new="3.0.0"} +### spacy.require_cpu {id="spacy.require_cpu",tag="function",version="3.0.0"} Allocate data and perform operations on CPU. If data has already been allocated on GPU, it will not be moved. Ideally, this function should be called right @@ -216,12 +218,12 @@ ensure that the model is loaded on the correct device. See | ----------- | --------------- | | **RETURNS** | `True` ~~bool~~ | -## displaCy {#displacy source="spacy/displacy"} +## displaCy {id="displacy",source="spacy/displacy"} As of v2.0, spaCy comes with a built-in visualization suite. For more info and examples, see the usage guide on [visualizing spaCy](/usage/visualizers). -### displacy.serve {#displacy.serve tag="method" new="2"} +### displacy.serve {id="displacy.serve",tag="method",version="2"} Serve a dependency parse tree or named entity visualization to view it in your browser. Will run a simple web server. @@ -237,18 +239,19 @@ browser. Will run a simple web server. > displacy.serve([doc1, doc2], style="dep") > ``` -| Name | Description | -| --------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `docs` | Document(s) or span(s) to visualize. ~~Union[Iterable[Union[Doc, Span]], Doc, Span]~~ | -| `style` | Visualization style, `"dep"`, `"ent"` or `"span"` 3.3. Defaults to `"dep"`. ~~str~~ | -| `page` | Render markup as full HTML page. Defaults to `True`. ~~bool~~ | -| `minify` | Minify HTML markup. Defaults to `False`. ~~bool~~ | -| `options` | [Visualizer-specific options](#displacy_options), e.g. colors. ~~Dict[str, Any]~~ | -| `manual` | Don't parse `Doc` and instead expect a dict or list of dicts. [See here](/usage/visualizers#manual-usage) for formats and examples. Defaults to `False`. ~~bool~~ | -| `port` | Port to serve visualization. Defaults to `5000`. ~~int~~ | -| `host` | Host to serve visualization. Defaults to `"0.0.0.0"`. ~~str~~ | +| Name | Description | +| ----------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `docs` | Document(s) or span(s) to visualize. ~~Union[Iterable[Union[Doc, Span]], Doc, Span]~~ | +| `style` 3.3 | Visualization style, `"dep"`, `"ent"` or `"span"`. Defaults to `"dep"`. ~~str~~ | +| `page` | Render markup as full HTML page. Defaults to `True`. ~~bool~~ | +| `minify` | Minify HTML markup. Defaults to `False`. ~~bool~~ | +| `options` | [Visualizer-specific options](#displacy_options), e.g. colors. ~~Dict[str, Any]~~ | +| `manual` | Don't parse `Doc` and instead expect a dict or list of dicts. [See here](/usage/visualizers#manual-usage) for formats and examples. Defaults to `False`. ~~bool~~ | +| `port` | Port to serve visualization. Defaults to `5000`. ~~int~~ | +| `host` | Host to serve visualization. Defaults to `"0.0.0.0"`. ~~str~~ | +| `auto_select_port` 3.5 | If `True`, automatically switch to a different port if the specified port is already in use. Defaults to `False`. ~~bool~~ | -### displacy.render {#displacy.render tag="method" new="2"} +### displacy.render {id="displacy.render",tag="method",version="2"} Render a dependency parse tree or named entity visualization. @@ -266,14 +269,14 @@ Render a dependency parse tree or named entity visualization. | ----------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | `docs` | Document(s) or span(s) to visualize. ~~Union[Iterable[Union[Doc, Span, dict]], Doc, Span, dict]~~ | | `style` | Visualization style, `"dep"`, `"ent"` or `"span"` 3.3. Defaults to `"dep"`. ~~str~~ | -| `page` | Render markup as full HTML page. Defaults to `True`. ~~bool~~ | +| `page` | Render markup as full HTML page. Defaults to `False`. ~~bool~~ | | `minify` | Minify HTML markup. Defaults to `False`. ~~bool~~ | | `options` | [Visualizer-specific options](#displacy_options), e.g. colors. ~~Dict[str, Any]~~ | | `manual` | Don't parse `Doc` and instead expect a dict or list of dicts. [See here](/usage/visualizers#manual-usage) for formats and examples. Defaults to `False`. ~~bool~~ | | `jupyter` | Explicitly enable or disable "[Jupyter](http://jupyter.org/) mode" to return markup ready to be rendered in a notebook. Detected automatically if `None` (default). ~~Optional[bool]~~ | | **RETURNS** | The rendered HTML markup. ~~str~~ | -### displacy.parse_deps {#displacy.parse_deps tag="method" new="2"} +### displacy.parse_deps {id="displacy.parse_deps",tag="method",version="2"} Generate dependency parse in `{'words': [], 'arcs': []}` format. For use with the `manual=True` argument in `displacy.render`. @@ -291,11 +294,11 @@ the `manual=True` argument in `displacy.render`. | Name | Description | | ----------- | ------------------------------------------------------------------- | -| `orig_doc` | Doc to parse dependencies. ~~Doc~~ | +| `orig_doc` | Doc or span to parse dependencies. ~~Union[Doc, Span]~~ | | `options` | Dependency parse specific visualisation options. ~~Dict[str, Any]~~ | | **RETURNS** | Generated dependency parse keyed by words and arcs. ~~dict~~ | -### displacy.parse_ents {#displacy.parse_ents tag="method" new="2"} +### displacy.parse_ents {id="displacy.parse_ents",tag="method",version="2"} Generate named entities in `[{start: i, end: i, label: 'label'}]` format. For use with the `manual=True` argument in `displacy.render`. @@ -317,7 +320,7 @@ use with the `manual=True` argument in `displacy.render`. | `options` | NER-specific visualisation options. ~~Dict[str, Any]~~ | | **RETURNS** | Generated entities keyed by text (original text) and ents. ~~dict~~ | -### displacy.parse_spans {#displacy.parse_spans tag="method" new="2"} +### displacy.parse_spans {id="displacy.parse_spans",tag="method",version="2"} Generate spans in `[{start_token: i, end_token: i, label: 'label'}]` format. For use with the `manual=True` argument in `displacy.render`. @@ -340,12 +343,12 @@ use with the `manual=True` argument in `displacy.render`. | `options` | Span-specific visualisation options. ~~Dict[str, Any]~~ | | **RETURNS** | Generated entities keyed by text (original text) and ents. ~~dict~~ | -### Visualizer options {#displacy_options} +### Visualizer options {id="displacy_options"} The `options` argument lets you specify additional settings for each visualizer. If a setting is not present in the options, the default value will be used. -#### Dependency Visualizer options {#options-dep} +#### Dependency Visualizer options {id="options-dep"} > #### Example > @@ -354,24 +357,24 @@ If a setting is not present in the options, the default value will be used. > displacy.serve(doc, style="dep", options=options) > ``` -| Name | Description | -| ------------------ | -------------------------------------------------------------------------------------------------------------------------------------------- | -| `fine_grained` | Use fine-grained part-of-speech tags (`Token.tag_`) instead of coarse-grained tags (`Token.pos_`). Defaults to `False`. ~~bool~~ | -| `add_lemma` | Print the lemmas in a separate row below the token texts. Defaults to `False`. ~~bool~~ | -| `collapse_punct` | Attach punctuation to tokens. Can make the parse more readable, as it prevents long arcs to attach punctuation. Defaults to `True`. ~~bool~~ | -| `collapse_phrases` | Merge noun phrases into one token. Defaults to `False`. ~~bool~~ | -| `compact` | "Compact mode" with square arrows that takes up less space. Defaults to `False`. ~~bool~~ | -| `color` | Text color (HEX, RGB or color names). Defaults to `"#000000"`. ~~str~~ | -| `bg` | Background color (HEX, RGB or color names). Defaults to `"#ffffff"`. ~~str~~ | -| `font` | Font name or font family for all text. Defaults to `"Arial"`. ~~str~~ | -| `offset_x` | Spacing on left side of the SVG in px. Defaults to `50`. ~~int~~ | -| `arrow_stroke` | Width of arrow path in px. Defaults to `2`. ~~int~~ | -| `arrow_width` | Width of arrow head in px. Defaults to `10` in regular mode and `8` in compact mode. ~~int~~ | -| `arrow_spacing` | Spacing between arrows in px to avoid overlaps. Defaults to `20` in regular mode and `12` in compact mode. ~~int~~ | -| `word_spacing` | Vertical spacing between words and arcs in px. Defaults to `45`. ~~int~~ | -| `distance` | Distance between words in px. Defaults to `175` in regular mode and `150` in compact mode. ~~int~~ | +| Name | Description | +| ------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `fine_grained` | Use fine-grained part-of-speech tags (`Token.tag_`) instead of coarse-grained tags (`Token.pos_`). Defaults to `False`. ~~bool~~ | +| `add_lemma` | Print the lemmas in a separate row below the token texts. Defaults to `False`. ~~bool~~ | +| `collapse_punct` | Attach punctuation to tokens. Can make the parse more readable, as it prevents long arcs to attach punctuation. Defaults to `True`. ~~bool~~ | +| `collapse_phrases` | Merge noun phrases into one token. Defaults to `False`. ~~bool~~ | +| `compact` | "Compact mode" with square arrows that takes up less space. Defaults to `False`. ~~bool~~ | +| `color` | Text color. Can be provided in any CSS legal format as a string e.g.: `"#00ff00"`, `"rgb(0, 255, 0)"`, `"hsl(120, 100%, 50%)"` and `"green"` all correspond to the color green (without transparency). Defaults to `"#000000"`. ~~str~~ | +| `bg` | Background color. Can be provided in any CSS legal format as a string e.g.: `"#00ff00"`, `"rgb(0, 255, 0)"`, `"hsl(120, 100%, 50%)"` and `"green"` all correspond to the color green (without transparency). Defaults to `"#ffffff"`. ~~str~~ | +| `font` | Font name or font family for all text. Defaults to `"Arial"`. ~~str~~ | +| `offset_x` | Spacing on left side of the SVG in px. Defaults to `50`. ~~int~~ | +| `arrow_stroke` | Width of arrow path in px. Defaults to `2`. ~~int~~ | +| `arrow_width` | Width of arrow head in px. Defaults to `10` in regular mode and `8` in compact mode. ~~int~~ | +| `arrow_spacing` | Spacing between arrows in px to avoid overlaps. Defaults to `20` in regular mode and `12` in compact mode. ~~int~~ | +| `word_spacing` | Vertical spacing between words and arcs in px. Defaults to `45`. ~~int~~ | +| `distance` | Distance between words in px. Defaults to `175` in regular mode and `150` in compact mode. ~~int~~ | -#### Named Entity Visualizer options {#displacy_options-ent} +#### Named Entity Visualizer options {id="displacy_options-ent"} > #### Example > @@ -388,7 +391,7 @@ If a setting is not present in the options, the default value will be used. | `template` | Optional template to overwrite the HTML used to render entity spans. Should be a format string and can use `{bg}`, `{text}` and `{label}`. See [`templates.py`](%%GITHUB_SPACY/spacy/displacy/templates.py) for examples. ~~Optional[str]~~ | | `kb_url_template` 3.2.1 | Optional template to construct the KB url for the entity to link to. Expects a python f-string format with single field to fill in. ~~Optional[str]~~ | -#### Span Visualizer options {#displacy_options-span} +#### Span Visualizer options {id="displacy_options-span"} > #### Example > @@ -419,7 +422,7 @@ span. If you wish to link an entity to their URL then consider using the should redirect you to their Wikidata page, in this case `https://www.wikidata.org/wiki/Q95`. -## registry {#registry source="spacy/util.py" new="3"} +## registry {id="registry",source="spacy/util.py",version="3"} spaCy's function registry extends [Thinc's `registry`](https://thinc.ai/docs/api-config#registry) and allows you @@ -469,7 +472,7 @@ factories. | `scorers` | Registry for functions that create scoring methods for user with the [`Scorer`](/api/scorer). Scoring methods are called with `Iterable[Example]` and arbitrary `\*\*kwargs` and return scores as `Dict[str, Any]`. | | `tokenizers` | Registry for tokenizer factories. Registered functions should return a callback that receives the `nlp` object and returns a [`Tokenizer`](/api/tokenizer) or a custom callable. | -### spacy-transformers registry {#registry-transformers} +### spacy-transformers registry {id="registry-transformers"} The following registries are added by the [`spacy-transformers`](https://github.com/explosion/spacy-transformers) package. @@ -494,7 +497,7 @@ See the [`Transformer`](/api/transformer) API reference and | [`span_getters`](/api/transformer#span_getters) | Registry for functions that take a batch of `Doc` objects and return a list of `Span` objects to process by the transformer, e.g. sentences. | | [`annotation_setters`](/api/transformer#annotation_setters) | Registry for functions that create annotation setters. Annotation setters are functions that take a batch of `Doc` objects and a [`FullTransformerBatch`](/api/transformer#fulltransformerbatch) and can set additional annotations on the `Doc`. | -## Loggers {#loggers source="spacy/training/loggers.py" new="3"} +## Loggers {id="loggers",source="spacy/training/loggers.py",version="3"} A logger records the training results. When a logger is created, two functions are returned: one for logging the information for each training step, and a @@ -513,7 +516,7 @@ a [Weights & Biases](https://www.wandb.com/) dashboard. Instead of using one of the built-in loggers, you can [implement your own](/usage/training#custom-logging). -#### spacy.ConsoleLogger.v2 {#ConsoleLogger tag="registered function"} +#### spacy.ConsoleLogger.v2 {tag="registered function"} > #### Example config > @@ -530,7 +533,7 @@ saves them to a `jsonl` file. -```cli +```bash $ python -m spacy train config.cfg ``` @@ -564,15 +567,37 @@ start decreasing across epochs. -| Name | Description | -| ---------------- | --------------------------------------------------------------------- | -| `progress_bar` | Whether the logger should print the progress bar ~~bool~~ | -| `console_output` | Whether the logger should print the logs on the console. ~~bool~~ | -| `output_file` | The file to save the training logs to. ~~Optional[Union[str, Path]]~~ | +| Name | Description | +| ---------------- | ---------------------------------------------------------------------------------------------------------------------------- | +| `progress_bar` | Whether the logger should print a progress bar tracking the steps till the next evaluation pass (default: `False`). ~~bool~~ | +| `console_output` | Whether the logger should print the logs in the console (default: `True`). ~~bool~~ | +| `output_file` | The file to save the training logs to (default: `None`). ~~Optional[Union[str, Path]]~~ | -## Readers {#readers} +#### spacy.ConsoleLogger.v3 {id="ConsoleLogger",tag="registered function"} -### File readers {#file-readers source="github.com/explosion/srsly" new="3"} +> #### Example config +> +> ```ini +> [training.logger] +> @loggers = "spacy.ConsoleLogger.v3" +> progress_bar = "eval" +> console_output = true +> output_file = "training_log.jsonl" +> ``` + +Writes the results of a training step to the console in a tabular format and +optionally saves them to a `jsonl` file. + +| Name | Description | +| ---------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `progress_bar` | Type of progress bar to show in the console: `"train"`, `"eval"` or `None`. | +| | The bar tracks the number of steps until `training.max_steps` and `training.eval_frequency` are reached respectively (default: `None`). ~~Optional[str]~~ | +| `console_output` | Whether the logger should print the logs in the console (default: `True`). ~~bool~~ | +| `output_file` | The file to save the training logs to (default: `None`). ~~Optional[Union[str, Path]]~~ | + +## Readers {id="readers"} + +### File readers {id="file-readers",source="github.com/explosion/srsly",version="3"} The following file readers are provided by our serialization library [`srsly`](https://github.com/explosion/srsly). All registered functions take one @@ -602,7 +627,7 @@ blocks that are **not executed at runtime** – for example, in `[training]` and -#### spacy.read_labels.v1 {#read_labels tag="registered function"} +#### spacy.read_labels.v1 {id="read_labels",tag="registered function"} Read a JSON-formatted labels file generated with [`init labels`](/api/cli#init-labels). Typically used in the @@ -628,7 +653,7 @@ label sets. | `require` | Whether to require the file to exist. If set to `False` and the labels file doesn't exist, the loader will return `None` and the `initialize` method will extract the labels from the data. Defaults to `False`. ~~bool~~ | | **CREATES** | The list of labels. ~~List[str]~~ | -### Corpus readers {#corpus-readers source="spacy/training/corpus.py" new="3"} +### Corpus readers {id="corpus-readers",source="spacy/training/corpus.py",version="3"} Corpus readers are registered functions that load data and return a function that takes the current `nlp` object and yields [`Example`](/api/example) objects @@ -638,7 +663,7 @@ with your own registered function in the [`@readers` registry](/api/top-level#registry) to customize the data loading and streaming. -#### spacy.Corpus.v1 {#corpus tag="registered function"} +#### spacy.Corpus.v1 {id="corpus",tag="registered function"} The `Corpus` reader manages annotated corpora and can be used for training and development datasets in the [DocBin](/api/docbin) (`.spacy`) format. Also see @@ -667,7 +692,7 @@ the [`Corpus`](/api/corpus) class. | `augmenter` | Apply some simply data augmentation, where we replace tokens with variations. This is especially useful for punctuation and case replacement, to help generalize beyond corpora that don't have smart-quotes, or only have smart quotes, etc. Defaults to `None`. ~~Optional[Callable]~~ | | **CREATES** | The corpus reader. ~~Corpus~~ | -#### spacy.JsonlCorpus.v1 {#jsonlcorpus tag="registered function"} +#### spacy.JsonlCorpus.v1 {id="jsonlcorpus",tag="registered function"} Create [`Example`](/api/example) objects from a JSONL (newline-delimited JSON) file of texts keyed by `"text"`. Can be used to read the raw text corpus for @@ -696,7 +721,7 @@ JSONL file. Also see the [`JsonlCorpus`](/api/corpus#jsonlcorpus) class. | `limit` | Limit corpus to a subset of examples, e.g. for debugging. Defaults to `0` for no limit. ~~int~~ | | **CREATES** | The corpus reader. ~~JsonlCorpus~~ | -## Batchers {#batchers source="spacy/training/batchers.py" new="3"} +## Batchers {id="batchers",source="spacy/training/batchers.py",version="3"} A data batcher implements a batching strategy that essentially turns a stream of items into a stream of batches, with each batch consisting of one item or a list @@ -710,7 +735,7 @@ Instead of using one of the built-in batchers listed here, you can also [implement your own](/usage/training#custom-code-readers-batchers), which may or may not use a custom schedule. -### spacy.batch_by_words.v1 {#batch_by_words tag="registered function"} +### spacy.batch_by_words.v1 {id="batch_by_words",tag="registered function"} Create minibatches of roughly a given number of words. If any examples are longer than the specified batch length, they will appear in a batch by @@ -738,7 +763,7 @@ themselves, or be discarded if `discard_oversize` is set to `True`. The argument | `get_length` | Optional function that receives a sequence item and returns its length. Defaults to the built-in `len()` if not set. ~~Optional[Callable[[Any], int]]~~ | | **CREATES** | The batcher that takes an iterable of items and returns batches. ~~Callable[[Iterable[Any]], Iterable[List[Any]]]~~ | -### spacy.batch_by_sequence.v1 {#batch_by_sequence tag="registered function"} +### spacy.batch_by_sequence.v1 {id="batch_by_sequence",tag="registered function"} > #### Example config > @@ -757,7 +782,7 @@ Create a batcher that creates batches of the specified size. | `get_length` | Optional function that receives a sequence item and returns its length. Defaults to the built-in `len()` if not set. ~~Optional[Callable[[Any], int]]~~ | | **CREATES** | The batcher that takes an iterable of items and returns batches. ~~Callable[[Iterable[Any]], Iterable[List[Any]]]~~ | -### spacy.batch_by_padded.v1 {#batch_by_padded tag="registered function"} +### spacy.batch_by_padded.v1 {id="batch_by_padded",tag="registered function"} > #### Example config > @@ -783,7 +808,7 @@ sequences in the batch. | `get_length` | Optional function that receives a sequence item and returns its length. Defaults to the built-in `len()` if not set. ~~Optional[Callable[[Any], int]]~~ | | **CREATES** | The batcher that takes an iterable of items and returns batches. ~~Callable[[Iterable[Any]], Iterable[List[Any]]]~~ | -## Augmenters {#augmenters source="spacy/training/augment.py" new="3"} +## Augmenters {id="augmenters",source="spacy/training/augment.py",version="3"} Data augmentation is the process of applying small modifications to the training data. It can be especially useful for punctuation and case replacement – for @@ -792,7 +817,7 @@ variations using regular quotes, or to make the model less sensitive to capitalization by including a mix of capitalized and lowercase examples. See the [usage guide](/usage/training#data-augmentation) for details and examples. -### spacy.orth_variants.v1 {#orth_variants tag="registered function"} +### spacy.orth_variants.v1 {id="orth_variants",tag="registered function"} > #### Example config > @@ -819,7 +844,7 @@ beyond corpora that don't have smart quotes, or only have smart quotes etc. | `orth_variants` | A dictionary containing the single and paired orth variants. Typically loaded from a JSON file. See [`en_orth_variants.json`](https://github.com/explosion/spacy-lookups-data/blob/master/spacy_lookups_data/data/en_orth_variants.json) for an example. ~~Dict[str, Dict[List[Union[str, List[str]]]]]~~ | | **CREATES** | A function that takes the current `nlp` object and an [`Example`](/api/example) and yields augmented `Example` objects. ~~Callable[[Language, Example], Iterator[Example]]~~ | -### spacy.lower_case.v1 {#lower_case tag="registered function"} +### spacy.lower_case.v1 {id="lower_case",tag="registered function"} > #### Example config > @@ -838,12 +863,12 @@ useful for making the model less sensitive to capitalization. | `level` | The percentage of texts that will be augmented. ~~float~~ | | **CREATES** | A function that takes the current `nlp` object and an [`Example`](/api/example) and yields augmented `Example` objects. ~~Callable[[Language, Example], Iterator[Example]]~~ | -## Callbacks {#callbacks source="spacy/training/callbacks.py" new="3"} +## Callbacks {id="callbacks",source="spacy/training/callbacks.py",version="3"} The config supports [callbacks](/usage/training#custom-code-nlp-callbacks) at several points in the lifecycle that can be used modify the `nlp` object. -### spacy.copy_from_base_model.v1 {#copy_from_base_model tag="registered function"} +### spacy.copy_from_base_model.v1 {id="copy_from_base_model",tag="registered function"} > #### Example config > @@ -867,7 +892,7 @@ from the specified model. Intended for use in `[initialize.before_init]`. | `vocab` | The pipeline to copy the vocab from. The vocab includes the lookups and vectors. Defaults to `None`. ~~Optional[str]~~ | | **CREATES** | A function that takes the current `nlp` object and modifies its `tokenizer` and `vocab`. ~~Callable[[Language], None]~~ | -### spacy.models_with_nvtx_range.v1 {#models_with_nvtx_range tag="registered function"} +### spacy.models_with_nvtx_range.v1 {id="models_with_nvtx_range",tag="registered function"} > #### Example config > @@ -887,7 +912,7 @@ backprop passes. | `backprop_color` | Color identifier for backpropagation passes. Defaults to `-1`. ~~int~~ | | **CREATES** | A function that takes the current `nlp` and wraps forward/backprop passes in NVTX ranges. ~~Callable[[Language], Language]~~ | -### spacy.models_and_pipes_with_nvtx_range.v1 {#models_and_pipes_with_nvtx_range tag="registered function" new="3.4"} +### spacy.models_and_pipes_with_nvtx_range.v1 {id="models_and_pipes_with_nvtx_range",tag="registered function",version="3.4"} > #### Example config > @@ -908,9 +933,9 @@ methods are wrapped: `pipe`, `predict`, `set_annotations`, `update`, `rehearse`, | `additional_pipe_functions` | Additional pipeline methods to wrap. Keys are pipeline names and values are lists of method identifiers. Defaults to `None`. ~~Optional[Dict[str, List[str]]]~~ | | **CREATES** | A function that takes the current `nlp` and wraps pipe models and methods in NVTX ranges. ~~Callable[[Language], Language]~~ | -## Training data and alignment {#gold source="spacy/training"} +## Training data and alignment {id="gold",source="spacy/training"} -### training.offsets_to_biluo_tags {#offsets_to_biluo_tags tag="function"} +### training.offsets_to_biluo_tags {id="offsets_to_biluo_tags",tag="function"} Encode labelled spans into per-token tags, using the [BILUO scheme](/usage/linguistic-features#accessing-ner) (Begin, In, Last, Unit, @@ -947,7 +972,7 @@ This method was previously available as `spacy.gold.biluo_tags_from_offsets`. | `missing` | The label used for missing values, e.g. if tokenization doesn't align with the entity offsets. Defaults to `"O"`. ~~str~~ | | **RETURNS** | A list of strings, describing the [BILUO](/usage/linguistic-features#accessing-ner) tags. ~~List[str]~~ | -### training.biluo_tags_to_offsets {#biluo_tags_to_offsets tag="function"} +### training.biluo_tags_to_offsets {id="biluo_tags_to_offsets",tag="function"} Encode per-token tags following the [BILUO scheme](/usage/linguistic-features#accessing-ner) into entity offsets. @@ -975,7 +1000,7 @@ This method was previously available as `spacy.gold.offsets_from_biluo_tags`. | `tags` | A sequence of [BILUO](/usage/linguistic-features#accessing-ner) tags with each tag describing one token. Each tag string will be of the form of either `""`, `"O"` or `"{action}-{label}"`, where action is one of `"B"`, `"I"`, `"L"`, `"U"`. ~~List[str]~~ | | **RETURNS** | A sequence of `(start, end, label)` triples. `start` and `end` will be character-offset integers denoting the slice into the original string. ~~List[Tuple[int, int, str]]~~ | -### training.biluo_tags_to_spans {#biluo_tags_to_spans tag="function" new="2.1"} +### training.biluo_tags_to_spans {id="biluo_tags_to_spans",tag="function",version="2.1"} Encode per-token tags following the [BILUO scheme](/usage/linguistic-features#accessing-ner) into @@ -1004,7 +1029,7 @@ This method was previously available as `spacy.gold.spans_from_biluo_tags`. | `tags` | A sequence of [BILUO](/usage/linguistic-features#accessing-ner) tags with each tag describing one token. Each tag string will be of the form of either `""`, `"O"` or `"{action}-{label}"`, where action is one of `"B"`, `"I"`, `"L"`, `"U"`. ~~List[str]~~ | | **RETURNS** | A sequence of `Span` objects with added entity labels. ~~List[Span]~~ | -### training.biluo_to_iob {#biluo_to_iob tag="function"} +### training.biluo_to_iob {id="biluo_to_iob",tag="function"} Convert a sequence of [BILUO](/usage/linguistic-features#accessing-ner) tags to [IOB](/usage/linguistic-features#accessing-ner) tags. This is useful if you want @@ -1025,7 +1050,7 @@ use the BILUO tags with a model that only supports IOB tags. | `tags` | A sequence of [BILUO](/usage/linguistic-features#accessing-ner) tags. ~~Iterable[str]~~ | | **RETURNS** | A list of [IOB](/usage/linguistic-features#accessing-ner) tags. ~~List[str]~~ | -### training.iob_to_biluo {#iob_to_biluo tag="function"} +### training.iob_to_biluo {id="iob_to_biluo",tag="function"} Convert a sequence of [IOB](/usage/linguistic-features#accessing-ner) tags to [BILUO](/usage/linguistic-features#accessing-ner) tags. This is useful if you @@ -1052,7 +1077,55 @@ This method was previously available as `spacy.gold.iob_to_biluo`. | `tags` | A sequence of [IOB](/usage/linguistic-features#accessing-ner) tags. ~~Iterable[str]~~ | | **RETURNS** | A list of [BILUO](/usage/linguistic-features#accessing-ner) tags. ~~List[str]~~ | -## Utility functions {#util source="spacy/util.py"} +### training.biluo_to_iob {id="biluo_to_iob",tag="function"} + +Convert a sequence of [BILUO](/usage/linguistic-features#accessing-ner) tags to +[IOB](/usage/linguistic-features#accessing-ner) tags. This is useful if you want +use the BILUO tags with a model that only supports IOB tags. + +> #### Example +> +> ```python +> from spacy.training import biluo_to_iob +> +> tags = ["O", "O", "B-LOC", "I-LOC", "L-LOC", "O"] +> iob_tags = biluo_to_iob(tags) +> assert iob_tags == ["O", "O", "B-LOC", "I-LOC", "I-LOC", "O"] +> ``` + +| Name | Description | +| ----------- | --------------------------------------------------------------------------------------- | +| `tags` | A sequence of [BILUO](/usage/linguistic-features#accessing-ner) tags. ~~Iterable[str]~~ | +| **RETURNS** | A list of [IOB](/usage/linguistic-features#accessing-ner) tags. ~~List[str]~~ | + +### training.iob_to_biluo {id="iob_to_biluo",tag="function"} + +Convert a sequence of [IOB](/usage/linguistic-features#accessing-ner) tags to +[BILUO](/usage/linguistic-features#accessing-ner) tags. This is useful if you +want use the IOB tags with a model that only supports BILUO tags. + + + +This method was previously available as `spacy.gold.iob_to_biluo`. + + + +> #### Example +> +> ```python +> from spacy.training import iob_to_biluo +> +> tags = ["O", "O", "B-LOC", "I-LOC", "O"] +> biluo_tags = iob_to_biluo(tags) +> assert biluo_tags == ["O", "O", "B-LOC", "L-LOC", "O"] +> ``` + +| Name | Description | +| ----------- | ------------------------------------------------------------------------------------- | +| `tags` | A sequence of [IOB](/usage/linguistic-features#accessing-ner) tags. ~~Iterable[str]~~ | +| **RETURNS** | A list of [BILUO](/usage/linguistic-features#accessing-ner) tags. ~~List[str]~~ | + +## Utility functions {id="util",source="spacy/util.py"} spaCy comes with a small collection of utility functions located in [`spacy/util.py`](%%GITHUB_SPACY/spacy/util.py). Because utility functions are @@ -1062,7 +1135,7 @@ use and we'll try to ensure backwards compatibility. However, we recommend having additional tests in place if your application depends on any of spaCy's utilities. -### util.get_lang_class {#util.get_lang_class tag="function"} +### util.get_lang_class {id="util.get_lang_class",tag="function"} Import and load a `Language` class. Allows lazy-loading [language data](/usage/linguistic-features#language-data) and importing @@ -1083,7 +1156,7 @@ custom language class, you can register it using the | `lang` | Two-letter language code, e.g. `"en"`. ~~str~~ | | **RETURNS** | The respective subclass. ~~Language~~ | -### util.lang_class_is_loaded {#util.lang_class_is_loaded tag="function" new="2.1"} +### util.lang_class_is_loaded {id="util.lang_class_is_loaded",tag="function",version="2.1"} Check whether a `Language` subclass is already loaded. `Language` subclasses are loaded lazily to avoid expensive setup code associated with the language data. @@ -1101,7 +1174,7 @@ loaded lazily to avoid expensive setup code associated with the language data. | `name` | Two-letter language code, e.g. `"en"`. ~~str~~ | | **RETURNS** | Whether the class has been loaded. ~~bool~~ | -### util.load_model {#util.load_model tag="function" new="2"} +### util.load_model {id="util.load_model",tag="function",version="2"} Load a pipeline from a package or data path. If called with a string name, spaCy will assume the pipeline is a Python package and import and call its `load()` @@ -1129,7 +1202,7 @@ and create a `Language` object. The model data will then be loaded in via | `config` 3 | Config overrides as nested dict or flat dict keyed by section values in dot notation, e.g. `"nlp.pipeline"`. ~~Union[Dict[str, Any], Config]~~ | | **RETURNS** | `Language` class with the loaded pipeline. ~~Language~~ | -### util.load_model_from_init_py {#util.load_model_from_init_py tag="function" new="2"} +### util.load_model_from_init_py {id="util.load_model_from_init_py",tag="function",version="2"} A helper function to use in the `load()` method of a pipeline package's [`__init__.py`](https://github.com/explosion/spacy-models/tree/master/template/model/xx_model_name/__init__.py). @@ -1154,7 +1227,7 @@ A helper function to use in the `load()` method of a pipeline package's | `config` 3 | Config overrides as nested dict or flat dict keyed by section values in dot notation, e.g. `"nlp.pipeline"`. ~~Union[Dict[str, Any], Config]~~ | | **RETURNS** | `Language` class with the loaded pipeline. ~~Language~~ | -### util.load_config {#util.load_config tag="function" new="3"} +### util.load_config {id="util.load_config",tag="function",version="3"} Load a pipeline's [`config.cfg`](/api/data-formats#config) from a file path. The config typically includes details about the components and how they're created, @@ -1174,7 +1247,7 @@ as well as all training settings and hyperparameters. | `interpolate` | Whether to interpolate the config and replace variables like `${paths.train}` with their values. Defaults to `False`. ~~bool~~ | | **RETURNS** | The pipeline's config. ~~Config~~ | -### util.load_meta {#util.load_meta tag="function" new="3"} +### util.load_meta {id="util.load_meta",tag="function",version="3"} Get a pipeline's [`meta.json`](/api/data-formats#meta) from a file path and validate its contents. The meta typically includes details about author, @@ -1191,7 +1264,7 @@ licensing, data sources and version. | `path` | Path to the pipeline's `meta.json`. ~~Union[str, Path]~~ | | **RETURNS** | The pipeline's meta data. ~~Dict[str, Any]~~ | -### util.get_installed_models {#util.get_installed_models tag="function" new="3"} +### util.get_installed_models {id="util.get_installed_models",tag="function",version="3"} List all pipeline packages installed in the current environment. This will include any spaCy pipeline that was packaged with @@ -1209,7 +1282,7 @@ object. | ----------- | ------------------------------------------------------------------------------------- | | **RETURNS** | The string names of the pipelines installed in the current environment. ~~List[str]~~ | -### util.is_package {#util.is_package tag="function"} +### util.is_package {id="util.is_package",tag="function"} Check if string maps to a package installed via pip. Mainly used to validate [pipeline packages](/usage/models). @@ -1226,7 +1299,7 @@ Check if string maps to a package installed via pip. Mainly used to validate | `name` | Name of package. ~~str~~ | | **RETURNS** | `True` if installed package, `False` if not. ~~bool~~ | -### util.get_package_path {#util.get_package_path tag="function" new="2"} +### util.get_package_path {id="util.get_package_path",tag="function",version="2"} Get path to an installed package. Mainly used to resolve the location of [pipeline packages](/usage/models). Currently imports the package to find its @@ -1244,7 +1317,7 @@ path. | `package_name` | Name of installed package. ~~str~~ | | **RETURNS** | Path to pipeline package directory. ~~Path~~ | -### util.is_in_jupyter {#util.is_in_jupyter tag="function" new="2"} +### util.is_in_jupyter {id="util.is_in_jupyter",tag="function",version="2"} Check if user is running spaCy from a [Jupyter](https://jupyter.org) notebook by detecting the IPython kernel. Mainly used for the @@ -1263,7 +1336,7 @@ detecting the IPython kernel. Mainly used for the | ----------- | ---------------------------------------------- | | **RETURNS** | `True` if in Jupyter, `False` if not. ~~bool~~ | -### util.compile_prefix_regex {#util.compile_prefix_regex tag="function"} +### util.compile_prefix_regex {id="util.compile_prefix_regex",tag="function"} Compile a sequence of prefix rules into a regex object. @@ -1280,7 +1353,7 @@ Compile a sequence of prefix rules into a regex object. | `entries` | The prefix rules, e.g. [`lang.punctuation.TOKENIZER_PREFIXES`](%%GITHUB_SPACY/spacy/lang/punctuation.py). ~~Iterable[Union[str, Pattern]]~~ | | **RETURNS** | The regex object to be used for [`Tokenizer.prefix_search`](/api/tokenizer#attributes). ~~Pattern~~ | -### util.compile_suffix_regex {#util.compile_suffix_regex tag="function"} +### util.compile_suffix_regex {id="util.compile_suffix_regex",tag="function"} Compile a sequence of suffix rules into a regex object. @@ -1297,7 +1370,7 @@ Compile a sequence of suffix rules into a regex object. | `entries` | The suffix rules, e.g. [`lang.punctuation.TOKENIZER_SUFFIXES`](%%GITHUB_SPACY/spacy/lang/punctuation.py). ~~Iterable[Union[str, Pattern]]~~ | | **RETURNS** | The regex object to be used for [`Tokenizer.suffix_search`](/api/tokenizer#attributes). ~~Pattern~~ | -### util.compile_infix_regex {#util.compile_infix_regex tag="function"} +### util.compile_infix_regex {id="util.compile_infix_regex",tag="function"} Compile a sequence of infix rules into a regex object. @@ -1314,7 +1387,7 @@ Compile a sequence of infix rules into a regex object. | `entries` | The infix rules, e.g. [`lang.punctuation.TOKENIZER_INFIXES`](%%GITHUB_SPACY/spacy/lang/punctuation.py). ~~Iterable[Union[str, Pattern]]~~ | | **RETURNS** | The regex object to be used for [`Tokenizer.infix_finditer`](/api/tokenizer#attributes). ~~Pattern~~ | -### util.minibatch {#util.minibatch tag="function" new="2"} +### util.minibatch {id="util.minibatch",tag="function",version="2"} Iterate over batches of items. `size` may be an iterator, so that batch-size can vary on each step. @@ -1333,7 +1406,7 @@ vary on each step. | `size` | The batch size(s). ~~Union[int, Sequence[int]]~~ | | **YIELDS** | The batches. | -### util.filter_spans {#util.filter_spans tag="function" new="2.1.4"} +### util.filter_spans {id="util.filter_spans",tag="function",version="2.1.4"} Filter a sequence of [`Span`](/api/span) objects and remove duplicates or overlaps. Useful for creating named entities (where one token can only be part @@ -1354,7 +1427,7 @@ of one entity) or when merging spans with | `spans` | The spans to filter. ~~Iterable[Span]~~ | | **RETURNS** | The filtered spans. ~~List[Span]~~ | -### util.get_words_and_spaces {#get_words_and_spaces tag="function" new="3"} +### util.get_words_and_spaces {id="get_words_and_spaces",tag="function",version="3"} Given a list of words and a text, reconstruct the original tokens and return a list of words and spaces that can be used to create a [`Doc`](/api/doc#init). diff --git a/website/docs/api/transformer.md b/website/docs/api/transformer.mdx similarity index 95% rename from website/docs/api/transformer.md rename to website/docs/api/transformer.mdx index e747ad383..ad8ecce54 100644 --- a/website/docs/api/transformer.md +++ b/website/docs/api/transformer.mdx @@ -3,7 +3,7 @@ title: Transformer teaser: Pipeline component for multi-task learning with transformer models tag: class source: github.com/explosion/spacy-transformers/blob/master/spacy_transformers/pipeline_component.py -new: 3 +version: 3 api_base_class: /api/pipe api_string_name: transformer --- @@ -44,7 +44,7 @@ package also adds the function registries [`@span_getters`](#span_getters) and functions. For more details, see the [usage documentation](/usage/embeddings-transformers). -## Assigned Attributes {#assigned-attributes} +## Assigned Attributes {id="assigned-attributes"} The component sets the following [custom extension attribute](/usage/processing-pipeline#custom-components-attributes): @@ -53,7 +53,7 @@ The component sets the following | ---------------- | ------------------------------------------------------------------------ | | `Doc._.trf_data` | Transformer tokens and outputs for the `Doc` object. ~~TransformerData~~ | -## Config and implementation {#config} +## Config and implementation {id="config"} The default config is defined by the pipeline component factory and describes how the component should be configured. You can override its settings via the @@ -81,7 +81,7 @@ on the transformer architectures and their arguments and hyperparameters. https://github.com/explosion/spacy-transformers/blob/master/spacy_transformers/pipeline_component.py ``` -## Transformer.\_\_init\_\_ {#init tag="method"} +## Transformer.\_\_init\_\_ {id="init",tag="method"} > #### Example > @@ -124,7 +124,7 @@ component using its string name and [`nlp.add_pipe`](/api/language#create_pipe). | `name` | String name of the component instance. Used to add entries to the `losses` during training. ~~str~~ | | `max_batch_items` | Maximum size of a padded batch. Defaults to `128*32`. ~~int~~ | -## Transformer.\_\_call\_\_ {#call tag="method"} +## Transformer.\_\_call\_\_ {id="call",tag="method"} Apply the pipe to one document. The document is modified in place, and returned. This usually happens under the hood when the `nlp` object is called on a text @@ -147,7 +147,7 @@ to the [`predict`](/api/transformer#predict) and | `doc` | The document to process. ~~Doc~~ | | **RETURNS** | The processed document. ~~Doc~~ | -## Transformer.pipe {#pipe tag="method"} +## Transformer.pipe {id="pipe",tag="method"} Apply the pipe to a stream of documents. This usually happens under the hood when the `nlp` object is called on a text and all pipeline components are @@ -171,7 +171,7 @@ applied to the `Doc` in order. Both [`__call__`](/api/transformer#call) and | `batch_size` | The number of documents to buffer. Defaults to `128`. ~~int~~ | | **YIELDS** | The processed documents in order. ~~Doc~~ | -## Transformer.initialize {#initialize tag="method"} +## Transformer.initialize {id="initialize",tag="method"} Initialize the component for training and return an [`Optimizer`](https://thinc.ai/docs/api-optimizers). `get_examples` should be a @@ -196,7 +196,7 @@ by [`Language.initialize`](/api/language#initialize). | _keyword-only_ | | | `nlp` | The current `nlp` object. Defaults to `None`. ~~Optional[Language]~~ | -## Transformer.predict {#predict tag="method"} +## Transformer.predict {id="predict",tag="method"} Apply the component's model to a batch of [`Doc`](/api/doc) objects without modifying them. @@ -213,7 +213,7 @@ modifying them. | `docs` | The documents to predict. ~~Iterable[Doc]~~ | | **RETURNS** | The model's prediction for each document. | -## Transformer.set_annotations {#set_annotations tag="method"} +## Transformer.set_annotations {id="set_annotations",tag="method"} Assign the extracted features to the `Doc` objects. By default, the [`TransformerData`](/api/transformer#transformerdata) object is written to the @@ -233,7 +233,7 @@ callback is then called, if provided. | `docs` | The documents to modify. ~~Iterable[Doc]~~ | | `scores` | The scores to set, produced by `Transformer.predict`. | -## Transformer.update {#update tag="method"} +## Transformer.update {id="update",tag="method"} Prepare for an update to the transformer. Like the [`Tok2Vec`](/api/tok2vec) component, the `Transformer` component is unusual in that it does not receive @@ -266,7 +266,7 @@ and call the optimizer, while the others simply increment the gradients. | `losses` | Optional record of the loss during training. Updated using the component name as the key. ~~Optional[Dict[str, float]]~~ | | **RETURNS** | The updated `losses` dictionary. ~~Dict[str, float]~~ | -## Transformer.create_optimizer {#create_optimizer tag="method"} +## Transformer.create_optimizer {id="create_optimizer",tag="method"} Create an optimizer for the pipeline component. @@ -281,7 +281,7 @@ Create an optimizer for the pipeline component. | ----------- | ---------------------------- | | **RETURNS** | The optimizer. ~~Optimizer~~ | -## Transformer.use_params {#use_params tag="method, contextmanager"} +## Transformer.use_params {id="use_params",tag="method, contextmanager"} Modify the pipe's model to use the given parameter values. At the end of the context, the original parameters are restored. @@ -298,7 +298,7 @@ context, the original parameters are restored. | -------- | -------------------------------------------------- | | `params` | The parameter values to use in the model. ~~dict~~ | -## Transformer.to_disk {#to_disk tag="method"} +## Transformer.to_disk {id="to_disk",tag="method"} Serialize the pipe to disk. @@ -315,7 +315,7 @@ Serialize the pipe to disk. | _keyword-only_ | | | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | -## Transformer.from_disk {#from_disk tag="method"} +## Transformer.from_disk {id="from_disk",tag="method"} Load the pipe from disk. Modifies the object in place and returns it. @@ -333,7 +333,7 @@ Load the pipe from disk. Modifies the object in place and returns it. | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | | **RETURNS** | The modified `Transformer` object. ~~Transformer~~ | -## Transformer.to_bytes {#to_bytes tag="method"} +## Transformer.to_bytes {id="to_bytes",tag="method"} > #### Example > @@ -350,7 +350,7 @@ Serialize the pipe to a bytestring. | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | | **RETURNS** | The serialized form of the `Transformer` object. ~~bytes~~ | -## Transformer.from_bytes {#from_bytes tag="method"} +## Transformer.from_bytes {id="from_bytes",tag="method"} Load the pipe from a bytestring. Modifies the object in place and returns it. @@ -369,7 +369,7 @@ Load the pipe from a bytestring. Modifies the object in place and returns it. | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | | **RETURNS** | The `Transformer` object. ~~Transformer~~ | -## Serialization fields {#serialization-fields} +## Serialization fields {id="serialization-fields"} During serialization, spaCy will export several data fields used to restore different aspects of the object. If needed, you can exclude them from @@ -387,7 +387,7 @@ serialization by passing in the string names via the `exclude` argument. | `cfg` | The config file. You usually don't want to exclude this. | | `model` | The binary model data. You usually don't want to exclude this. | -## TransformerData {#transformerdata tag="dataclass"} +## TransformerData {id="transformerdata",tag="dataclass"} Transformer tokens and outputs for one `Doc` object. The transformer models return tensors that refer to a whole padded batch of documents. These tensors @@ -405,7 +405,7 @@ by this class. Instances of this class are typically assigned to the | `align` | Alignment from the `Doc`'s tokenization to the wordpieces. This is a ragged array, where `align.lengths[i]` indicates the number of wordpiece tokens that token `i` aligns against. The actual indices are provided at `align[i].dataXd`. ~~Ragged~~ | | `width` | The width of the last hidden layer. ~~int~~ | -### TransformerData.empty {#transformerdata-emoty tag="classmethod"} +### TransformerData.empty {id="transformerdata-emoty",tag="classmethod"} Create an empty `TransformerData` container. @@ -425,7 +425,7 @@ model. -## FullTransformerBatch {#fulltransformerbatch tag="dataclass"} +## FullTransformerBatch {id="fulltransformerbatch",tag="dataclass"} Holds a batch of input and output objects for a transformer model. The data can then be split to a list of [`TransformerData`](/api/transformer#transformerdata) @@ -440,7 +440,7 @@ objects to associate the outputs to each [`Doc`](/api/doc) in the batch. | `align` | Alignment from the spaCy tokenization to the wordpieces. This is a ragged array, where `align.lengths[i]` indicates the number of wordpiece tokens that token `i` aligns against. The actual indices are provided at `align[i].dataXd`. ~~Ragged~~ | | `doc_data` | The outputs, split per `Doc` object. ~~List[TransformerData]~~ | -### FullTransformerBatch.unsplit_by_doc {#fulltransformerbatch-unsplit_by_doc tag="method"} +### FullTransformerBatch.unsplit_by_doc {id="fulltransformerbatch-unsplit_by_doc",tag="method"} Return a new `FullTransformerBatch` from a split batch of activations, using the current object's spans, tokens and alignment. This is used during the backward @@ -452,7 +452,7 @@ model. | `arrays` | The split batch of activations. ~~List[List[Floats3d]]~~ | | **RETURNS** | The transformer batch. ~~FullTransformerBatch~~ | -### FullTransformerBatch.split_by_doc {#fulltransformerbatch-split_by_doc tag="method"} +### FullTransformerBatch.split_by_doc {id="fulltransformerbatch-split_by_doc",tag="method"} Split a `TransformerData` object that represents a batch into a list with one `TransformerData` per `Doc`. @@ -468,7 +468,7 @@ In `spacy-transformers` v1.0, the model output is stored in
-## Span getters {#span_getters source="github.com/explosion/spacy-transformers/blob/master/spacy_transformers/span_getters.py"} +## Span getters {id="span_getters",source="github.com/explosion/spacy-transformers/blob/master/spacy_transformers/span_getters.py"} Span getters are functions that take a batch of [`Doc`](/api/doc) objects and return a lists of [`Span`](/api/span) objects for each doc to be processed by @@ -498,7 +498,7 @@ using the `@spacy.registry.span_getters` decorator. | `docs` | A batch of `Doc` objects. ~~Iterable[Doc]~~ | | **RETURNS** | The spans to process by the transformer. ~~List[List[Span]]~~ | -### doc_spans.v1 {#doc_spans tag="registered function"} +### doc_spans.v1 {id="doc_spans",tag="registered function"} > #### Example config > @@ -511,7 +511,7 @@ Create a span getter that uses the whole document as its spans. This is the best approach if your [`Doc`](/api/doc) objects already refer to relatively short texts. -### sent_spans.v1 {#sent_spans tag="registered function"} +### sent_spans.v1 {id="sent_spans",tag="registered function"} > #### Example config > @@ -531,7 +531,7 @@ To set sentence boundaries with the `sentencizer` during training, add a [`[training.annotating_components]`](/usage/training#annotating-components) to have it set the sentence boundaries before the `transformer` component runs. -### strided_spans.v1 {#strided_spans tag="registered function"} +### strided_spans.v1 {id="strided_spans",tag="registered function"} > #### Example config > @@ -553,7 +553,7 @@ right context. | `window` | The window size. ~~int~~ | | `stride` | The stride size. ~~int~~ | -## Annotation setters {#annotation_setters tag="registered functions" source="github.com/explosion/spacy-transformers/blob/master/spacy_transformers/annotation_setters.py"} +## Annotation setters {id="annotation_setters",tag="registered functions",source="github.com/explosion/spacy-transformers/blob/master/spacy_transformers/annotation_setters.py"} Annotation setters are functions that take a batch of `Doc` objects and a [`FullTransformerBatch`](/api/transformer#fulltransformerbatch) and can set diff --git a/website/docs/api/vectors.md b/website/docs/api/vectors.mdx similarity index 94% rename from website/docs/api/vectors.md rename to website/docs/api/vectors.mdx index d4702b592..d6033c096 100644 --- a/website/docs/api/vectors.md +++ b/website/docs/api/vectors.mdx @@ -3,7 +3,7 @@ title: Vectors teaser: Store, save and load word vectors tag: class source: spacy/vectors.pyx -new: 2 +version: 2 --- Vectors data is kept in the `Vectors.data` attribute, which should be an @@ -25,7 +25,7 @@ As of spaCy v3.2, `Vectors` supports two types of vector tables: the sum of one or more rows as determined by the settings related to character ngrams and the hash table. -## Vectors.\_\_init\_\_ {#init tag="method"} +## Vectors.\_\_init\_\_ {id="init",tag="method"} Create a new vector store. With the default mode, you can set the vector values and keys directly on initialization, or supply a `shape` keyword argument to @@ -61,7 +61,7 @@ modified later. | `bow` 3.2 | The floret BOW string (default: `"<"`). ~~str~~ | | `eow` 3.2 | The floret EOW string (default: `">"`). ~~str~~ | -## Vectors.\_\_getitem\_\_ {#getitem tag="method"} +## Vectors.\_\_getitem\_\_ {id="getitem",tag="method"} Get a vector by key. If the key is not found in the table, a `KeyError` is raised. @@ -79,7 +79,7 @@ raised. | `key` | The key to get the vector for. ~~Union[int, str]~~ | | **RETURNS** | The vector for the key. ~~numpy.ndarray[ndim=1, dtype=float32]~~ | -## Vectors.\_\_setitem\_\_ {#setitem tag="method"} +## Vectors.\_\_setitem\_\_ {id="setitem",tag="method"} Set a vector for the given key. Not supported for `floret` mode. @@ -96,7 +96,7 @@ Set a vector for the given key. Not supported for `floret` mode. | `key` | The key to set the vector for. ~~int~~ | | `vector` | The vector to set. ~~numpy.ndarray[ndim=1, dtype=float32]~~ | -## Vectors.\_\_iter\_\_ {#iter tag="method"} +## Vectors.\_\_iter\_\_ {id="iter",tag="method"} Iterate over the keys in the table. In `floret` mode, the keys table is not used. @@ -112,7 +112,7 @@ used. | ---------- | --------------------------- | | **YIELDS** | A key in the table. ~~int~~ | -## Vectors.\_\_len\_\_ {#len tag="method"} +## Vectors.\_\_len\_\_ {id="len",tag="method"} Return the number of vectors in the table. @@ -127,7 +127,7 @@ Return the number of vectors in the table. | ----------- | ------------------------------------------- | | **RETURNS** | The number of vectors in the table. ~~int~~ | -## Vectors.\_\_contains\_\_ {#contains tag="method"} +## Vectors.\_\_contains\_\_ {id="contains",tag="method"} Check whether a key has been mapped to a vector entry in the table. In `floret` mode, returns `True` for all keys. @@ -145,7 +145,7 @@ mode, returns `True` for all keys. | `key` | The key to check. ~~int~~ | | **RETURNS** | Whether the key has a vector entry. ~~bool~~ | -## Vectors.add {#add tag="method"} +## Vectors.add {id="add",tag="method"} Add a key to the table, optionally setting a vector value as well. Keys can be mapped to an existing vector by setting `row`, or a new vector can be added. Not @@ -168,7 +168,7 @@ supported for `floret` mode. | `row` | An optional row number of a vector to map the key to. ~~int~~ | | **RETURNS** | The row the vector was added to. ~~int~~ | -## Vectors.resize {#resize tag="method"} +## Vectors.resize {id="resize",tag="method"} Resize the underlying vectors array. If `inplace=True`, the memory is reallocated. This may cause other references to the data to become invalid, so @@ -189,7 +189,7 @@ for `floret` mode. | `inplace` | Reallocate the memory. ~~bool~~ | | **RETURNS** | The removed items as a list of `(key, row)` tuples. ~~List[Tuple[int, int]]~~ | -## Vectors.keys {#keys tag="method"} +## Vectors.keys {id="keys",tag="method"} A sequence of the keys in the table. In `floret` mode, the keys table is not used. @@ -205,7 +205,7 @@ used. | ----------- | --------------------------- | | **RETURNS** | The keys. ~~Iterable[int]~~ | -## Vectors.values {#values tag="method"} +## Vectors.values {id="values",tag="method"} Iterate over vectors that have been assigned to at least one key. Note that some vectors may be unassigned, so the number of vectors returned may be less than @@ -222,7 +222,7 @@ the length of the vectors table. In `floret` mode, the keys table is not used. | ---------- | --------------------------------------------------------------- | | **YIELDS** | A vector in the table. ~~numpy.ndarray[ndim=1, dtype=float32]~~ | -## Vectors.items {#items tag="method"} +## Vectors.items {id="items",tag="method"} Iterate over `(key, vector)` pairs, in order. In `floret` mode, the keys table is empty. @@ -238,7 +238,7 @@ is empty. | ---------- | ------------------------------------------------------------------------------------- | | **YIELDS** | `(key, vector)` pairs, in order. ~~Tuple[int, numpy.ndarray[ndim=1, dtype=float32]]~~ | -## Vectors.find {#find tag="method"} +## Vectors.find {id="find",tag="method"} Look up one or more keys by row, or vice versa. Not supported for `floret` mode. @@ -260,7 +260,7 @@ Look up one or more keys by row, or vice versa. Not supported for `floret` mode. | `rows` | Find the keys that point to the rows. Returns `numpy.ndarray`. ~~Iterable[int]~~ | | **RETURNS** | The requested key, keys, row or rows. ~~Union[int, numpy.ndarray[ndim=1, dtype=float32]]~~ | -## Vectors.shape {#shape tag="property"} +## Vectors.shape {id="shape",tag="property"} Get `(rows, dims)` tuples of number of rows and number of dimensions in the vector table. @@ -279,7 +279,7 @@ vector table. | ----------- | ------------------------------------------ | | **RETURNS** | A `(rows, dims)` pair. ~~Tuple[int, int]~~ | -## Vectors.size {#size tag="property"} +## Vectors.size {id="size",tag="property"} The vector size, i.e. `rows * dims`. @@ -294,7 +294,7 @@ The vector size, i.e. `rows * dims`. | ----------- | ------------------------ | | **RETURNS** | The vector size. ~~int~~ | -## Vectors.is_full {#is_full tag="property"} +## Vectors.is_full {id="is_full",tag="property"} Whether the vectors table is full and has no slots are available for new keys. If a table is full, it can be resized using @@ -313,7 +313,7 @@ full and cannot be resized. | ----------- | ------------------------------------------- | | **RETURNS** | Whether the vectors table is full. ~~bool~~ | -## Vectors.n_keys {#n_keys tag="property"} +## Vectors.n_keys {id="n_keys",tag="property"} Get the number of keys in the table. Note that this is the number of _all_ keys, not just unique vectors. If several keys are mapped to the same vectors, they @@ -331,7 +331,7 @@ will be counted individually. In `floret` mode, the keys table is not used. | ----------- | ----------------------------------------------------------------------------- | | **RETURNS** | The number of all keys in the table. Returns `-1` for floret vectors. ~~int~~ | -## Vectors.most_similar {#most_similar tag="method"} +## Vectors.most_similar {id="most_similar",tag="method"} For each of the given vectors, find the `n` most similar entries to it by cosine. Queries are by vector. Results are returned as a @@ -356,7 +356,7 @@ supported for `floret` mode. | `sort` | Whether to sort the entries returned by score. Defaults to `True`. ~~bool~~ | | **RETURNS** | The most similar entries as a `(keys, best_rows, scores)` tuple. ~~Tuple[numpy.ndarray, numpy.ndarray, numpy.ndarray]~~ | -## Vectors.get_batch {#get_batch tag="method" new="3.2"} +## Vectors.get_batch {id="get_batch",tag="method",version="3.2"} Get the vectors for the provided keys efficiently as a batch. @@ -371,7 +371,7 @@ Get the vectors for the provided keys efficiently as a batch. | ------ | --------------------------------------- | | `keys` | The keys. ~~Iterable[Union[int, str]]~~ | -## Vectors.to_ops {#to_ops tag="method"} +## Vectors.to_ops {id="to_ops",tag="method"} Change the embedding matrix to use different Thinc ops. @@ -388,7 +388,7 @@ Change the embedding matrix to use different Thinc ops. | ----- | -------------------------------------------------------- | | `ops` | The Thinc ops to switch the embedding matrix to. ~~Ops~~ | -## Vectors.to_disk {#to_disk tag="method"} +## Vectors.to_disk {id="to_disk",tag="method"} Save the current state to a directory. @@ -403,7 +403,7 @@ Save the current state to a directory. | ------ | ------------------------------------------------------------------------------------------------------------------------------------------ | | `path` | A path to a directory, which will be created if it doesn't exist. Paths may be either strings or `Path`-like objects. ~~Union[str, Path]~~ | -## Vectors.from_disk {#from_disk tag="method"} +## Vectors.from_disk {id="from_disk",tag="method"} Loads state from a directory. Modifies the object in place and returns it. @@ -419,7 +419,7 @@ Loads state from a directory. Modifies the object in place and returns it. | `path` | A path to a directory. Paths may be either strings or `Path`-like objects. ~~Union[str, Path]~~ | | **RETURNS** | The modified `Vectors` object. ~~Vectors~~ | -## Vectors.to_bytes {#to_bytes tag="method"} +## Vectors.to_bytes {id="to_bytes",tag="method"} Serialize the current state to a binary string. @@ -433,7 +433,7 @@ Serialize the current state to a binary string. | ----------- | ------------------------------------------------------ | | **RETURNS** | The serialized form of the `Vectors` object. ~~bytes~~ | -## Vectors.from_bytes {#from_bytes tag="method"} +## Vectors.from_bytes {id="from_bytes",tag="method"} Load state from a binary string. @@ -451,7 +451,7 @@ Load state from a binary string. | `data` | The data to load from. ~~bytes~~ | | **RETURNS** | The `Vectors` object. ~~Vectors~~ | -## Attributes {#attributes} +## Attributes {id="attributes"} | Name | Description | | --------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------- | diff --git a/website/docs/api/vocab.md b/website/docs/api/vocab.mdx similarity index 93% rename from website/docs/api/vocab.md rename to website/docs/api/vocab.mdx index 5e4de219a..fe774d1a8 100644 --- a/website/docs/api/vocab.md +++ b/website/docs/api/vocab.mdx @@ -10,7 +10,14 @@ The `Vocab` object provides a lookup table that allows you to access [`StringStore`](/api/stringstore). It also owns underlying C-data that is shared between `Doc` objects. -## Vocab.\_\_init\_\_ {#init tag="method"} + + +Note that a `Vocab` instance is not static. It increases in size as texts with +new tokens are processed. + + + +## Vocab.\_\_init\_\_ {id="init",tag="method"} Create the vocabulary. @@ -31,7 +38,7 @@ Create the vocabulary. | `writing_system` | A dictionary describing the language's writing system. Typically provided by [`Language.Defaults`](/api/language#defaults). ~~Dict[str, Any]~~ | | `get_noun_chunks` | A function that yields base noun phrases used for [`Doc.noun_chunks`](/api/doc#noun_chunks). ~~Optional[Callable[[Union[Doc, Span], Iterator[Tuple[int, int, int]]]]]~~ | -## Vocab.\_\_len\_\_ {#len tag="method"} +## Vocab.\_\_len\_\_ {id="len",tag="method"} Get the current number of lexemes in the vocabulary. @@ -46,7 +53,7 @@ Get the current number of lexemes in the vocabulary. | ----------- | ------------------------------------------------ | | **RETURNS** | The number of lexemes in the vocabulary. ~~int~~ | -## Vocab.\_\_getitem\_\_ {#getitem tag="method"} +## Vocab.\_\_getitem\_\_ {id="getitem",tag="method"} Retrieve a lexeme, given an int ID or a string. If a previously unseen string is given, a new lexeme is created and stored. @@ -63,7 +70,7 @@ given, a new lexeme is created and stored. | `id_or_string` | The hash value of a word, or its string. ~~Union[int, str]~~ | | **RETURNS** | The lexeme indicated by the given ID. ~~Lexeme~~ | -## Vocab.\_\_iter\_\_ {#iter tag="method"} +## Vocab.\_\_iter\_\_ {id="iter",tag="method"} Iterate over the lexemes in the vocabulary. @@ -77,7 +84,7 @@ Iterate over the lexemes in the vocabulary. | ---------- | -------------------------------------- | | **YIELDS** | An entry in the vocabulary. ~~Lexeme~~ | -## Vocab.\_\_contains\_\_ {#contains tag="method"} +## Vocab.\_\_contains\_\_ {id="contains",tag="method"} Check whether the string has an entry in the vocabulary. To get the ID for a given string, you need to look it up in @@ -97,7 +104,7 @@ given string, you need to look it up in | `string` | The ID string. ~~str~~ | | **RETURNS** | Whether the string has an entry in the vocabulary. ~~bool~~ | -## Vocab.add_flag {#add_flag tag="method"} +## Vocab.add_flag {id="add_flag",tag="method"} Set a new boolean flag to words in the vocabulary. The `flag_getter` function will be called over the words currently in the vocab, and then applied to new @@ -122,7 +129,7 @@ using `token.check_flag(flag_id)`. | `flag_id` | An integer between `1` and `63` (inclusive), specifying the bit at which the flag will be stored. If `-1`, the lowest available bit will be chosen. ~~int~~ | | **RETURNS** | The integer ID by which the flag value can be checked. ~~int~~ | -## Vocab.reset_vectors {#reset_vectors tag="method" new="2"} +## Vocab.reset_vectors {id="reset_vectors",tag="method",version="2"} Drop the current vector table. Because all vectors must be the same width, you have to call this to change the size of the vectors. Only one of the `width` and @@ -140,7 +147,7 @@ have to call this to change the size of the vectors. Only one of the `width` and | `width` | The new width. ~~int~~ | | `shape` | The new shape. ~~int~~ | -## Vocab.prune_vectors {#prune_vectors tag="method" new="2"} +## Vocab.prune_vectors {id="prune_vectors",tag="method",version="2"} Reduce the current vector table to `nr_row` unique entries. Words mapped to the discarded vectors will be remapped to the closest vector among those remaining. @@ -165,7 +172,7 @@ cosines are calculated in minibatches to reduce memory usage. | `batch_size` | Batch of vectors for calculating the similarities. Larger batch sizes might be faster, while temporarily requiring more memory. ~~int~~ | | **RETURNS** | A dictionary keyed by removed words mapped to `(string, score)` tuples, where `string` is the entry the removed word was mapped to, and `score` the similarity score between the two words. ~~Dict[str, Tuple[str, float]]~~ | -## Vocab.deduplicate_vectors {#deduplicate_vectors tag="method" new="3.3"} +## Vocab.deduplicate_vectors {id="deduplicate_vectors",tag="method",version="3.3"} > #### Example > @@ -176,7 +183,7 @@ cosines are calculated in minibatches to reduce memory usage. Remove any duplicate rows from the current vector table, maintaining the mappings for all words in the vectors. -## Vocab.get_vector {#get_vector tag="method" new="2"} +## Vocab.get_vector {id="get_vector",tag="method",version="2"} Retrieve a vector for a word in the vocabulary. Words can be looked up by string or hash value. If the current vectors do not contain an entry for the word, a @@ -194,7 +201,7 @@ or hash value. If the current vectors do not contain an entry for the word, a | `orth` | The hash value of a word, or its unicode string. ~~Union[int, str]~~ | | **RETURNS** | A word vector. Size and shape are determined by the `Vocab.vectors` instance. ~~numpy.ndarray[ndim=1, dtype=float32]~~ | -## Vocab.set_vector {#set_vector tag="method" new="2"} +## Vocab.set_vector {id="set_vector",tag="method",version="2"} Set a vector for a word in the vocabulary. Words can be referenced by string or hash value. @@ -210,7 +217,7 @@ hash value. | `orth` | The hash value of a word, or its unicode string. ~~Union[int, str]~~ | | `vector` | The vector to set. ~~numpy.ndarray[ndim=1, dtype=float32]~~ | -## Vocab.has_vector {#has_vector tag="method" new="2"} +## Vocab.has_vector {id="has_vector",tag="method",version="2"} Check whether a word has a vector. Returns `False` if no vectors are loaded. Words can be looked up by string or hash value. @@ -227,7 +234,7 @@ Words can be looked up by string or hash value. | `orth` | The hash value of a word, or its unicode string. ~~Union[int, str]~~ | | **RETURNS** | Whether the word has a vector. ~~bool~~ | -## Vocab.to_disk {#to_disk tag="method" new="2"} +## Vocab.to_disk {id="to_disk",tag="method",version="2"} Save the current state to a directory. @@ -243,7 +250,7 @@ Save the current state to a directory. | _keyword-only_ | | | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | -## Vocab.from_disk {#from_disk tag="method" new="2"} +## Vocab.from_disk {id="from_disk",tag="method",version="2"} Loads state from a directory. Modifies the object in place and returns it. @@ -261,7 +268,7 @@ Loads state from a directory. Modifies the object in place and returns it. | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | | **RETURNS** | The modified `Vocab` object. ~~Vocab~~ | -## Vocab.to_bytes {#to_bytes tag="method"} +## Vocab.to_bytes {id="to_bytes",tag="method"} Serialize the current state to a binary string. @@ -277,7 +284,7 @@ Serialize the current state to a binary string. | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | | **RETURNS** | The serialized form of the `Vocab` object. ~~Vocab~~ | -## Vocab.from_bytes {#from_bytes tag="method"} +## Vocab.from_bytes {id="from_bytes",tag="method"} Load state from a binary string. @@ -297,7 +304,7 @@ Load state from a binary string. | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | | **RETURNS** | The `Vocab` object. ~~Vocab~~ | -## Attributes {#attributes} +## Attributes {id="attributes"} > #### Example > @@ -317,7 +324,7 @@ Load state from a binary string. | `writing_system` | A dict with information about the language's writing system. ~~Dict[str, Any]~~ | | `get_noun_chunks` 3.0 | A function that yields base noun phrases used for [`Doc.noun_chunks`](/api/doc#noun_chunks). ~~Optional[Callable[[Union[Doc, Span], Iterator[Tuple[int, int, int]]]]]~~ | -## Serialization fields {#serialization-fields} +## Serialization fields {id="serialization-fields"} During serialization, spaCy will export several data fields used to restore different aspects of the object. If needed, you can exclude them from diff --git a/website/docs/images/displacy-dep-founded.html b/website/docs/images/displacy-dep-founded.html deleted file mode 100644 index e22984ee1..000000000 --- a/website/docs/images/displacy-dep-founded.html +++ /dev/null @@ -1,58 +0,0 @@ - - - Smith - - - - - founded - - - - - a - - - - - healthcare - - - - - company - - - - - - - nsubj - - - - - - - - det - - - - - - - - compound - - - - - - - - dobj - - - - diff --git a/website/docs/images/displacy-ent-custom.html b/website/docs/images/displacy-ent-custom.html deleted file mode 100644 index 709c6f631..000000000 --- a/website/docs/images/displacy-ent-custom.html +++ /dev/null @@ -1,33 +0,0 @@ -
But - Google - ORGis starting from behind. The company made a late push into hardware, and - Apple - ORG’s Siri, available on iPhones, and - Amazon - ORG’s Alexa software, which runs on its Echo and Dot devices, have clear leads in consumer - adoption.
diff --git a/website/docs/images/displacy-ent-snek.html b/website/docs/images/displacy-ent-snek.html deleted file mode 100644 index c8b416d8d..000000000 --- a/website/docs/images/displacy-ent-snek.html +++ /dev/null @@ -1,26 +0,0 @@ -
- 🌱🌿 - 🐍 - SNEK - ____ 🌳🌲 ____ - 👨‍🌾 - HUMAN - 🏘️ -
diff --git a/website/docs/images/displacy-ent1.html b/website/docs/images/displacy-ent1.html deleted file mode 100644 index 708df8093..000000000 --- a/website/docs/images/displacy-ent1.html +++ /dev/null @@ -1,37 +0,0 @@ -
- - Apple - ORG - - is looking at buying - - U.K. - GPE - - startup for - - $1 billion - MONEY - -
diff --git a/website/docs/images/displacy-ent2.html b/website/docs/images/displacy-ent2.html deleted file mode 100644 index 5e1833ca0..000000000 --- a/website/docs/images/displacy-ent2.html +++ /dev/null @@ -1,39 +0,0 @@ -
- When - - Sebastian Thrun - PERSON - - started working on self-driving cars at - - Google - ORG - - in - - 2007 - DATE - - , few people outside of the company took him seriously. -
diff --git a/website/docs/images/displacy-long2.html b/website/docs/images/displacy-long2.html deleted file mode 100644 index abe18c42a..000000000 --- a/website/docs/images/displacy-long2.html +++ /dev/null @@ -1,84 +0,0 @@ - - - Autonomous - ADJ - - - - cars - NOUN - - - - shift - VERB - - - - insurance - NOUN - - - - liability - NOUN - - - - toward - ADP - - - - manufacturers - NOUN - - - - - - amod - - - - - - - - nsubj - - - - - - - - compound - - - - - - - - dobj - - - - - - - - prep - - - - - - - - pobj - - - - diff --git a/website/docs/images/displacy-span-custom.html b/website/docs/images/displacy-span-custom.html deleted file mode 100644 index 97dd3b140..000000000 --- a/website/docs/images/displacy-span-custom.html +++ /dev/null @@ -1,31 +0,0 @@ -
- Welcome to the - - Bank - - - - - BANK - - - - - of - - - - - China - - - - - . -
\ No newline at end of file diff --git a/website/docs/images/displacy-span.html b/website/docs/images/displacy-span.html deleted file mode 100644 index 9bbc6403c..000000000 --- a/website/docs/images/displacy-span.html +++ /dev/null @@ -1,41 +0,0 @@ -
- Welcome to the - - Bank - - - - - ORG - - - - - of - - - - - - China - - - - - - - GPE - - - - . -
\ No newline at end of file diff --git a/website/docs/index.md b/website/docs/index.md deleted file mode 100644 index 48e487d08..000000000 --- a/website/docs/index.md +++ /dev/null @@ -1,6 +0,0 @@ ---- ---- - -import Landing from 'widgets/landing.js' - - diff --git a/website/docs/models/index.md b/website/docs/models/index.mdx similarity index 94% rename from website/docs/models/index.md rename to website/docs/models/index.mdx index 203555651..366d44f0e 100644 --- a/website/docs/models/index.md +++ b/website/docs/models/index.mdx @@ -7,7 +7,7 @@ menu: - ['Pipeline Design', 'design'] --- - +{/* TODO: include interactive demo */} ### Quickstart {hidden="true"} @@ -16,15 +16,13 @@ menu: > For more details on how to use trained pipelines with spaCy, see the > [usage guide](/usage/models). -import QuickstartModels from 'widgets/quickstart-models.js' - -## Package naming conventions {#conventions} +## Package naming conventions {id="conventions"} In general, spaCy expects all pipeline packages to follow the naming convention -of `[lang]\_[name]`. For spaCy's pipelines, we also chose to divide the name -into three components: +of `[lang]_[name]`. For spaCy's pipelines, we also chose to divide the name into +three components: 1. **Type:** Capabilities (e.g. `core` for general-purpose pipeline with tagging, parsing, lemmatization and named entity recognition, or `dep` for @@ -45,7 +43,7 @@ For example, [`en_core_web_sm`](/models/en#en_core_web_sm) is a small English pipeline trained on written web text (blogs, news, comments), that includes vocabulary, syntax and entities. -### Package versioning {#model-versioning} +### Package versioning {id="model-versioning"} Additionally, the pipeline package versioning reflects both the compatibility with spaCy, as well as the model version. A package version `a.b.c` translates @@ -62,7 +60,7 @@ For a detailed compatibility overview, see the This is also the source of spaCy's internal compatibility check, performed when you run the [`download`](/api/cli#download) command. -## Trained pipeline design {#design} +## Trained pipeline design {id="design"} The spaCy v3 trained pipelines are designed to be efficient and configurable. For example, multiple components can share a common "token-to-vector" model and @@ -89,9 +87,9 @@ Main changes from spaCy v2 models: - The lemmatizer tables and processing move from the vocab and tagger to a separate `lemmatizer` component. -### CNN/CPU pipeline design {#design-cnn} +### CNN/CPU pipeline design {id="design-cnn"} -![Components and their dependencies in the CNN pipelines](../images/pipeline-design.svg) +![Components and their dependencies in the CNN pipelines](/images/pipeline-design.svg) In the `sm`/`md`/`lg` models: @@ -132,13 +130,13 @@ vector keys for default vectors. - [`Vectors.most_similar`](/api/vectors#most_similar) is not supported because there's no fixed list of vectors to compare your vectors to. -### Transformer pipeline design {#design-trf} +### Transformer pipeline design {id="design-trf"} In the transformer (`trf`) models, the `tagger`, `parser` and `ner` (if present) all listen to the `transformer` component. The `attribute_ruler` and `lemmatizer` have the same configuration as in the CNN models. -### Modifying the default pipeline {#design-modify} +### Modifying the default pipeline {id="design-modify"} For faster processing, you may only want to run a subset of the components in a trained pipeline. The `disable` and `exclude` arguments to @@ -189,8 +187,8 @@ than the rule-based `sentencizer`. #### Switch from trainable lemmatizer to default lemmatizer -Since v3.3, a number of pipelines use a trainable lemmatizer. You can check whether -the lemmatizer is trainable: +Since v3.3, a number of pipelines use a trainable lemmatizer. You can check +whether the lemmatizer is trainable: ```python nlp = spacy.load("de_core_web_sm") diff --git a/website/docs/styleguide.md b/website/docs/styleguide.mdx similarity index 86% rename from website/docs/styleguide.md rename to website/docs/styleguide.mdx index 47bca1ed4..276137aab 100644 --- a/website/docs/styleguide.md +++ b/website/docs/styleguide.mdx @@ -42,9 +42,7 @@ enough, JSX components can be used. > For more details on editing the site locally, see the installation > instructions and markdown reference below. -## Logo {#logo source="website/src/images/logo.svg"} - -import { Logos } from 'widgets/styleguide' +## Logo {id="logo",source="website/src/images/logo.svg"} If you would like to use the spaCy logo on your site, please get in touch and ask us first. However, if you want to show support and tell others that your @@ -53,9 +51,7 @@ project is using spaCy, you can grab one of our -## Colors {#colors} - -import { Colors, Patterns } from 'widgets/styleguide' +## Colors {id="colors"} @@ -63,17 +59,16 @@ import { Colors, Patterns } from 'widgets/styleguide' -## Typography {#typography} - -import { H1, H2, H3, H4, H5, Label, InlineList, Comment } from -'components/typography' +## Typography {id="typography"} > #### Markdown > -> ```markdown_ +> ```markdown > ## Headline 2 -> ## Headline 2 {#some_id} -> ## Headline 2 {#some_id tag="method"} +> +> ## Headline 2 {id="some_id"} +> +> ## Headline 2 {id="some_id" tag="method"} > ``` > > #### JSX @@ -101,12 +96,11 @@ in the sidebar menu.
-

Headline 1

-

Headline 2

-

Headline 3

-

Headline 4

-
Headline 5
- +

Headline 2

+

Headline 3

+

Headline 4

+
Headline 5
+
--- @@ -116,16 +110,16 @@ example, to add a tag for the documented type or mark features that have been introduced in a specific version or require statistical models to be loaded. Tags are also available as standalone `` components. -| Argument | Example | Result | -| -------- | -------------------------- | ----------------------------------------- | -| `tag` | `{tag="method"}` | method | -| `new` | `{new="3"}` | 3 | -| `model` | `{model="tagger, parser"}` | tagger, parser | -| `hidden` | `{hidden="true"}` | | +| Argument | Example | Result | +| --------- | -------------------------- | ----------------------------------------- | +| `tag` | `{tag="method"}` | method | +| `version` | `{version="3"}` | 3 | +| `model` | `{model="tagger, parser"}` | tagger, parser | +| `hidden` | `{hidden="true"}` | | -## Elements {#elements} +## Elements {id="elements"} -### Links {#links} +### Links {id="links"} > #### Markdown > @@ -147,9 +141,7 @@ Special link styles are used depending on the link URL. - [I am a link to a model](/models/en#en_core_web_sm) - [I am a link to GitHub](https://github.com/explosion/spaCy) -### Abbreviations {#abbr} - -import { Abbr } from 'components/typography' +### Abbreviations {id="abbr"} > #### JSX > @@ -161,13 +153,11 @@ Some text with an abbreviation. On small screens, I collapse and the explanation text is displayed next to the abbreviation. -### Tags {#tags} - -import Tag from 'components/tag' +### Tags {id="tags"} > ```jsx > method -> 4 +> 4 > tagger, parser > ``` @@ -180,16 +170,13 @@ new anymore. Setting `variant="model"` takes a description of model capabilities and can be used to mark features that require a respective model to be installed. - +

+ method + 4 + tagger, parser +

-method 4 tagger, -parser - -
- -### Buttons {#buttons} - -import Button from 'components/button' +### Buttons {id="buttons"} > ```jsx > @@ -200,21 +187,29 @@ Link buttons come in two variants, `primary` and `secondary` and two sizes, with an optional `large` size modifier. Since they're mostly used as enhanced links, the buttons are implemented as styled links instead of native button elements. - - +

+ -
+{' '} - - + +

+ +

+ + +{' '} + + +

## Components -### Table {#table} +### Table {id="table"} > #### Markdown > -> ```markdown_ +> ```markdown > | Header 1 | Header 2 | > | -------- | -------- | > | Column 1 | Column 2 | @@ -248,7 +243,7 @@ be italicized: > #### Markdown > -> ```markdown_ +> ```markdown > | Header 1 | Header 2 | Header 3 | > | -------- | -------- | -------- | > | Column 1 | Column 2 | Column 3 | @@ -262,11 +257,11 @@ be italicized: | _Hello_ | | | | Column 1 | Column 2 | Column 3 | -### Type Annotations {#type-annotations} +### Type Annotations {id="type-annotations"} > #### Markdown > -> ```markdown_ +> ```markdown > ~~Model[List[Doc], Floats2d]~~ > ``` > @@ -295,9 +290,9 @@ always be the **last element** in the row. > #### Markdown > -> ```markdown_ -> | Header 1 | Header 2 | -> | -------- | ----------------------- | +> ```markdown +> | Header 1 | Header 2 | +> | -------- | ---------------------- | > | Column 1 | Column 2 ~~List[Doc]~~ | > ``` @@ -307,11 +302,11 @@ always be the **last element** in the row. | `model` | The Thinc [`Model`](https://thinc.ai/docs/api-model) wrapping the transformer. ~~Model[List[Doc], FullTransformerBatch]~~ | | `set_extra_annotations` | Function that takes a batch of `Doc` objects and transformer outputs and can set additional annotations on the `Doc`. ~~Callable[[List[Doc], FullTransformerBatch], None]~~ | -### List {#list} +### List {id="list"} > #### Markdown > -> ```markdown_ +> ```markdown > 1. One > 2. Two > ``` @@ -338,12 +333,13 @@ automatically. 3. Lorem ipsum dolor 4. consectetur adipiscing elit -### Aside {#aside} +### Aside {id="aside"} > #### Markdown > -> ```markdown_ +> ```markdown > > #### Aside title +> > > > This is aside text. > ``` > @@ -363,11 +359,11 @@ To make them easier to use in Markdown, paragraphs formatted as blockquotes will turn into asides by default. Level 4 headlines (with a leading `####`) will become aside titles. -### Code Block {#code-block} +### Code Block {id="code-block"} > #### Markdown > -> ````markdown_ +> ````markdown > ```python > ### This is a title > import spacy @@ -388,8 +384,7 @@ to raw text with no highlighting. An optional label can be added as the first line with the prefix `####` (Python-like) and `///` (JavaScript-like). the indented block as plain text and preserve whitespace. -```python -### Using spaCy +```python {title="Using spaCy"} import spacy nlp = spacy.load("en_core_web_sm") doc = nlp("This is a sentence.") @@ -403,7 +398,7 @@ adding `{highlight="..."}` to the headline. Acceptable ranges are spans like > #### Markdown > -> ````markdown_ +> ````markdown > ```python > ### This is a title {highlight="1-2"} > import spacy @@ -411,8 +406,7 @@ adding `{highlight="..."}` to the headline. Acceptable ranges are spans like > ``` > ```` -```python -### Using the matcher {highlight="5-7"} +```python {title="Using the matcher",highlight="5-7"} import spacy from spacy.matcher import Matcher @@ -431,7 +425,7 @@ interactive widget defaults to a regular code block. > #### Markdown > -> ````markdown_ +> ````markdown > ```python > ### {executable="true"} > import spacy @@ -439,8 +433,7 @@ interactive widget defaults to a regular code block. > ``` > ```` -```python -### {executable="true"} +```python {executable="true"} import spacy nlp = spacy.load("en_core_web_sm") doc = nlp("This is a sentence.") @@ -454,7 +447,7 @@ original file is shown at the top of the widget. > #### Markdown > -> ````markdown_ +> ````markdown > ```python > https://github.com/... > ``` @@ -470,9 +463,7 @@ original file is shown at the top of the widget. https://github.com/explosion/spaCy/tree/master/spacy/language.py ``` -### Infobox {#infobox} - -import Infobox from 'components/infobox' +### Infobox {id="infobox"} > #### JSX > @@ -508,9 +499,7 @@ blocks. -### Accordion {#accordion} - -import Accordion from 'components/accordion' +### Accordion {id="accordion"} > #### JSX > @@ -537,9 +526,9 @@ sit amet dignissim justo congue. -## Markdown reference {#markdown} +## Markdown reference {id="markdown"} -All page content and page meta lives in the `.md` files in the `/docs` +All page content and page meta lives in the `.mdx` files in the `/docs` directory. The frontmatter block at the top of each file defines the page title and other settings like the sidebar menu. @@ -548,7 +537,7 @@ and other settings like the sidebar menu. title: Page title --- -## Headline starting a section {#some_id} +## Headline starting a section {id="some_id"} This is a regular paragraph with a [link](https://spacy.io) and **bold text**. @@ -562,8 +551,7 @@ This is a regular paragraph with a [link](https://spacy.io) and **bold text**. | -------- | -------- | | Column 1 | Column 2 | -```python -### Code block title {highlight="2-3"} +```python {title="Code block title",highlight="2-3"} import spacy nlp = spacy.load("en_core_web_sm") doc = nlp("Hello world") @@ -585,7 +573,7 @@ In addition to the native markdown elements, you can use the components [abbr]: https://spacy.io/styleguide#abbr [tag]: https://spacy.io/styleguide#tag -## Editorial {#editorial} +## Editorial {id="editorial"} - "spaCy" should always be spelled with a lowercase "s" and a capital "C", unless it specifically refers to the Python package or Python import `spacy` @@ -609,21 +597,16 @@ In addition to the native markdown elements, you can use the components - ❌ The [`Span`](/api/span) and [`Token`](/api/token) objects are views of a [`Doc`](/api/doc). [`Span.as_doc`](/api/span#as_doc) creates a [`Doc`](/api/doc) object from a [`Span`](/api/span). - -* Other things we format as code are: references to trained pipeline packages +- Other things we format as code are: references to trained pipeline packages like `en_core_web_sm` or file names like `code.py` or `meta.json`. - - ✅ After training, the `config.cfg` is saved to disk. - -* [Type annotations](#type-annotations) are a special type of code formatting, +- [Type annotations](#type-annotations) are a special type of code formatting, expressed by wrapping the text in `~~` instead of backticks. The result looks like this: ~~List[Doc]~~. All references to known types will be linked automatically. - - ✅ The model has the input type ~~List[Doc]~~ and it outputs a ~~List[Array2d]~~. - -* We try to keep links meaningful but short. +- We try to keep links meaningful but short. - ✅ For details, see the usage guide on [training with custom code](/usage/training#custom-code). - ❌ For details, see diff --git a/website/docs/usage/101/_architecture.md b/website/docs/usage/101/_architecture.mdx similarity index 96% rename from website/docs/usage/101/_architecture.md rename to website/docs/usage/101/_architecture.mdx index 4ebca2756..2a63a3741 100644 --- a/website/docs/usage/101/_architecture.md +++ b/website/docs/usage/101/_architecture.mdx @@ -14,9 +14,9 @@ of the pipeline. The `Language` object coordinates these components. It takes raw text and sends it through the pipeline, returning an **annotated document**. It also orchestrates training and serialization. -![Library architecture](../../images/architecture.svg) +![Library architecture {{w:1080, h:1254}}](/images/architecture.svg) -### Container objects {#architecture-containers} +### Container objects {id="architecture-containers"} | Name | Description | | ----------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------- | @@ -29,7 +29,7 @@ It also orchestrates training and serialization. | [`SpanGroup`](/api/spangroup) | A named collection of spans belonging to a `Doc`. | | [`Token`](/api/token) | An individual token — i.e. a word, punctuation symbol, whitespace, etc. | -### Processing pipeline {#architecture-pipeline} +### Processing pipeline {id="architecture-pipeline"} The processing pipeline consists of one or more **pipeline components** that are called on the `Doc` in order. The tokenizer runs before the components. Pipeline @@ -39,7 +39,7 @@ rule-based modifications to the `Doc`. spaCy provides a range of built-in components for different language processing tasks and also allows adding [custom components](/usage/processing-pipelines#custom-components). -![The processing pipeline](../../images/pipeline.svg) +![The processing pipeline](/images/pipeline.svg) | Name | Description | | ----------------------------------------------- | ------------------------------------------------------------------------------------------- | @@ -61,7 +61,7 @@ components for different language processing tasks and also allows adding | [`Transformer`](/api/transformer) | Use a transformer model and set its outputs. | | [Other functions](/api/pipeline-functions) | Automatically apply something to the `Doc`, e.g. to merge spans of tokens. | -### Matchers {#architecture-matchers} +### Matchers {id="architecture-matchers"} Matchers help you find and extract information from [`Doc`](/api/doc) objects based on match patterns describing the sequences you're looking for. A matcher @@ -73,13 +73,13 @@ operates on a `Doc` and gives you access to the matched tokens **in context**. | [`Matcher`](/api/matcher) | Match sequences of tokens, based on pattern rules, similar to regular expressions. | | [`PhraseMatcher`](/api/phrasematcher) | Match sequences of tokens based on phrases. | -### Other classes {#architecture-other} +### Other classes {id="architecture-other"} | Name | Description | | ------------------------------------------------ | -------------------------------------------------------------------------------------------------- | | [`Corpus`](/api/corpus) | Class for managing annotated corpora for training and evaluation data. | | [`KnowledgeBase`](/api/kb) | Abstract base class for storage and retrieval of data for entity linking. | -| [`InMemoryLookupKB`](/api/kb_in_memory) | Implementation of `KnowledgeBase` storing all data in memory. | +| [`InMemoryLookupKB`](/api/inmemorylookupkb) | Implementation of `KnowledgeBase` storing all data in memory. | | [`Candidate`](/api/kb#candidate) | Object associating a textual mention with a specific entity contained in a `KnowledgeBase`. | | [`Lookups`](/api/lookups) | Container for convenient access to large lookup tables and dictionaries. | | [`MorphAnalysis`](/api/morphology#morphanalysis) | A morphological analysis. | diff --git a/website/docs/usage/101/_language-data.md b/website/docs/usage/101/_language-data.mdx similarity index 100% rename from website/docs/usage/101/_language-data.md rename to website/docs/usage/101/_language-data.mdx diff --git a/website/docs/usage/101/_named-entities.md b/website/docs/usage/101/_named-entities.mdx similarity index 75% rename from website/docs/usage/101/_named-entities.md rename to website/docs/usage/101/_named-entities.mdx index 2abc45cbd..9ae4134d8 100644 --- a/website/docs/usage/101/_named-entities.md +++ b/website/docs/usage/101/_named-entities.mdx @@ -1,14 +1,13 @@ A named entity is a "real-world object" that's assigned a name – for example, a person, a country, a product or a book title. spaCy can **recognize various -types of named entities in a document, by asking the model for a -prediction**. Because models are statistical and strongly depend on the -examples they were trained on, this doesn't always work _perfectly_ and might -need some tuning later, depending on your use case. +types of named entities in a document, by asking the model for a prediction**. +Because models are statistical and strongly depend on the examples they were +trained on, this doesn't always work _perfectly_ and might need some tuning +later, depending on your use case. Named entities are available as the `ents` property of a `Doc`: -```python -### {executable="true"} +```python {executable="true"} import spacy nlp = spacy.load("en_core_web_sm") @@ -32,7 +31,8 @@ for ent in doc.ents: Using spaCy's built-in [displaCy visualizer](/usage/visualizers), here's what our example sentence and its named entities look like: -import DisplaCyEntHtml from 'images/displacy-ent1.html'; import { Iframe } from -'components/embed' - -