mirror of
https://github.com/explosion/spaCy.git
synced 2025-08-08 06:04:57 +03:00
Merge branch 'master' into spacy.io
This commit is contained in:
Adriane Boyd
commit
f85d00f11b
2
.github/ISSUE_TEMPLATE/01_bugs.md
vendored
2
.github/ISSUE_TEMPLATE/01_bugs.md
vendored
|
|
@ -10,7 +10,7 @@ about: Use this template if you came across a bug or unexpected behaviour differ
|
|||
<!-- Include a code example or the steps that led to the problem. Please try to be as specific as possible. -->
|
||||
|
||||
## Your Environment
|
||||
<!-- Include details of your environment. If you're using spaCy 1.7+, you can also type `python -m spacy info --markdown` and copy-paste the result here.-->
|
||||
<!-- Include details of your environment. You can also type `python -m spacy info --markdown` and copy-paste the result here.-->
|
||||
* Operating System:
|
||||
* Python Version Used:
|
||||
* spaCy Version Used:
|
||||
|
|
|
|||
6
.github/azure-steps.yml
vendored
6
.github/azure-steps.yml
vendored
|
|
@ -10,6 +10,7 @@ steps:
|
|||
inputs:
|
||||
versionSpec: ${{ parameters.python_version }}
|
||||
architecture: ${{ parameters.architecture }}
|
||||
allowUnstable: true
|
||||
|
||||
- bash: |
|
||||
echo "##vso[task.setvariable variable=python_version]${{ parameters.python_version }}"
|
||||
|
|
@ -27,6 +28,7 @@ steps:
|
|||
|
||||
- script: python -m mypy spacy
|
||||
displayName: 'Run mypy'
|
||||
condition: ne(variables['python_version'], '3.6')
|
||||
|
||||
- task: DeleteFiles@1
|
||||
inputs:
|
||||
|
|
@ -54,12 +56,12 @@ steps:
|
|||
condition: eq(${{ parameters.gpu }}, true)
|
||||
|
||||
- script: |
|
||||
${{ parameters.prefix }} python -m pytest --pyargs spacy
|
||||
${{ parameters.prefix }} python -m pytest --pyargs spacy -W error
|
||||
displayName: "Run CPU tests"
|
||||
condition: eq(${{ parameters.gpu }}, false)
|
||||
|
||||
- script: |
|
||||
${{ parameters.prefix }} python -m pytest --pyargs spacy -p spacy.tests.enable_gpu
|
||||
${{ parameters.prefix }} python -m pytest --pyargs spacy -W error -p spacy.tests.enable_gpu
|
||||
displayName: "Run GPU tests"
|
||||
condition: eq(${{ parameters.gpu }}, true)
|
||||
|
||||
|
|
|
|||
13
.github/no-response.yml
vendored
13
.github/no-response.yml
vendored
|
|
@ -1,13 +0,0 @@
|
|||
# Configuration for probot-no-response - https://github.com/probot/no-response
|
||||
|
||||
# Number of days of inactivity before an Issue is closed for lack of response
|
||||
daysUntilClose: 14
|
||||
# Label requiring a response
|
||||
responseRequiredLabel: more-info-needed
|
||||
# Comment to post when closing an Issue for lack of response. Set to `false` to disable
|
||||
closeComment: >
|
||||
This issue has been automatically closed because there has been no response
|
||||
to a request for more information from the original author. With only the
|
||||
information that is currently in the issue, there's not enough information
|
||||
to take action. If you're the original author, feel free to reopen the issue
|
||||
if you have or find the answers needed to investigate further.
|
||||
8
.github/workflows/issue-manager.yml
vendored
8
.github/workflows/issue-manager.yml
vendored
|
|
@ -15,7 +15,7 @@ jobs:
|
|||
issue-manager:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: tiangolo/issue-manager@0.2.1
|
||||
- uses: tiangolo/issue-manager@0.4.0
|
||||
with:
|
||||
token: ${{ secrets.GITHUB_TOKEN }}
|
||||
config: >
|
||||
|
|
@ -25,5 +25,11 @@ jobs:
|
|||
"message": "This issue has been automatically closed because it was answered and there was no follow-up discussion.",
|
||||
"remove_label_on_comment": true,
|
||||
"remove_label_on_close": true
|
||||
},
|
||||
"more-info-needed": {
|
||||
"delay": "P7D",
|
||||
"message": "This issue has been automatically closed because there has been no response to a request for more information from the original author. With only the information that is currently in the issue, there's not enough information to take action. If you're the original author, feel free to reopen the issue if you have or find the answers needed to investigate further.",
|
||||
"remove_label_on_comment": true,
|
||||
"remove_label_on_close": true
|
||||
}
|
||||
}
|
||||
|
|
|
|||
1
.gitignore
vendored
1
.gitignore
vendored
|
|
@ -24,6 +24,7 @@ quickstart-training-generator.js
|
|||
cythonize.json
|
||||
spacy/*.html
|
||||
*.cpp
|
||||
*.c
|
||||
*.so
|
||||
|
||||
# Vim / VSCode / editors
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ repos:
|
|||
language_version: python3.7
|
||||
additional_dependencies: ['click==8.0.4']
|
||||
- repo: https://gitlab.com/pycqa/flake8
|
||||
rev: 3.9.2
|
||||
rev: 5.0.4
|
||||
hooks:
|
||||
- id: flake8
|
||||
args:
|
||||
|
|
|
|||
|
|
@ -31,8 +31,8 @@ jobs:
|
|||
inputs:
|
||||
versionSpec: "3.7"
|
||||
- script: |
|
||||
pip install flake8==3.9.2
|
||||
python -m flake8 spacy --count --select=E901,E999,F821,F822,F823 --show-source --statistics
|
||||
pip install flake8==5.0.4
|
||||
python -m flake8 spacy --count --select=E901,E999,F821,F822,F823,W605 --show-source --statistics
|
||||
displayName: "flake8"
|
||||
|
||||
- job: "Test"
|
||||
|
|
@ -85,6 +85,15 @@ jobs:
|
|||
Python310Mac:
|
||||
imageName: "macos-latest"
|
||||
python.version: "3.10"
|
||||
Python311Linux:
|
||||
imageName: 'ubuntu-latest'
|
||||
python.version: '3.11.0-rc.2'
|
||||
Python311Windows:
|
||||
imageName: 'windows-latest'
|
||||
python.version: '3.11.0-rc.2'
|
||||
Python311Mac:
|
||||
imageName: 'macos-latest'
|
||||
python.version: '3.11.0-rc.2'
|
||||
maxParallel: 4
|
||||
pool:
|
||||
vmImage: $(imageName)
|
||||
|
|
|
|||
|
|
@ -191,6 +191,8 @@ def load_model(name: str) -> "Language":
|
|||
...
|
||||
```
|
||||
|
||||
Note that we typically put the `from typing` import statements on the first line(s) of the Python module.
|
||||
|
||||
## Structuring logic
|
||||
|
||||
### Positional and keyword arguments
|
||||
|
|
@ -275,6 +277,27 @@ If you have to use `try`/`except`, make sure to only include what's **absolutely
|
|||
+ return [v.strip() for v in value.split(",")]
|
||||
```
|
||||
|
||||
### Numeric comparisons
|
||||
|
||||
For numeric comparisons, as a general rule we always use `<` and `>=` and avoid the usage of `<=` and `>`. This is to ensure we consistently
|
||||
apply inclusive lower bounds and exclusive upper bounds, helping to prevent off-by-one errors.
|
||||
|
||||
One exception to this rule is the ternary case. With a chain like
|
||||
|
||||
```python
|
||||
if value >= 0 and value < max:
|
||||
...
|
||||
```
|
||||
|
||||
it's fine to rewrite this to the shorter form
|
||||
|
||||
```python
|
||||
if 0 <= value < max:
|
||||
...
|
||||
```
|
||||
|
||||
even though this requires the usage of the `<=` operator.
|
||||
|
||||
### Iteration and comprehensions
|
||||
|
||||
We generally avoid using built-in functions like `filter` or `map` in favor of list or generator comprehensions.
|
||||
|
|
|
|||
|
|
@ -36,7 +36,7 @@ Some things to note:
|
|||
@explosion-bot please test_gpu --run-on spacy-transformers --run-on-branch master --spacy-branch current_pr
|
||||
```
|
||||
|
||||
This will launch the GPU pipeline for the `spacy-transformers` repo on its `master` branch, using the current spaCy PR's branch to build spaCy.
|
||||
This will launch the GPU pipeline for the `spacy-transformers` repo on its `master` branch, using the current spaCy PR's branch to build spaCy. The name of the repository passed to `--run-on` is case-sensitive, e.g: use `spaCy` instead of `spacy`.
|
||||
|
||||
- General info about supported commands.
|
||||
|
||||
|
|
|
|||
82
extra/DEVELOPER_DOCS/Satellite Packages.md
Normal file
82
extra/DEVELOPER_DOCS/Satellite Packages.md
Normal file
|
|
@ -0,0 +1,82 @@
|
|||
# spaCy Satellite Packages
|
||||
|
||||
This is a list of all the active repos relevant to spaCy besides the main one, with short descriptions, history, and current status. Archived repos will not be covered.
|
||||
|
||||
## Always Included in spaCy
|
||||
|
||||
These packages are always pulled in when you install spaCy. Most of them are direct dependencies, but some are transitive dependencies through other packages.
|
||||
|
||||
- [spacy-legacy](https://github.com/explosion/spacy-legacy): When an architecture in spaCy changes enough to get a new version, the old version is frozen and moved to spacy-legacy. This allows us to keep the core library slim while also preserving backwards compatability.
|
||||
- [thinc](https://github.com/explosion/thinc): Thinc is the machine learning library that powers trainable components in spaCy. It wraps backends like Numpy, PyTorch, and Tensorflow to provide a functional interface for specifying architectures.
|
||||
- [catalogue](https://github.com/explosion/catalogue): Small library for adding function registries, like those used for model architectures in spaCy.
|
||||
- [confection](https://github.com/explosion/confection): This library contains the functionality for config parsing that was formerly contained directly in Thinc.
|
||||
- [spacy-loggers](https://github.com/explosion/spacy-loggers): Contains loggers beyond the default logger available in spaCy's core code base. This includes loggers integrated with third-party services, which may differ in release cadence from spaCy itself.
|
||||
- [wasabi](https://github.com/explosion/wasabi): A command line formatting library, used for terminal output in spaCy.
|
||||
- [srsly](https://github.com/explosion/srsly): A wrapper that vendors several serialization libraries for spaCy. Includes parsers for JSON, JSONL, MessagePack, (extended) Pickle, and YAML.
|
||||
- [preshed](https://github.com/explosion/preshed): A Cython library for low-level data structures like hash maps, used for memory efficient data storage.
|
||||
- [cython-blis](https://github.com/explosion/cython-blis): Fast matrix multiplication using BLIS without depending on system libraries. Required by Thinc, rather than spaCy directly.
|
||||
- [murmurhash](https://github.com/explosion/murmurhash): A wrapper library for a C++ murmurhash implementation, used for string IDs in spaCy and preshed.
|
||||
- [cymem](https://github.com/explosion/cymem): A small library for RAII-style memory management in Cython.
|
||||
|
||||
## Optional Extensions for spaCy
|
||||
|
||||
These are repos that can be used by spaCy but aren't part of a default installation. Many of these are wrappers to integrate various kinds of third-party libraries.
|
||||
|
||||
- [spacy-transformers](https://github.com/explosion/spacy-transformers): A wrapper for the [HuggingFace Transformers](https://huggingface.co/docs/transformers/index) library, this handles the extensive conversion necessary to coordinate spaCy's powerful `Doc` representation, training pipeline, and the Transformer embeddings. When released, this was known as `spacy-pytorch-transformers`, but it changed to the current name when HuggingFace update the name of their library as well.
|
||||
- [spacy-huggingface-hub](https://github.com/explosion/spacy-huggingface-hub): This package has a CLI script for uploading a packaged spaCy pipeline (created with `spacy package`) to the [Hugging Face Hub](https://huggingface.co/models).
|
||||
- [spacy-alignments](https://github.com/explosion/spacy-alignments): A wrapper for the tokenizations library (mentioned below) with a modified build system to simplify cross-platform wheel creation. Used in spacy-transformers for aligning spaCy and HuggingFace tokenizations.
|
||||
- [spacy-experimental](https://github.com/explosion/spacy-experimental): Experimental components that are not quite ready for inclusion in the main spaCy library. Usually there are unresolved questions around their APIs, so the experimental library allows us to expose them to the community for feedback before fully integrating them.
|
||||
- [spacy-lookups-data](https://github.com/explosion/spacy-lookups-data): A repository of linguistic data, such as lemmas, that takes up a lot of disk space. Originally created to reduce the size of the spaCy core library. This is mainly useful if you want the data included but aren't using a pretrained pipeline; for the affected languages, the relevant data is included in pretrained pipelines directly.
|
||||
- [coreferee](https://github.com/explosion/coreferee): Coreference resolution for English, French, German and Polish, optimised for limited training data and easily extensible for further languages. Used as a spaCy pipeline component.
|
||||
- [spacy-stanza](https://github.com/explosion/spacy-stanza): This is a wrapper that allows the use of Stanford's Stanza library in spaCy.
|
||||
- [spacy-streamlit](https://github.com/explosion/spacy-streamlit): A wrapper for the Streamlit dashboard building library to help with integrating [displaCy](https://spacy.io/api/top-level/#displacy).
|
||||
- [spacymoji](https://github.com/explosion/spacymoji): A library to add extra support for emoji to spaCy, such as including character names.
|
||||
- [thinc-apple-ops](https://github.com/explosion/thinc-apple-ops): A special backend for OSX that uses Apple's native libraries for improved performance.
|
||||
- [os-signpost](https://github.com/explosion/os-signpost): A Python package that allows you to use the `OSSignposter` API in OSX for performance analysis.
|
||||
- [spacy-ray](https://github.com/explosion/spacy-ray): A wrapper to integrate spaCy with Ray, a distributed training framework. Currently a work in progress.
|
||||
|
||||
## Prodigy
|
||||
|
||||
[Prodigy](https://prodi.gy) is Explosion's easy to use and highly customizable tool for annotating data. Prodigy itself requires a license, but the repos below contain documentation, examples, and editor or notebook integrations.
|
||||
|
||||
- [prodigy-recipes](https://github.com/explosion/prodigy-recipes): Sample recipes for Prodigy, along with notebooks and other examples of usage.
|
||||
- [vscode-prodigy](https://github.com/explosion/vscode-prodigy): A VS Code extension that lets you run Prodigy inside VS Code.
|
||||
- [jupyterlab-prodigy](https://github.com/explosion/jupyterlab-prodigy): An extension for JupyterLab that lets you run Prodigy inside JupyterLab.
|
||||
|
||||
## Independent Tools or Projects
|
||||
|
||||
These are tools that may be related to or use spaCy, but are functional independent projects in their own right as well.
|
||||
|
||||
- [floret](https://github.com/explosion/floret): A modification of fastText to use Bloom Embeddings. Can be used to add vectors with subword features to spaCy, and also works independently in the same manner as fastText.
|
||||
- [sense2vec](https://github.com/explosion/sense2vec): A library to make embeddings of noun phrases or words coupled with their part of speech. This library uses spaCy.
|
||||
- [spacy-vectors-builder](https://github.com/explosion/spacy-vectors-builder): This is a spaCy project that builds vectors using floret and a lot of input text. It handles downloading the input data as well as the actual building of vectors.
|
||||
- [holmes-extractor](https://github.com/explosion/holmes-extractor): Information extraction from English and German texts based on predicate logic. Uses spaCy.
|
||||
- [healthsea](https://github.com/explosion/healthsea): Healthsea is a project to extract information from comments about health supplements. Structurally, it's a self-contained, large spaCy project.
|
||||
- [spacy-pkuseg](https://github.com/explosion/spacy-pkuseg): A fork of the pkuseg Chinese tokenizer. Used for Chinese support in spaCy, but also works independently.
|
||||
- [ml-datasets](https://github.com/explosion/ml-datasets): This repo includes loaders for several standard machine learning datasets, like MNIST or WikiNER, and has historically been used in spaCy example code and documentation.
|
||||
|
||||
## Documentation and Informational Repos
|
||||
|
||||
These repos are used to support the spaCy docs or otherwise present information about spaCy or other Explosion projects.
|
||||
|
||||
- [projects](https://github.com/explosion/projects): The projects repo is used to show detailed examples of spaCy usage. Individual projects can be checked out using the spaCy command line tool, rather than checking out the projects repo directly.
|
||||
- [spacy-course](https://github.com/explosion/spacy-course): Home to the interactive spaCy course for learning about how to use the library and some basic NLP principles.
|
||||
- [spacy-io-binder](https://github.com/explosion/spacy-io-binder): Home to the notebooks used for interactive examples in the documentation.
|
||||
|
||||
## Organizational / Meta
|
||||
|
||||
These repos are used for organizing data around spaCy, but are not something an end user would need to install as part of using the library.
|
||||
|
||||
- [spacy-models](https://github.com/explosion/spacy-models): This repo contains metadata (but not training data) for all the spaCy models. This includes information about where their training data came from, version compatability, and performance information. It also includes tests for the model packages, and the built models are hosted as releases of this repo.
|
||||
- [wheelwright](https://github.com/explosion/wheelwright): A tool for automating our PyPI builds and releases.
|
||||
- [ec2buildwheel](https://github.com/explosion/ec2buildwheel): A small project that allows you to build Python packages in the manner of cibuildwheel, but on any EC2 image. Used by wheelwright.
|
||||
|
||||
## Other
|
||||
|
||||
Repos that don't fit in any of the above categories.
|
||||
|
||||
- [blis](https://github.com/explosion/blis): A fork of the official BLIS library. The main branch is not updated, but work continues in various branches. This is used for cython-blis.
|
||||
- [tokenizations](https://github.com/explosion/tokenizations): A library originally by Yohei Tamura to align strings with tolerance to some variations in features like case and diacritics, used for aligning tokens and wordpieces. Adopted and maintained by Explosion, but usually spacy-alignments is used instead.
|
||||
- [conll-2012](https://github.com/explosion/conll-2012): A repo to hold some slightly cleaned up versions of the official scripts for the CoNLL 2012 shared task involving coreference resolution. Used in the coref project.
|
||||
- [fastapi-explosion-extras](https://github.com/explosion/fastapi-explosion-extras): Some small tweaks to FastAPI used at Explosion.
|
||||
|
||||
|
|
@ -127,3 +127,34 @@ distributed under the License is distributed on an "AS IS" BASIS,
|
|||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
|
||||
|
||||
polyleven
|
||||
---------
|
||||
|
||||
* Files: spacy/matcher/polyleven.c
|
||||
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2021 Fujimoto Seiji <fujimoto@ceptord.net>
|
||||
Copyright (c) 2021 Max Bachmann <kontakt@maxbachmann.de>
|
||||
Copyright (c) 2022 Nick Mazuk
|
||||
Copyright (c) 2022 Michael Weiss <code@mweiss.ch>
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
|
|
|
|||
|
|
@ -6,7 +6,6 @@ requires = [
|
|||
"preshed>=3.0.2,<3.1.0",
|
||||
"murmurhash>=0.28.0,<1.1.0",
|
||||
"thinc>=8.1.0,<8.2.0",
|
||||
"pathy",
|
||||
"numpy>=1.15.0",
|
||||
]
|
||||
build-backend = "setuptools.build_meta"
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
# Our libraries
|
||||
spacy-legacy>=3.0.9,<3.1.0
|
||||
spacy-legacy>=3.0.10,<3.1.0
|
||||
spacy-loggers>=1.0.0,<2.0.0
|
||||
cymem>=2.0.2,<2.1.0
|
||||
preshed>=3.0.2,<3.1.0
|
||||
|
|
@ -15,7 +15,7 @@ pathy>=0.3.5
|
|||
numpy>=1.15.0
|
||||
requests>=2.13.0,<3.0.0
|
||||
tqdm>=4.38.0,<5.0.0
|
||||
pydantic>=1.7.4,!=1.8,!=1.8.1,<1.10.0
|
||||
pydantic>=1.7.4,!=1.8,!=1.8.1,<1.11.0
|
||||
jinja2
|
||||
langcodes>=3.2.0,<4.0.0
|
||||
# Official Python utilities
|
||||
|
|
@ -28,10 +28,12 @@ cython>=0.25,<3.0
|
|||
pytest>=5.2.0,!=7.1.0
|
||||
pytest-timeout>=1.3.0,<2.0.0
|
||||
mock>=2.0.0,<3.0.0
|
||||
flake8>=3.8.0,<3.10.0
|
||||
flake8>=3.8.0,<6.0.0
|
||||
hypothesis>=3.27.0,<7.0.0
|
||||
mypy>=0.910,<0.970; platform_machine!='aarch64'
|
||||
mypy>=0.980,<0.990; platform_machine != "aarch64" and python_version >= "3.7"
|
||||
types-dataclasses>=0.1.3; python_version < "3.7"
|
||||
types-mock>=0.1.1
|
||||
types-setuptools>=57.0.0
|
||||
types-requests
|
||||
types-setuptools>=57.0.0
|
||||
black>=22.0,<23.0
|
||||
|
|
|
|||
44
setup.cfg
44
setup.cfg
|
|
@ -41,7 +41,7 @@ setup_requires =
|
|||
thinc>=8.1.0,<8.2.0
|
||||
install_requires =
|
||||
# Our libraries
|
||||
spacy-legacy>=3.0.9,<3.1.0
|
||||
spacy-legacy>=3.0.10,<3.1.0
|
||||
spacy-loggers>=1.0.0,<2.0.0
|
||||
murmurhash>=0.28.0,<1.1.0
|
||||
cymem>=2.0.2,<2.1.0
|
||||
|
|
@ -50,13 +50,13 @@ install_requires =
|
|||
wasabi>=0.9.1,<1.1.0
|
||||
srsly>=2.4.3,<3.0.0
|
||||
catalogue>=2.0.6,<2.1.0
|
||||
# Third-party dependencies
|
||||
typer>=0.3.0,<0.5.0
|
||||
pathy>=0.3.5
|
||||
# Third-party dependencies
|
||||
tqdm>=4.38.0,<5.0.0
|
||||
numpy>=1.15.0
|
||||
requests>=2.13.0,<3.0.0
|
||||
pydantic>=1.7.4,!=1.8,!=1.8.1,<1.10.0
|
||||
pydantic>=1.7.4,!=1.8,!=1.8.1,<1.11.0
|
||||
jinja2
|
||||
# Official Python utilities
|
||||
setuptools
|
||||
|
|
@ -76,37 +76,41 @@ transformers =
|
|||
ray =
|
||||
spacy_ray>=0.1.0,<1.0.0
|
||||
cuda =
|
||||
cupy>=5.0.0b4,<11.0.0
|
||||
cupy>=5.0.0b4,<12.0.0
|
||||
cuda80 =
|
||||
cupy-cuda80>=5.0.0b4,<11.0.0
|
||||
cupy-cuda80>=5.0.0b4,<12.0.0
|
||||
cuda90 =
|
||||
cupy-cuda90>=5.0.0b4,<11.0.0
|
||||
cupy-cuda90>=5.0.0b4,<12.0.0
|
||||
cuda91 =
|
||||
cupy-cuda91>=5.0.0b4,<11.0.0
|
||||
cupy-cuda91>=5.0.0b4,<12.0.0
|
||||
cuda92 =
|
||||
cupy-cuda92>=5.0.0b4,<11.0.0
|
||||
cupy-cuda92>=5.0.0b4,<12.0.0
|
||||
cuda100 =
|
||||
cupy-cuda100>=5.0.0b4,<11.0.0
|
||||
cupy-cuda100>=5.0.0b4,<12.0.0
|
||||
cuda101 =
|
||||
cupy-cuda101>=5.0.0b4,<11.0.0
|
||||
cupy-cuda101>=5.0.0b4,<12.0.0
|
||||
cuda102 =
|
||||
cupy-cuda102>=5.0.0b4,<11.0.0
|
||||
cupy-cuda102>=5.0.0b4,<12.0.0
|
||||
cuda110 =
|
||||
cupy-cuda110>=5.0.0b4,<11.0.0
|
||||
cupy-cuda110>=5.0.0b4,<12.0.0
|
||||
cuda111 =
|
||||
cupy-cuda111>=5.0.0b4,<11.0.0
|
||||
cupy-cuda111>=5.0.0b4,<12.0.0
|
||||
cuda112 =
|
||||
cupy-cuda112>=5.0.0b4,<11.0.0
|
||||
cupy-cuda112>=5.0.0b4,<12.0.0
|
||||
cuda113 =
|
||||
cupy-cuda113>=5.0.0b4,<11.0.0
|
||||
cupy-cuda113>=5.0.0b4,<12.0.0
|
||||
cuda114 =
|
||||
cupy-cuda114>=5.0.0b4,<11.0.0
|
||||
cupy-cuda114>=5.0.0b4,<12.0.0
|
||||
cuda115 =
|
||||
cupy-cuda115>=5.0.0b4,<11.0.0
|
||||
cupy-cuda115>=5.0.0b4,<12.0.0
|
||||
cuda116 =
|
||||
cupy-cuda116>=5.0.0b4,<11.0.0
|
||||
cupy-cuda116>=5.0.0b4,<12.0.0
|
||||
cuda117 =
|
||||
cupy-cuda117>=5.0.0b4,<11.0.0
|
||||
cupy-cuda117>=5.0.0b4,<12.0.0
|
||||
cuda11x =
|
||||
cupy-cuda11x>=11.0.0,<12.0.0
|
||||
cuda-autodetect =
|
||||
cupy-wheel>=11.0.0,<12.0.0
|
||||
apple =
|
||||
thinc-apple-ops>=0.1.0.dev0,<1.0.0
|
||||
# Language tokenizers with external dependencies
|
||||
|
|
@ -114,7 +118,7 @@ ja =
|
|||
sudachipy>=0.5.2,!=0.6.1
|
||||
sudachidict_core>=20211220
|
||||
ko =
|
||||
natto-py==0.9.0
|
||||
natto-py>=0.9.0
|
||||
th =
|
||||
pythainlp>=2.0
|
||||
|
||||
|
|
|
|||
11
setup.py
11
setup.py
|
|
@ -205,6 +205,17 @@ def setup_package():
|
|||
get_python_inc(plat_specific=True),
|
||||
]
|
||||
ext_modules = []
|
||||
ext_modules.append(
|
||||
Extension(
|
||||
"spacy.matcher.levenshtein",
|
||||
[
|
||||
"spacy/matcher/levenshtein.pyx",
|
||||
"spacy/matcher/polyleven.c",
|
||||
],
|
||||
language="c",
|
||||
include_dirs=include_dirs,
|
||||
)
|
||||
)
|
||||
for name in MOD_NAMES:
|
||||
mod_path = name.replace(".", "/") + ".pyx"
|
||||
ext = Extension(
|
||||
|
|
|
|||
|
|
@ -31,21 +31,21 @@ def load(
|
|||
name: Union[str, Path],
|
||||
*,
|
||||
vocab: Union[Vocab, bool] = True,
|
||||
disable: Iterable[str] = util.SimpleFrozenList(),
|
||||
enable: Iterable[str] = util.SimpleFrozenList(),
|
||||
exclude: Iterable[str] = util.SimpleFrozenList(),
|
||||
disable: Union[str, Iterable[str]] = util._DEFAULT_EMPTY_PIPES,
|
||||
enable: Union[str, Iterable[str]] = util._DEFAULT_EMPTY_PIPES,
|
||||
exclude: Union[str, Iterable[str]] = util._DEFAULT_EMPTY_PIPES,
|
||||
config: Union[Dict[str, Any], Config] = util.SimpleFrozenDict(),
|
||||
) -> Language:
|
||||
"""Load a spaCy model from an installed package or a local path.
|
||||
|
||||
name (str): Package name or model path.
|
||||
vocab (Vocab): A Vocab object. If True, a vocab is created.
|
||||
disable (Iterable[str]): Names of pipeline components to disable. Disabled
|
||||
disable (Union[str, Iterable[str]]): Name(s) of pipeline component(s) to disable. Disabled
|
||||
pipes will be loaded but they won't be run unless you explicitly
|
||||
enable them by calling nlp.enable_pipe.
|
||||
enable (Iterable[str]): Names of pipeline components to enable. All other
|
||||
enable (Union[str, Iterable[str]]): Name(s) of pipeline component(s) to enable. All other
|
||||
pipes will be disabled (but can be enabled later using nlp.enable_pipe).
|
||||
exclude (Iterable[str]): Names of pipeline components to exclude. Excluded
|
||||
exclude (Union[str, Iterable[str]]): Name(s) of pipeline component(s) to exclude. Excluded
|
||||
components won't be loaded.
|
||||
config (Dict[str, Any] / Config): Config overrides as nested dict or dict
|
||||
keyed by section values in dot notation.
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
# fmt: off
|
||||
__title__ = "spacy"
|
||||
__version__ = "3.4.1"
|
||||
__version__ = "3.4.2"
|
||||
__download_url__ = "https://github.com/explosion/spacy-models/releases/download"
|
||||
__compatibility__ = "https://raw.githubusercontent.com/explosion/spacy-models/master/compatibility.json"
|
||||
__projects__ = "https://github.com/explosion/projects"
|
||||
|
|
|
|||
|
|
@ -573,3 +573,12 @@ def setup_gpu(use_gpu: int, silent=None) -> None:
|
|||
local_msg.info("Using CPU")
|
||||
if gpu_is_available():
|
||||
local_msg.info("To switch to GPU 0, use the option: --gpu-id 0")
|
||||
|
||||
|
||||
def _format_number(number: Union[int, float], ndigits: int = 2) -> str:
|
||||
"""Formats a number (float or int) rounding to `ndigits`, without truncating trailing 0s,
|
||||
as happens with `round(number, ndigits)`"""
|
||||
if isinstance(number, float):
|
||||
return f"{number:.{ndigits}f}"
|
||||
else:
|
||||
return str(number)
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ import typer
|
|||
import math
|
||||
|
||||
from ._util import app, Arg, Opt, show_validation_error, parse_config_overrides
|
||||
from ._util import import_code, debug_cli
|
||||
from ._util import import_code, debug_cli, _format_number
|
||||
from ..training import Example, remove_bilu_prefix
|
||||
from ..training.initialize import get_sourced_components
|
||||
from ..schemas import ConfigSchemaTraining
|
||||
|
|
@ -989,7 +989,8 @@ def _get_kl_divergence(p: Counter, q: Counter) -> float:
|
|||
def _format_span_row(span_data: List[Dict], labels: List[str]) -> List[Any]:
|
||||
"""Compile into one list for easier reporting"""
|
||||
d = {
|
||||
label: [label] + list(round(d[label], 2) for d in span_data) for label in labels
|
||||
label: [label] + list(_format_number(d[label]) for d in span_data)
|
||||
for label in labels
|
||||
}
|
||||
return list(d.values())
|
||||
|
||||
|
|
@ -1004,6 +1005,10 @@ def _get_span_characteristics(
|
|||
label: _gmean(l)
|
||||
for label, l in compiled_gold["spans_length"][spans_key].items()
|
||||
}
|
||||
spans_per_type = {
|
||||
label: len(spans)
|
||||
for label, spans in compiled_gold["spans_per_type"][spans_key].items()
|
||||
}
|
||||
min_lengths = [min(l) for l in compiled_gold["spans_length"][spans_key].values()]
|
||||
max_lengths = [max(l) for l in compiled_gold["spans_length"][spans_key].values()]
|
||||
|
||||
|
|
@ -1031,6 +1036,7 @@ def _get_span_characteristics(
|
|||
return {
|
||||
"sd": span_distinctiveness,
|
||||
"bd": sb_distinctiveness,
|
||||
"spans_per_type": spans_per_type,
|
||||
"lengths": span_length,
|
||||
"min_length": min(min_lengths),
|
||||
"max_length": max(max_lengths),
|
||||
|
|
@ -1045,12 +1051,15 @@ def _get_span_characteristics(
|
|||
|
||||
def _print_span_characteristics(span_characteristics: Dict[str, Any]):
|
||||
"""Print all span characteristics into a table"""
|
||||
headers = ("Span Type", "Length", "SD", "BD")
|
||||
headers = ("Span Type", "Length", "SD", "BD", "N")
|
||||
# Wasabi has this at 30 by default, but we might have some long labels
|
||||
max_col = max(30, max(len(label) for label in span_characteristics["labels"]))
|
||||
# Prepare table data with all span characteristics
|
||||
table_data = [
|
||||
span_characteristics["lengths"],
|
||||
span_characteristics["sd"],
|
||||
span_characteristics["bd"],
|
||||
span_characteristics["spans_per_type"],
|
||||
]
|
||||
table = _format_span_row(
|
||||
span_data=table_data, labels=span_characteristics["labels"]
|
||||
|
|
@ -1061,8 +1070,18 @@ def _print_span_characteristics(span_characteristics: Dict[str, Any]):
|
|||
span_characteristics["avg_sd"],
|
||||
span_characteristics["avg_bd"],
|
||||
]
|
||||
footer = ["Wgt. Average"] + [str(round(f, 2)) for f in footer_data]
|
||||
msg.table(table, footer=footer, header=headers, divider=True)
|
||||
|
||||
footer = (
|
||||
["Wgt. Average"] + ["{:.2f}".format(round(f, 2)) for f in footer_data] + ["-"]
|
||||
)
|
||||
msg.table(
|
||||
table,
|
||||
footer=footer,
|
||||
header=headers,
|
||||
divider=True,
|
||||
aligns=["l"] + ["r"] * (len(footer_data) + 1),
|
||||
max_col=max_col,
|
||||
)
|
||||
|
||||
|
||||
def _get_spans_length_freq_dist(
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ import typer
|
|||
from ._util import app, Arg, Opt, WHEEL_SUFFIX, SDIST_SUFFIX
|
||||
from .. import about
|
||||
from ..util import is_package, get_minor_version, run_command
|
||||
from ..util import is_prerelease_version
|
||||
from ..errors import OLD_MODEL_SHORTCUTS
|
||||
|
||||
|
||||
|
|
@ -19,7 +20,7 @@ def download_cli(
|
|||
ctx: typer.Context,
|
||||
model: str = Arg(..., help="Name of pipeline package to download"),
|
||||
direct: bool = Opt(False, "--direct", "-d", "-D", help="Force direct download of name + version"),
|
||||
sdist: bool = Opt(False, "--sdist", "-S", help="Download sdist (.tar.gz) archive instead of pre-built binary wheel")
|
||||
sdist: bool = Opt(False, "--sdist", "-S", help="Download sdist (.tar.gz) archive instead of pre-built binary wheel"),
|
||||
# fmt: on
|
||||
):
|
||||
"""
|
||||
|
|
@ -35,7 +36,12 @@ def download_cli(
|
|||
download(model, direct, sdist, *ctx.args)
|
||||
|
||||
|
||||
def download(model: str, direct: bool = False, sdist: bool = False, *pip_args) -> None:
|
||||
def download(
|
||||
model: str,
|
||||
direct: bool = False,
|
||||
sdist: bool = False,
|
||||
*pip_args,
|
||||
) -> None:
|
||||
if (
|
||||
not (is_package("spacy") or is_package("spacy-nightly"))
|
||||
and "--no-deps" not in pip_args
|
||||
|
|
@ -49,13 +55,10 @@ def download(model: str, direct: bool = False, sdist: bool = False, *pip_args) -
|
|||
"dependencies, you'll have to install them manually."
|
||||
)
|
||||
pip_args = pip_args + ("--no-deps",)
|
||||
suffix = SDIST_SUFFIX if sdist else WHEEL_SUFFIX
|
||||
dl_tpl = "{m}-{v}/{m}-{v}{s}#egg={m}=={v}"
|
||||
if direct:
|
||||
components = model.split("-")
|
||||
model_name = "".join(components[:-1])
|
||||
version = components[-1]
|
||||
download_model(dl_tpl.format(m=model_name, v=version, s=suffix), pip_args)
|
||||
else:
|
||||
model_name = model
|
||||
if model in OLD_MODEL_SHORTCUTS:
|
||||
|
|
@ -66,15 +69,31 @@ def download(model: str, direct: bool = False, sdist: bool = False, *pip_args) -
|
|||
model_name = OLD_MODEL_SHORTCUTS[model]
|
||||
compatibility = get_compatibility()
|
||||
version = get_version(model_name, compatibility)
|
||||
download_model(dl_tpl.format(m=model_name, v=version, s=suffix), pip_args)
|
||||
|
||||
filename = get_model_filename(model_name, version, sdist)
|
||||
|
||||
download_model(filename, pip_args)
|
||||
msg.good(
|
||||
"Download and installation successful",
|
||||
f"You can now load the package via spacy.load('{model_name}')",
|
||||
)
|
||||
|
||||
|
||||
def get_model_filename(model_name: str, version: str, sdist: bool = False) -> str:
|
||||
dl_tpl = "{m}-{v}/{m}-{v}{s}"
|
||||
egg_tpl = "#egg={m}=={v}"
|
||||
suffix = SDIST_SUFFIX if sdist else WHEEL_SUFFIX
|
||||
filename = dl_tpl.format(m=model_name, v=version, s=suffix)
|
||||
if sdist:
|
||||
filename += egg_tpl.format(m=model_name, v=version)
|
||||
return filename
|
||||
|
||||
|
||||
def get_compatibility() -> dict:
|
||||
version = get_minor_version(about.__version__)
|
||||
if is_prerelease_version(about.__version__):
|
||||
version: Optional[str] = about.__version__
|
||||
else:
|
||||
version = get_minor_version(about.__version__)
|
||||
r = requests.get(about.__compatibility__)
|
||||
if r.status_code != 200:
|
||||
msg.fail(
|
||||
|
|
@ -101,6 +120,11 @@ def get_version(model: str, comp: dict) -> str:
|
|||
return comp[model][0]
|
||||
|
||||
|
||||
def get_latest_version(model: str) -> str:
|
||||
comp = get_compatibility()
|
||||
return get_version(model, comp)
|
||||
|
||||
|
||||
def download_model(
|
||||
filename: str, user_pip_args: Optional[Sequence[str]] = None
|
||||
) -> None:
|
||||
|
|
|
|||
|
|
@ -1,10 +1,13 @@
|
|||
from typing import Optional, Dict, Any, Union, List
|
||||
import platform
|
||||
import pkg_resources
|
||||
import json
|
||||
from pathlib import Path
|
||||
from wasabi import Printer, MarkdownRenderer
|
||||
import srsly
|
||||
|
||||
from ._util import app, Arg, Opt, string_to_list
|
||||
from .download import get_model_filename, get_latest_version
|
||||
from .. import util
|
||||
from .. import about
|
||||
|
||||
|
|
@ -16,6 +19,7 @@ def info_cli(
|
|||
markdown: bool = Opt(False, "--markdown", "-md", help="Generate Markdown for GitHub issues"),
|
||||
silent: bool = Opt(False, "--silent", "-s", "-S", help="Don't print anything (just return)"),
|
||||
exclude: str = Opt("labels", "--exclude", "-e", help="Comma-separated keys to exclude from the print-out"),
|
||||
url: bool = Opt(False, "--url", "-u", help="Print the URL to download the most recent compatible version of the pipeline"),
|
||||
# fmt: on
|
||||
):
|
||||
"""
|
||||
|
|
@ -23,10 +27,19 @@ def info_cli(
|
|||
print its meta information. Flag --markdown prints details in Markdown for easy
|
||||
copy-pasting to GitHub issues.
|
||||
|
||||
Flag --url prints only the download URL of the most recent compatible
|
||||
version of the pipeline.
|
||||
|
||||
DOCS: https://spacy.io/api/cli#info
|
||||
"""
|
||||
exclude = string_to_list(exclude)
|
||||
info(model, markdown=markdown, silent=silent, exclude=exclude)
|
||||
info(
|
||||
model,
|
||||
markdown=markdown,
|
||||
silent=silent,
|
||||
exclude=exclude,
|
||||
url=url,
|
||||
)
|
||||
|
||||
|
||||
def info(
|
||||
|
|
@ -35,11 +48,20 @@ def info(
|
|||
markdown: bool = False,
|
||||
silent: bool = True,
|
||||
exclude: Optional[List[str]] = None,
|
||||
url: bool = False,
|
||||
) -> Union[str, dict]:
|
||||
msg = Printer(no_print=silent, pretty=not silent)
|
||||
if not exclude:
|
||||
exclude = []
|
||||
if model:
|
||||
if url:
|
||||
if model is not None:
|
||||
title = f"Download info for pipeline '{model}'"
|
||||
data = info_model_url(model)
|
||||
print(data["download_url"])
|
||||
return data
|
||||
else:
|
||||
msg.fail("--url option requires a pipeline name", exits=1)
|
||||
elif model:
|
||||
title = f"Info about pipeline '{model}'"
|
||||
data = info_model(model, silent=silent)
|
||||
else:
|
||||
|
|
@ -99,11 +121,44 @@ def info_model(model: str, *, silent: bool = True) -> Dict[str, Any]:
|
|||
meta["source"] = str(model_path.resolve())
|
||||
else:
|
||||
meta["source"] = str(model_path)
|
||||
download_url = info_installed_model_url(model)
|
||||
if download_url:
|
||||
meta["download_url"] = download_url
|
||||
return {
|
||||
k: v for k, v in meta.items() if k not in ("accuracy", "performance", "speed")
|
||||
}
|
||||
|
||||
|
||||
def info_installed_model_url(model: str) -> Optional[str]:
|
||||
"""Given a pipeline name, get the download URL if available, otherwise
|
||||
return None.
|
||||
|
||||
This is only available for pipelines installed as modules that have
|
||||
dist-info available.
|
||||
"""
|
||||
try:
|
||||
dist = pkg_resources.get_distribution(model)
|
||||
data = json.loads(dist.get_metadata("direct_url.json"))
|
||||
return data["url"]
|
||||
except pkg_resources.DistributionNotFound:
|
||||
# no such package
|
||||
return None
|
||||
except Exception:
|
||||
# something else, like no file or invalid JSON
|
||||
return None
|
||||
|
||||
|
||||
def info_model_url(model: str) -> Dict[str, Any]:
|
||||
"""Return the download URL for the latest version of a pipeline."""
|
||||
version = get_latest_version(model)
|
||||
|
||||
filename = get_model_filename(model, version)
|
||||
download_url = about.__download_url__ + "/" + filename
|
||||
release_tpl = "https://github.com/explosion/spacy-models/releases/tag/{m}-{v}"
|
||||
release_url = release_tpl.format(m=model, v=version)
|
||||
return {"download_url": download_url, "release_url": release_url}
|
||||
|
||||
|
||||
def get_markdown(
|
||||
data: Dict[str, Any],
|
||||
title: Optional[str] = None,
|
||||
|
|
|
|||
|
|
@ -299,8 +299,8 @@ def get_meta(
|
|||
}
|
||||
nlp = util.load_model_from_path(Path(model_path))
|
||||
meta.update(nlp.meta)
|
||||
meta.update(existing_meta)
|
||||
meta["spacy_version"] = util.get_minor_version_range(about.__version__)
|
||||
meta.update(existing_meta)
|
||||
meta["vectors"] = {
|
||||
"width": nlp.vocab.vectors_length,
|
||||
"vectors": len(nlp.vocab.vectors),
|
||||
|
|
|
|||
|
|
@ -25,6 +25,7 @@ def project_update_dvc_cli(
|
|||
project_dir: Path = Arg(Path.cwd(), help="Location of project directory. Defaults to current working directory.", exists=True, file_okay=False),
|
||||
workflow: Optional[str] = Arg(None, help=f"Name of workflow defined in {PROJECT_FILE}. Defaults to first workflow if not set."),
|
||||
verbose: bool = Opt(False, "--verbose", "-V", help="Print more info"),
|
||||
quiet: bool = Opt(False, "--quiet", "-q", help="Print less info"),
|
||||
force: bool = Opt(False, "--force", "-F", help="Force update DVC config"),
|
||||
# fmt: on
|
||||
):
|
||||
|
|
@ -36,7 +37,7 @@ def project_update_dvc_cli(
|
|||
|
||||
DOCS: https://spacy.io/api/cli#project-dvc
|
||||
"""
|
||||
project_update_dvc(project_dir, workflow, verbose=verbose, force=force)
|
||||
project_update_dvc(project_dir, workflow, verbose=verbose, quiet=quiet, force=force)
|
||||
|
||||
|
||||
def project_update_dvc(
|
||||
|
|
@ -44,6 +45,7 @@ def project_update_dvc(
|
|||
workflow: Optional[str] = None,
|
||||
*,
|
||||
verbose: bool = False,
|
||||
quiet: bool = False,
|
||||
force: bool = False,
|
||||
) -> None:
|
||||
"""Update the auto-generated Data Version Control (DVC) config file. A DVC
|
||||
|
|
@ -54,11 +56,12 @@ def project_update_dvc(
|
|||
workflow (Optional[str]): Optional name of workflow defined in project.yml.
|
||||
If not set, the first workflow will be used.
|
||||
verbose (bool): Print more info.
|
||||
quiet (bool): Print less info.
|
||||
force (bool): Force update DVC config.
|
||||
"""
|
||||
config = load_project_config(project_dir)
|
||||
updated = update_dvc_config(
|
||||
project_dir, config, workflow, verbose=verbose, force=force
|
||||
project_dir, config, workflow, verbose=verbose, quiet=quiet, force=force
|
||||
)
|
||||
help_msg = "To execute the workflow with DVC, run: dvc repro"
|
||||
if updated:
|
||||
|
|
@ -72,7 +75,7 @@ def update_dvc_config(
|
|||
config: Dict[str, Any],
|
||||
workflow: Optional[str] = None,
|
||||
verbose: bool = False,
|
||||
silent: bool = False,
|
||||
quiet: bool = False,
|
||||
force: bool = False,
|
||||
) -> bool:
|
||||
"""Re-run the DVC commands in dry mode and update dvc.yaml file in the
|
||||
|
|
@ -83,7 +86,7 @@ def update_dvc_config(
|
|||
path (Path): The path to the project directory.
|
||||
config (Dict[str, Any]): The loaded project.yml.
|
||||
verbose (bool): Whether to print additional info (via DVC).
|
||||
silent (bool): Don't output anything (via DVC).
|
||||
quiet (bool): Don't output anything (via DVC).
|
||||
force (bool): Force update, even if hashes match.
|
||||
RETURNS (bool): Whether the DVC config file was updated.
|
||||
"""
|
||||
|
|
@ -105,6 +108,14 @@ def update_dvc_config(
|
|||
dvc_config_path.unlink()
|
||||
dvc_commands = []
|
||||
config_commands = {cmd["name"]: cmd for cmd in config.get("commands", [])}
|
||||
|
||||
# some flags that apply to every command
|
||||
flags = []
|
||||
if verbose:
|
||||
flags.append("--verbose")
|
||||
if quiet:
|
||||
flags.append("--quiet")
|
||||
|
||||
for name in workflows[workflow]:
|
||||
command = config_commands[name]
|
||||
deps = command.get("deps", [])
|
||||
|
|
@ -118,14 +129,26 @@ def update_dvc_config(
|
|||
deps_cmd = [c for cl in [["-d", p] for p in deps] for c in cl]
|
||||
outputs_cmd = [c for cl in [["-o", p] for p in outputs] for c in cl]
|
||||
outputs_nc_cmd = [c for cl in [["-O", p] for p in outputs_no_cache] for c in cl]
|
||||
dvc_cmd = ["run", "-n", name, "-w", str(path), "--no-exec"]
|
||||
|
||||
dvc_cmd = ["run", *flags, "-n", name, "-w", str(path), "--no-exec"]
|
||||
if command.get("no_skip"):
|
||||
dvc_cmd.append("--always-changed")
|
||||
full_cmd = [*dvc_cmd, *deps_cmd, *outputs_cmd, *outputs_nc_cmd, *project_cmd]
|
||||
dvc_commands.append(join_command(full_cmd))
|
||||
|
||||
if not dvc_commands:
|
||||
# If we don't check for this, then there will be an error when reading the
|
||||
# config, since DVC wouldn't create it.
|
||||
msg.fail(
|
||||
"No usable commands for DVC found. This can happen if none of your "
|
||||
"commands have dependencies or outputs.",
|
||||
exits=1,
|
||||
)
|
||||
|
||||
with working_dir(path):
|
||||
dvc_flags = {"--verbose": verbose, "--quiet": silent}
|
||||
run_dvc_commands(dvc_commands, flags=dvc_flags)
|
||||
for c in dvc_commands:
|
||||
dvc_command = "dvc " + c
|
||||
run_command(dvc_command)
|
||||
with dvc_config_path.open("r+", encoding="utf8") as f:
|
||||
content = f.read()
|
||||
f.seek(0, 0)
|
||||
|
|
@ -133,26 +156,6 @@ def update_dvc_config(
|
|||
return True
|
||||
|
||||
|
||||
def run_dvc_commands(
|
||||
commands: Iterable[str] = SimpleFrozenList(), flags: Dict[str, bool] = {}
|
||||
) -> None:
|
||||
"""Run a sequence of DVC commands in a subprocess, in order.
|
||||
|
||||
commands (List[str]): The string commands without the leading "dvc".
|
||||
flags (Dict[str, bool]): Conditional flags to be added to command. Makes it
|
||||
easier to pass flags like --quiet that depend on a variable or
|
||||
command-line setting while avoiding lots of nested conditionals.
|
||||
"""
|
||||
for c in commands:
|
||||
command = split_command(c)
|
||||
dvc_command = ["dvc", *command]
|
||||
# Add the flags if they are set to True
|
||||
for flag, is_active in flags.items():
|
||||
if is_active:
|
||||
dvc_command.append(flag)
|
||||
run_command(dvc_command)
|
||||
|
||||
|
||||
def check_workflows(workflows: List[str], workflow: Optional[str] = None) -> None:
|
||||
"""Validate workflows provided in project.yml and check that a given
|
||||
workflow can be used to generate a DVC config.
|
||||
|
|
|
|||
|
|
@ -1,5 +1,8 @@
|
|||
from typing import Optional, List, Dict, Sequence, Any, Iterable
|
||||
from typing import Optional, List, Dict, Sequence, Any, Iterable, Tuple
|
||||
import os.path
|
||||
from pathlib import Path
|
||||
|
||||
import pkg_resources
|
||||
from wasabi import msg
|
||||
from wasabi.util import locale_escape
|
||||
import sys
|
||||
|
|
@ -71,6 +74,12 @@ def project_run(
|
|||
commands = {cmd["name"]: cmd for cmd in config.get("commands", [])}
|
||||
workflows = config.get("workflows", {})
|
||||
validate_subcommand(list(commands.keys()), list(workflows.keys()), subcommand)
|
||||
|
||||
req_path = project_dir / "requirements.txt"
|
||||
if config.get("check_requirements", True) and os.path.exists(req_path):
|
||||
with req_path.open() as requirements_file:
|
||||
_check_requirements([req.replace("\n", "") for req in requirements_file])
|
||||
|
||||
if subcommand in workflows:
|
||||
msg.info(f"Running workflow '{subcommand}'")
|
||||
for cmd in workflows[subcommand]:
|
||||
|
|
@ -195,6 +204,8 @@ def validate_subcommand(
|
|||
msg.fail(f"No commands or workflows defined in {PROJECT_FILE}", exits=1)
|
||||
if subcommand not in commands and subcommand not in workflows:
|
||||
help_msg = []
|
||||
if subcommand in ["assets", "asset"]:
|
||||
help_msg.append("Did you mean to run: python -m spacy project assets?")
|
||||
if commands:
|
||||
help_msg.append(f"Available commands: {', '.join(commands)}")
|
||||
if workflows:
|
||||
|
|
@ -308,3 +319,32 @@ def get_fileinfo(project_dir: Path, paths: List[str]) -> List[Dict[str, Optional
|
|||
md5 = get_checksum(file_path) if file_path.exists() else None
|
||||
data.append({"path": path, "md5": md5})
|
||||
return data
|
||||
|
||||
|
||||
def _check_requirements(requirements: List[str]) -> Tuple[bool, bool]:
|
||||
"""Checks whether requirements are installed and free of version conflicts.
|
||||
requirements (List[str]): List of requirements.
|
||||
RETURNS (Tuple[bool, bool]): Whether (1) any packages couldn't be imported, (2) any packages with version conflicts
|
||||
exist.
|
||||
"""
|
||||
|
||||
failed_pkgs_msgs: List[str] = []
|
||||
conflicting_pkgs_msgs: List[str] = []
|
||||
|
||||
for req in requirements:
|
||||
try:
|
||||
pkg_resources.require(req)
|
||||
except pkg_resources.DistributionNotFound as dnf:
|
||||
failed_pkgs_msgs.append(dnf.report())
|
||||
except pkg_resources.VersionConflict as vc:
|
||||
conflicting_pkgs_msgs.append(vc.report())
|
||||
|
||||
if len(failed_pkgs_msgs) or len(conflicting_pkgs_msgs):
|
||||
msg.warn(
|
||||
title="Missing requirements or requirement conflicts detected. Make sure your Python environment is set up "
|
||||
"correctly and you installed all requirements specified in your project's requirements.txt: "
|
||||
)
|
||||
for pgk_msg in failed_pkgs_msgs + conflicting_pkgs_msgs:
|
||||
msg.text(pgk_msg)
|
||||
|
||||
return len(failed_pkgs_msgs) > 0, len(conflicting_pkgs_msgs) > 0
|
||||
|
|
|
|||
|
|
@ -16,8 +16,8 @@ def setup_default_warnings():
|
|||
filter_warning("ignore", error_msg="numpy.dtype size changed") # noqa
|
||||
filter_warning("ignore", error_msg="numpy.ufunc size changed") # noqa
|
||||
|
||||
# warn about entity_ruler & matcher having no patterns only once
|
||||
for pipe in ["matcher", "entity_ruler"]:
|
||||
# warn about entity_ruler, span_ruler & matcher having no patterns only once
|
||||
for pipe in ["matcher", "entity_ruler", "span_ruler"]:
|
||||
filter_warning("once", error_msg=Warnings.W036.format(name=pipe))
|
||||
|
||||
# warn once about lemmatizer without required POS
|
||||
|
|
@ -212,6 +212,8 @@ class Warnings(metaclass=ErrorsWithCodes):
|
|||
W121 = ("Attempting to trace non-existent method '{method}' in pipe '{pipe}'")
|
||||
W122 = ("Couldn't trace method '{method}' in pipe '{pipe}'. This can happen if the pipe class "
|
||||
"is a Cython extension type.")
|
||||
W123 = ("Argument {arg} with value {arg_value} is used instead of {config_value} as specified in the config. Be "
|
||||
"aware that this might affect other components in your pipeline.")
|
||||
|
||||
|
||||
class Errors(metaclass=ErrorsWithCodes):
|
||||
|
|
@ -230,8 +232,9 @@ class Errors(metaclass=ErrorsWithCodes):
|
|||
"initialized component.")
|
||||
E004 = ("Can't set up pipeline component: a factory for '{name}' already "
|
||||
"exists. Existing factory: {func}. New factory: {new_func}")
|
||||
E005 = ("Pipeline component '{name}' returned None. If you're using a "
|
||||
"custom component, maybe you forgot to return the processed Doc?")
|
||||
E005 = ("Pipeline component '{name}' returned {returned_type} instead of a "
|
||||
"Doc. If you're using a custom component, maybe you forgot to "
|
||||
"return the processed Doc?")
|
||||
E006 = ("Invalid constraints for adding pipeline component. You can only "
|
||||
"set one of the following: before (component name or index), "
|
||||
"after (component name or index), first (True) or last (True). "
|
||||
|
|
@ -389,7 +392,7 @@ class Errors(metaclass=ErrorsWithCodes):
|
|||
"consider using doc.spans instead.")
|
||||
E106 = ("Can't find `doc._.{attr}` attribute specified in the underscore "
|
||||
"settings: {opts}")
|
||||
E107 = ("Value of `doc._.{attr}` is not JSON-serializable: {value}")
|
||||
E107 = ("Value of custom attribute `{attr}` is not JSON-serializable: {value}")
|
||||
E109 = ("Component '{name}' could not be run. Did you forget to "
|
||||
"call `initialize()`?")
|
||||
E110 = ("Invalid displaCy render wrapper. Expected callable, got: {obj}")
|
||||
|
|
@ -535,11 +538,12 @@ class Errors(metaclass=ErrorsWithCodes):
|
|||
E198 = ("Unable to return {n} most similar vectors for the current vectors "
|
||||
"table, which contains {n_rows} vectors.")
|
||||
E199 = ("Unable to merge 0-length span at `doc[{start}:{end}]`.")
|
||||
E200 = ("Can't yet set {attr} from Span. Vote for this feature on the "
|
||||
"issue tracker: http://github.com/explosion/spaCy/issues")
|
||||
E200 = ("Can't set {attr} from Span.")
|
||||
E202 = ("Unsupported {name} mode '{mode}'. Supported modes: {modes}.")
|
||||
|
||||
# New errors added in v3.x
|
||||
E853 = ("Unsupported component factory name '{name}'. The character '.' is "
|
||||
"not permitted in factory names.")
|
||||
E854 = ("Unable to set doc.ents. Check that the 'ents_filter' does not "
|
||||
"permit overlapping spans.")
|
||||
E855 = ("Invalid {obj}: {obj} is not from the same doc.")
|
||||
|
|
@ -705,7 +709,7 @@ class Errors(metaclass=ErrorsWithCodes):
|
|||
"need to modify the pipeline, use the built-in methods like "
|
||||
"`nlp.add_pipe`, `nlp.remove_pipe`, `nlp.disable_pipe` or "
|
||||
"`nlp.enable_pipe` instead.")
|
||||
E927 = ("Can't write to frozen list Maybe you're trying to modify a computed "
|
||||
E927 = ("Can't write to frozen list. Maybe you're trying to modify a computed "
|
||||
"property or default function argument?")
|
||||
E928 = ("A KnowledgeBase can only be serialized to/from from a directory, "
|
||||
"but the provided argument {loc} points to a file.")
|
||||
|
|
@ -935,8 +939,9 @@ class Errors(metaclass=ErrorsWithCodes):
|
|||
E1040 = ("Doc.from_json requires all tokens to have the same attributes. "
|
||||
"Some tokens do not contain annotation for: {partial_attrs}")
|
||||
E1041 = ("Expected a string, Doc, or bytes as input, but got: {type}")
|
||||
E1042 = ("Function was called with `{arg1}`={arg1_values} and "
|
||||
"`{arg2}`={arg2_values} but these arguments are conflicting.")
|
||||
E1042 = ("`enable={enable}` and `disable={disable}` are inconsistent with each other.\nIf you only passed "
|
||||
"one of `enable` or `disable`, the other argument is specified in your pipeline's configuration.\nIn that "
|
||||
"case pass an empty list for the previously not specified argument to avoid this error.")
|
||||
E1043 = ("Expected None or a value in range [{range_start}, {range_end}] for entity linker threshold, but got "
|
||||
"{value}.")
|
||||
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ from ..punctuation import TOKENIZER_INFIXES as BASE_TOKENIZER_INFIXES
|
|||
|
||||
|
||||
_infixes = (
|
||||
["·", "ㆍ", "\(", "\)"]
|
||||
["·", "ㆍ", r"\(", r"\)"]
|
||||
+ [r"(?<=[0-9])~(?=[0-9-])"]
|
||||
+ LIST_QUOTES
|
||||
+ BASE_TOKENIZER_INFIXES
|
||||
|
|
|
|||
18
spacy/lang/la/__init__.py
Normal file
18
spacy/lang/la/__init__.py
Normal file
|
|
@ -0,0 +1,18 @@
|
|||
from ...language import Language, BaseDefaults
|
||||
from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS
|
||||
from .stop_words import STOP_WORDS
|
||||
from .lex_attrs import LEX_ATTRS
|
||||
|
||||
|
||||
class LatinDefaults(BaseDefaults):
|
||||
tokenizer_exceptions = TOKENIZER_EXCEPTIONS
|
||||
stop_words = STOP_WORDS
|
||||
lex_attr_getters = LEX_ATTRS
|
||||
|
||||
|
||||
class Latin(Language):
|
||||
lang = "la"
|
||||
Defaults = LatinDefaults
|
||||
|
||||
|
||||
__all__ = ["Latin"]
|
||||
34
spacy/lang/la/lex_attrs.py
Normal file
34
spacy/lang/la/lex_attrs.py
Normal file
|
|
@ -0,0 +1,34 @@
|
|||
from ...attrs import LIKE_NUM
|
||||
import re
|
||||
|
||||
# cf. Goyvaerts/Levithan 2009; case-insensitive, allow 4
|
||||
roman_numerals_compile = re.compile(
|
||||
r"(?i)^(?=[MDCLXVI])M*(C[MD]|D?C{0,4})(X[CL]|L?X{0,4})(I[XV]|V?I{0,4})$"
|
||||
)
|
||||
|
||||
_num_words = set(
|
||||
"""
|
||||
unus una unum duo duae tres tria quattuor quinque sex septem octo novem decem
|
||||
""".split()
|
||||
)
|
||||
|
||||
_ordinal_words = set(
|
||||
"""
|
||||
primus prima primum secundus secunda secundum tertius tertia tertium
|
||||
""".split()
|
||||
)
|
||||
|
||||
|
||||
def like_num(text):
|
||||
if text.isdigit():
|
||||
return True
|
||||
if roman_numerals_compile.match(text):
|
||||
return True
|
||||
if text.lower() in _num_words:
|
||||
return True
|
||||
if text.lower() in _ordinal_words:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
LEX_ATTRS = {LIKE_NUM: like_num}
|
||||
37
spacy/lang/la/stop_words.py
Normal file
37
spacy/lang/la/stop_words.py
Normal file
|
|
@ -0,0 +1,37 @@
|
|||
# Corrected Perseus list, cf. https://wiki.digitalclassicist.org/Stopwords_for_Greek_and_Latin
|
||||
|
||||
STOP_WORDS = set(
|
||||
"""
|
||||
ab ac ad adhuc aliqui aliquis an ante apud at atque aut autem
|
||||
|
||||
cum cur
|
||||
|
||||
de deinde dum
|
||||
|
||||
ego enim ergo es est et etiam etsi ex
|
||||
|
||||
fio
|
||||
|
||||
haud hic
|
||||
|
||||
iam idem igitur ille in infra inter interim ipse is ita
|
||||
|
||||
magis modo mox
|
||||
|
||||
nam ne nec necque neque nisi non nos
|
||||
|
||||
o ob
|
||||
|
||||
per possum post pro
|
||||
|
||||
quae quam quare qui quia quicumque quidem quilibet quis quisnam quisquam quisque quisquis quo quoniam
|
||||
|
||||
sed si sic sive sub sui sum super suus
|
||||
|
||||
tam tamen trans tu tum
|
||||
|
||||
ubi uel uero
|
||||
|
||||
vel vero
|
||||
""".split()
|
||||
)
|
||||
76
spacy/lang/la/tokenizer_exceptions.py
Normal file
76
spacy/lang/la/tokenizer_exceptions.py
Normal file
|
|
@ -0,0 +1,76 @@
|
|||
from ..tokenizer_exceptions import BASE_EXCEPTIONS
|
||||
from ...symbols import ORTH
|
||||
from ...util import update_exc
|
||||
|
||||
|
||||
## TODO: Look into systematically handling u/v
|
||||
_exc = {
|
||||
"mecum": [{ORTH: "me"}, {ORTH: "cum"}],
|
||||
"tecum": [{ORTH: "te"}, {ORTH: "cum"}],
|
||||
"nobiscum": [{ORTH: "nobis"}, {ORTH: "cum"}],
|
||||
"vobiscum": [{ORTH: "vobis"}, {ORTH: "cum"}],
|
||||
"uobiscum": [{ORTH: "uobis"}, {ORTH: "cum"}],
|
||||
}
|
||||
|
||||
for orth in [
|
||||
"A.",
|
||||
"Agr.",
|
||||
"Ap.",
|
||||
"C.",
|
||||
"Cn.",
|
||||
"D.",
|
||||
"F.",
|
||||
"K.",
|
||||
"L.",
|
||||
"M'.",
|
||||
"M.",
|
||||
"Mam.",
|
||||
"N.",
|
||||
"Oct.",
|
||||
"Opet.",
|
||||
"P.",
|
||||
"Paul.",
|
||||
"Post.",
|
||||
"Pro.",
|
||||
"Q.",
|
||||
"S.",
|
||||
"Ser.",
|
||||
"Sert.",
|
||||
"Sex.",
|
||||
"St.",
|
||||
"Sta.",
|
||||
"T.",
|
||||
"Ti.",
|
||||
"V.",
|
||||
"Vol.",
|
||||
"Vop.",
|
||||
"U.",
|
||||
"Uol.",
|
||||
"Uop.",
|
||||
"Ian.",
|
||||
"Febr.",
|
||||
"Mart.",
|
||||
"Apr.",
|
||||
"Mai.",
|
||||
"Iun.",
|
||||
"Iul.",
|
||||
"Aug.",
|
||||
"Sept.",
|
||||
"Oct.",
|
||||
"Nov.",
|
||||
"Nou.",
|
||||
"Dec.",
|
||||
"Non.",
|
||||
"Id.",
|
||||
"A.D.",
|
||||
"Coll.",
|
||||
"Cos.",
|
||||
"Ord.",
|
||||
"Pl.",
|
||||
"S.C.",
|
||||
"Suff.",
|
||||
"Trib.",
|
||||
]:
|
||||
_exc[orth] = [{ORTH: orth}]
|
||||
|
||||
TOKENIZER_EXCEPTIONS = update_exc(BASE_EXCEPTIONS, _exc)
|
||||
18
spacy/lang/lg/__init__.py
Normal file
18
spacy/lang/lg/__init__.py
Normal file
|
|
@ -0,0 +1,18 @@
|
|||
from .stop_words import STOP_WORDS
|
||||
from .lex_attrs import LEX_ATTRS
|
||||
from .punctuation import TOKENIZER_INFIXES
|
||||
from ...language import Language, BaseDefaults
|
||||
|
||||
|
||||
class LugandaDefaults(BaseDefaults):
|
||||
lex_attr_getters = LEX_ATTRS
|
||||
infixes = TOKENIZER_INFIXES
|
||||
stop_words = STOP_WORDS
|
||||
|
||||
|
||||
class Luganda(Language):
|
||||
lang = "lg"
|
||||
Defaults = LugandaDefaults
|
||||
|
||||
|
||||
__all__ = ["Luganda"]
|
||||
17
spacy/lang/lg/examples.py
Normal file
17
spacy/lang/lg/examples.py
Normal file
|
|
@ -0,0 +1,17 @@
|
|||
"""
|
||||
Example sentences to test spaCy and its language models.
|
||||
|
||||
>>> from spacy.lang.lg.examples import sentences
|
||||
>>> docs = nlp.pipe(sentences)
|
||||
"""
|
||||
|
||||
sentences = [
|
||||
"Mpa ebyafaayo ku byalo Nakatu ne Nkajja",
|
||||
"Okuyita Ttembo kitegeeza kugwa ddalu",
|
||||
"Ekifumu kino kyali kya mulimu ki?",
|
||||
"Ekkovu we liyise wayitibwa mukululo",
|
||||
"Akola mulimu ki oguvaamu ssente?",
|
||||
"Emisumaali egikomerera embaawo giyitibwa nninga",
|
||||
"Abooluganda ab’emmamba ababiri",
|
||||
"Ekisaawe ky'ebyenjigiriza kya mugaso nnyo",
|
||||
]
|
||||
95
spacy/lang/lg/lex_attrs.py
Normal file
95
spacy/lang/lg/lex_attrs.py
Normal file
|
|
@ -0,0 +1,95 @@
|
|||
from ...attrs import LIKE_NUM
|
||||
|
||||
_num_words = [
|
||||
"nnooti", # Zero
|
||||
"zeero", # zero
|
||||
"emu", # one
|
||||
"bbiri", # two
|
||||
"ssatu", # three
|
||||
"nnya", # four
|
||||
"ttaano", # five
|
||||
"mukaaga", # six
|
||||
"musanvu", # seven
|
||||
"munaana", # eight
|
||||
"mwenda", # nine
|
||||
"kkumi", # ten
|
||||
"kkumi n'emu", # eleven
|
||||
"kkumi na bbiri", # twelve
|
||||
"kkumi na ssatu", # thirteen
|
||||
"kkumi na nnya", # forteen
|
||||
"kkumi na ttaano", # fifteen
|
||||
"kkumi na mukaaga", # sixteen
|
||||
"kkumi na musanvu", # seventeen
|
||||
"kkumi na munaana", # eighteen
|
||||
"kkumi na mwenda", # nineteen
|
||||
"amakumi abiri", # twenty
|
||||
"amakumi asatu", # thirty
|
||||
"amakumi ana", # forty
|
||||
"amakumi ataano", # fifty
|
||||
"nkaaga", # sixty
|
||||
"nsanvu", # seventy
|
||||
"kinaana", # eighty
|
||||
"kyenda", # ninety
|
||||
"kikumi", # hundred
|
||||
"lukumi", # thousand
|
||||
"kakadde", # million
|
||||
"kawumbi", # billion
|
||||
"kase", # trillion
|
||||
"katabalika", # quadrillion
|
||||
"keesedde", # gajillion
|
||||
"kafukunya", # bazillion
|
||||
"ekisooka", # first
|
||||
"ekyokubiri", # second
|
||||
"ekyokusatu", # third
|
||||
"ekyokuna", # fourth
|
||||
"ekyokutaano", # fifith
|
||||
"ekyomukaaga", # sixth
|
||||
"ekyomusanvu", # seventh
|
||||
"eky'omunaana", # eighth
|
||||
"ekyomwenda", # nineth
|
||||
"ekyekkumi", # tenth
|
||||
"ekyekkumi n'ekimu", # eleventh
|
||||
"ekyekkumi n'ebibiri", # twelveth
|
||||
"ekyekkumi n'ebisatu", # thirteenth
|
||||
"ekyekkumi n'ebina", # fourteenth
|
||||
"ekyekkumi n'ebitaano", # fifteenth
|
||||
"ekyekkumi n'omukaaga", # sixteenth
|
||||
"ekyekkumi n'omusanvu", # seventeenth
|
||||
"ekyekkumi n'omunaana", # eigteenth
|
||||
"ekyekkumi n'omwenda", # nineteenth
|
||||
"ekyamakumi abiri", # twentieth
|
||||
"ekyamakumi asatu", # thirtieth
|
||||
"ekyamakumi ana", # fortieth
|
||||
"ekyamakumi ataano", # fiftieth
|
||||
"ekyenkaaga", # sixtieth
|
||||
"ekyensanvu", # seventieth
|
||||
"ekyekinaana", # eightieth
|
||||
"ekyekyenda", # ninetieth
|
||||
"ekyekikumi", # hundredth
|
||||
"ekyolukumi", # thousandth
|
||||
"ekyakakadde", # millionth
|
||||
"ekyakawumbi", # billionth
|
||||
"ekyakase", # trillionth
|
||||
"ekyakatabalika", # quadrillionth
|
||||
"ekyakeesedde", # gajillionth
|
||||
"ekyakafukunya", # bazillionth
|
||||
]
|
||||
|
||||
|
||||
def like_num(text):
|
||||
if text.startswith(("+", "-", "±", "~")):
|
||||
text = text[1:]
|
||||
text = text.replace(",", "").replace(".", "")
|
||||
if text.isdigit():
|
||||
return True
|
||||
if text.count("/") == 1:
|
||||
num, denom = text.split("/")
|
||||
if num.isdigit() and denom.isdigit():
|
||||
return True
|
||||
text_lower = text.lower()
|
||||
if text_lower in _num_words:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
LEX_ATTRS = {LIKE_NUM: like_num}
|
||||
19
spacy/lang/lg/punctuation.py
Normal file
19
spacy/lang/lg/punctuation.py
Normal file
|
|
@ -0,0 +1,19 @@
|
|||
from ..char_classes import LIST_ELLIPSES, LIST_ICONS, HYPHENS
|
||||
from ..char_classes import CONCAT_QUOTES, ALPHA_LOWER, ALPHA_UPPER, ALPHA
|
||||
|
||||
_infixes = (
|
||||
LIST_ELLIPSES
|
||||
+ LIST_ICONS
|
||||
+ [
|
||||
r"(?<=[0-9])[+\-\*^](?=[0-9-])",
|
||||
r"(?<=[{al}{q}])\.(?=[{au}{q}])".format(
|
||||
al=ALPHA_LOWER, au=ALPHA_UPPER, q=CONCAT_QUOTES
|
||||
),
|
||||
r"(?<=[{a}]),(?=[{a}])".format(a=ALPHA),
|
||||
r"(?<=[{a}0-9])(?:{h})(?=[{a}])".format(a=ALPHA, h=HYPHENS),
|
||||
r"(?<=[{a}0-9])[:<>=/](?=[{a}])".format(a=ALPHA),
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
TOKENIZER_INFIXES = _infixes
|
||||
19
spacy/lang/lg/stop_words.py
Normal file
19
spacy/lang/lg/stop_words.py
Normal file
|
|
@ -0,0 +1,19 @@
|
|||
STOP_WORDS = set(
|
||||
"""
|
||||
abadde abalala abamu abangi abava ajja ali alina ani anti ateekeddwa atewamu
|
||||
atya awamu aweebwa ayinza ba baali babadde babalina bajja
|
||||
bajjanewankubade bali balina bandi bangi bano bateekeddwa baweebwa bayina bebombi beera bibye
|
||||
bimu bingi bino bo bokka bonna buli bulijjo bulungi bwabwe bwaffe bwayo bwe bwonna bya byabwe
|
||||
byaffe byebimu byonna ddaa ddala ddi e ebimu ebiri ebweruobulungi ebyo edda ejja ekirala ekyo
|
||||
endala engeri ennyo era erimu erina ffe ffenna ga gujja gumu gunno guno gwa gwe kaseera kati
|
||||
kennyini ki kiki kikino kikye kikyo kino kirungi kki ku kubangabyombi kubangaolwokuba kudda
|
||||
kuva kuwa kwegamba kyaffe kye kyekimuoyo kyekyo kyonna leero liryo lwa lwaki lyabwezaabwe
|
||||
lyaffe lyange mbadde mingi mpozzi mu mulinaoyina munda mwegyabwe nolwekyo nabadde nabo nandiyagadde
|
||||
nandiye nanti naye ne nedda neera nga nnyingi nnyini nnyinza nnyo nti nyinza nze oba ojja okudda
|
||||
okugenda okuggyako okutuusa okuva okuwa oli olina oluvannyuma olwekyobuva omuli ono osobola otya
|
||||
oyina oyo seetaaga si sinakindi singa talina tayina tebaali tebaalina tebayina terina tetulina
|
||||
tetuteekeddwa tewali teyalina teyayina tolina tu tuyina tulina tuyina twafuna twetaaga wa wabula
|
||||
wabweru wadde waggulunnina wakati waliwobangi waliyo wandi wange wano wansi weebwa yabadde yaffe
|
||||
ye yenna yennyini yina yonna ziba zijja zonna
|
||||
""".split()
|
||||
)
|
||||
|
|
@ -40,6 +40,7 @@ def noun_chunks(doclike: Union[Doc, Span]) -> Iterator[Tuple[int, int, int]]:
|
|||
span_label = doc.vocab.strings.add("NP")
|
||||
|
||||
# Only NOUNS and PRONOUNS matter
|
||||
end_span = -1
|
||||
for i, word in enumerate(filter(lambda x: x.pos in [PRON, NOUN], doclike)):
|
||||
# For NOUNS
|
||||
# Pick children from syntactic parse (only those with certain dependencies)
|
||||
|
|
@ -58,15 +59,17 @@ def noun_chunks(doclike: Union[Doc, Span]) -> Iterator[Tuple[int, int, int]]:
|
|||
children_i = [c.i for c in children] + [word.i]
|
||||
|
||||
start_span = min(children_i)
|
||||
end_span = max(children_i) + 1
|
||||
yield start_span, end_span, span_label
|
||||
if start_span >= end_span:
|
||||
end_span = max(children_i) + 1
|
||||
yield start_span, end_span, span_label
|
||||
|
||||
# PRONOUNS only if it is the subject of a verb
|
||||
elif word.pos == PRON:
|
||||
if word.dep in pronoun_deps:
|
||||
start_span = word.i
|
||||
end_span = word.i + 1
|
||||
yield start_span, end_span, span_label
|
||||
if start_span >= end_span:
|
||||
end_span = word.i + 1
|
||||
yield start_span, end_span, span_label
|
||||
|
||||
|
||||
SYNTAX_ITERATORS = {"noun_chunks": noun_chunks}
|
||||
|
|
|
|||
|
|
@ -23,7 +23,7 @@ class RussianLemmatizer(Lemmatizer):
|
|||
overwrite: bool = False,
|
||||
scorer: Optional[Callable] = lemmatizer_score,
|
||||
) -> None:
|
||||
if mode == "pymorphy2":
|
||||
if mode in {"pymorphy2", "pymorphy2_lookup"}:
|
||||
try:
|
||||
from pymorphy2 import MorphAnalyzer
|
||||
except ImportError:
|
||||
|
|
|
|||
|
|
@ -18,7 +18,7 @@ class UkrainianLemmatizer(RussianLemmatizer):
|
|||
overwrite: bool = False,
|
||||
scorer: Optional[Callable] = lemmatizer_score,
|
||||
) -> None:
|
||||
if mode == "pymorphy2":
|
||||
if mode in {"pymorphy2", "pymorphy2_lookup"}:
|
||||
try:
|
||||
from pymorphy2 import MorphAnalyzer
|
||||
except ImportError:
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
from typing import Iterator, Optional, Any, Dict, Callable, Iterable, Collection
|
||||
from typing import Iterator, Optional, Any, Dict, Callable, Iterable
|
||||
from typing import Union, Tuple, List, Set, Pattern, Sequence
|
||||
from typing import NoReturn, TYPE_CHECKING, TypeVar, cast, overload
|
||||
|
||||
|
|
@ -10,6 +10,7 @@ from contextlib import contextmanager
|
|||
from copy import deepcopy
|
||||
from pathlib import Path
|
||||
import warnings
|
||||
|
||||
from thinc.api import get_current_ops, Config, CupyOps, Optimizer
|
||||
import srsly
|
||||
import multiprocessing as mp
|
||||
|
|
@ -24,7 +25,7 @@ from .pipe_analysis import validate_attrs, analyze_pipes, print_pipe_analysis
|
|||
from .training import Example, validate_examples
|
||||
from .training.initialize import init_vocab, init_tok2vec
|
||||
from .scorer import Scorer
|
||||
from .util import registry, SimpleFrozenList, _pipe, raise_error
|
||||
from .util import registry, SimpleFrozenList, _pipe, raise_error, _DEFAULT_EMPTY_PIPES
|
||||
from .util import SimpleFrozenDict, combine_score_weights, CONFIG_SECTION_ORDER
|
||||
from .util import warn_if_jupyter_cupy
|
||||
from .lang.tokenizer_exceptions import URL_MATCH, BASE_EXCEPTIONS
|
||||
|
|
@ -465,6 +466,8 @@ class Language:
|
|||
"""
|
||||
if not isinstance(name, str):
|
||||
raise ValueError(Errors.E963.format(decorator="factory"))
|
||||
if "." in name:
|
||||
raise ValueError(Errors.E853.format(name=name))
|
||||
if not isinstance(default_config, dict):
|
||||
err = Errors.E962.format(
|
||||
style="default config", name=name, cfg_type=type(default_config)
|
||||
|
|
@ -543,8 +546,11 @@ class Language:
|
|||
|
||||
DOCS: https://spacy.io/api/language#component
|
||||
"""
|
||||
if name is not None and not isinstance(name, str):
|
||||
raise ValueError(Errors.E963.format(decorator="component"))
|
||||
if name is not None:
|
||||
if not isinstance(name, str):
|
||||
raise ValueError(Errors.E963.format(decorator="component"))
|
||||
if "." in name:
|
||||
raise ValueError(Errors.E853.format(name=name))
|
||||
component_name = name if name is not None else util.get_object_name(func)
|
||||
|
||||
def add_component(component_func: "Pipe") -> Callable:
|
||||
|
|
@ -1023,8 +1029,8 @@ class Language:
|
|||
raise ValueError(Errors.E109.format(name=name)) from e
|
||||
except Exception as e:
|
||||
error_handler(name, proc, [doc], e)
|
||||
if doc is None:
|
||||
raise ValueError(Errors.E005.format(name=name))
|
||||
if not isinstance(doc, Doc):
|
||||
raise ValueError(Errors.E005.format(name=name, returned_type=type(doc)))
|
||||
return doc
|
||||
|
||||
def disable_pipes(self, *names) -> "DisabledPipes":
|
||||
|
|
@ -1058,7 +1064,7 @@ class Language:
|
|||
"""
|
||||
if enable is None and disable is None:
|
||||
raise ValueError(Errors.E991)
|
||||
if disable is not None and isinstance(disable, str):
|
||||
if isinstance(disable, str):
|
||||
disable = [disable]
|
||||
if enable is not None:
|
||||
if isinstance(enable, str):
|
||||
|
|
@ -1693,9 +1699,9 @@ class Language:
|
|||
config: Union[Dict[str, Any], Config] = {},
|
||||
*,
|
||||
vocab: Union[Vocab, bool] = True,
|
||||
disable: Iterable[str] = SimpleFrozenList(),
|
||||
enable: Iterable[str] = SimpleFrozenList(),
|
||||
exclude: Iterable[str] = SimpleFrozenList(),
|
||||
disable: Union[str, Iterable[str]] = _DEFAULT_EMPTY_PIPES,
|
||||
enable: Union[str, Iterable[str]] = _DEFAULT_EMPTY_PIPES,
|
||||
exclude: Union[str, Iterable[str]] = _DEFAULT_EMPTY_PIPES,
|
||||
meta: Dict[str, Any] = SimpleFrozenDict(),
|
||||
auto_fill: bool = True,
|
||||
validate: bool = True,
|
||||
|
|
@ -1706,12 +1712,12 @@ class Language:
|
|||
|
||||
config (Dict[str, Any] / Config): The loaded config.
|
||||
vocab (Vocab): A Vocab object. If True, a vocab is created.
|
||||
disable (Iterable[str]): Names of pipeline components to disable.
|
||||
disable (Union[str, Iterable[str]]): Name(s) of pipeline component(s) to disable.
|
||||
Disabled pipes will be loaded but they won't be run unless you
|
||||
explicitly enable them by calling nlp.enable_pipe.
|
||||
enable (Iterable[str]): Names of pipeline components to enable. All other
|
||||
enable (Union[str, Iterable[str]]): Name(s) of pipeline component(s) to enable. All other
|
||||
pipes will be disabled (and can be enabled using `nlp.enable_pipe`).
|
||||
exclude (Iterable[str]): Names of pipeline components to exclude.
|
||||
exclude (Union[str, Iterable[str]]): Name(s) of pipeline component(s) to exclude.
|
||||
Excluded components won't be loaded.
|
||||
meta (Dict[str, Any]): Meta overrides for nlp.meta.
|
||||
auto_fill (bool): Automatically fill in missing values in config based
|
||||
|
|
@ -1866,9 +1872,38 @@ class Language:
|
|||
nlp.vocab.from_bytes(vocab_b)
|
||||
|
||||
# Resolve disabled/enabled settings.
|
||||
if isinstance(disable, str):
|
||||
disable = [disable]
|
||||
if isinstance(enable, str):
|
||||
enable = [enable]
|
||||
if isinstance(exclude, str):
|
||||
exclude = [exclude]
|
||||
|
||||
def fetch_pipes_status(value: Iterable[str], key: str) -> Iterable[str]:
|
||||
"""Fetch value for `enable` or `disable` w.r.t. the specified config and passed arguments passed to
|
||||
.load(). If both arguments and config specified values for this field, the passed arguments take precedence
|
||||
and a warning is printed.
|
||||
value (Iterable[str]): Passed value for `enable` or `disable`.
|
||||
key (str): Key for field in config (either "enabled" or "disabled").
|
||||
RETURN (Iterable[str]):
|
||||
"""
|
||||
# We assume that no argument was passed if the value is the specified default value.
|
||||
if id(value) == id(_DEFAULT_EMPTY_PIPES):
|
||||
return config["nlp"].get(key, [])
|
||||
else:
|
||||
if len(config["nlp"].get(key, [])):
|
||||
warnings.warn(
|
||||
Warnings.W123.format(
|
||||
arg=key[:-1],
|
||||
arg_value=value,
|
||||
config_value=config["nlp"][key],
|
||||
)
|
||||
)
|
||||
return value
|
||||
|
||||
disabled_pipes = cls._resolve_component_status(
|
||||
[*config["nlp"]["disabled"], *disable],
|
||||
[*config["nlp"].get("enabled", []), *enable],
|
||||
fetch_pipes_status(disable, "disabled"),
|
||||
fetch_pipes_status(enable, "enabled"),
|
||||
config["nlp"]["pipeline"],
|
||||
)
|
||||
nlp._disabled = set(p for p in disabled_pipes if p not in exclude)
|
||||
|
|
@ -2026,37 +2061,34 @@ class Language:
|
|||
|
||||
@staticmethod
|
||||
def _resolve_component_status(
|
||||
disable: Iterable[str], enable: Iterable[str], pipe_names: Collection[str]
|
||||
disable: Union[str, Iterable[str]],
|
||||
enable: Union[str, Iterable[str]],
|
||||
pipe_names: Iterable[str],
|
||||
) -> Tuple[str, ...]:
|
||||
"""Derives whether (1) `disable` and `enable` values are consistent and (2)
|
||||
resolves those to a single set of disabled components. Raises an error in
|
||||
case of inconsistency.
|
||||
|
||||
disable (Iterable[str]): Names of components or serialization fields to disable.
|
||||
enable (Iterable[str]): Names of pipeline components to enable.
|
||||
disable (Union[str, Iterable[str]]): Name(s) of component(s) or serialization fields to disable.
|
||||
enable (Union[str, Iterable[str]]): Name(s) of pipeline component(s) to enable.
|
||||
pipe_names (Iterable[str]): Names of all pipeline components.
|
||||
|
||||
RETURNS (Tuple[str, ...]): Names of components to exclude from pipeline w.r.t.
|
||||
specified includes and excludes.
|
||||
"""
|
||||
|
||||
if disable is not None and isinstance(disable, str):
|
||||
if isinstance(disable, str):
|
||||
disable = [disable]
|
||||
to_disable = disable
|
||||
|
||||
if enable:
|
||||
if isinstance(enable, str):
|
||||
enable = [enable]
|
||||
to_disable = [
|
||||
pipe_name for pipe_name in pipe_names if pipe_name not in enable
|
||||
]
|
||||
if disable and disable != to_disable:
|
||||
raise ValueError(
|
||||
Errors.E1042.format(
|
||||
arg1="enable",
|
||||
arg2="disable",
|
||||
arg1_values=enable,
|
||||
arg2_values=disable,
|
||||
)
|
||||
)
|
||||
raise ValueError(Errors.E1042.format(enable=enable, disable=disable))
|
||||
|
||||
return tuple(to_disable)
|
||||
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
from .matcher import Matcher
|
||||
from .phrasematcher import PhraseMatcher
|
||||
from .dependencymatcher import DependencyMatcher
|
||||
from .levenshtein import levenshtein
|
||||
|
||||
__all__ = ["Matcher", "PhraseMatcher", "DependencyMatcher"]
|
||||
__all__ = ["Matcher", "PhraseMatcher", "DependencyMatcher", "levenshtein"]
|
||||
|
|
|
|||
|
|
@ -82,6 +82,10 @@ cdef class DependencyMatcher:
|
|||
"$-": self._imm_left_sib,
|
||||
"$++": self._right_sib,
|
||||
"$--": self._left_sib,
|
||||
">++": self._right_child,
|
||||
">--": self._left_child,
|
||||
"<++": self._right_parent,
|
||||
"<--": self._left_parent,
|
||||
}
|
||||
|
||||
def __reduce__(self):
|
||||
|
|
@ -423,6 +427,22 @@ cdef class DependencyMatcher:
|
|||
def _left_sib(self, doc, node):
|
||||
return [doc[child.i] for child in doc[node].head.children if child.i < node]
|
||||
|
||||
def _right_child(self, doc, node):
|
||||
return [doc[child.i] for child in doc[node].children if child.i > node]
|
||||
|
||||
def _left_child(self, doc, node):
|
||||
return [doc[child.i] for child in doc[node].children if child.i < node]
|
||||
|
||||
def _right_parent(self, doc, node):
|
||||
if doc[node].head.i > node:
|
||||
return [doc[node].head]
|
||||
return []
|
||||
|
||||
def _left_parent(self, doc, node):
|
||||
if doc[node].head.i < node:
|
||||
return [doc[node].head]
|
||||
return []
|
||||
|
||||
def _normalize_key(self, key):
|
||||
if isinstance(key, str):
|
||||
return self.vocab.strings.add(key)
|
||||
|
|
|
|||
15
spacy/matcher/levenshtein.pyx
Normal file
15
spacy/matcher/levenshtein.pyx
Normal file
|
|
@ -0,0 +1,15 @@
|
|||
# cython: profile=True, binding=True, infer_types=True
|
||||
from cpython.object cimport PyObject
|
||||
from libc.stdint cimport int64_t
|
||||
|
||||
from typing import Optional
|
||||
|
||||
|
||||
cdef extern from "polyleven.c":
|
||||
int64_t polyleven(PyObject *o1, PyObject *o2, int64_t k)
|
||||
|
||||
|
||||
cpdef int64_t levenshtein(a: str, b: str, k: Optional[int] = None):
|
||||
if k is None:
|
||||
k = -1
|
||||
return polyleven(<PyObject*>a, <PyObject*>b, k)
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
# cython: infer_types=True, cython: profile=True
|
||||
from typing import List
|
||||
from typing import List, Iterable
|
||||
|
||||
from libcpp.vector cimport vector
|
||||
from libc.stdint cimport int32_t, int8_t
|
||||
|
|
@ -867,20 +867,27 @@ class _SetPredicate:
|
|||
|
||||
def __call__(self, Token token):
|
||||
if self.is_extension:
|
||||
value = get_string_id(token._.get(self.attr))
|
||||
value = token._.get(self.attr)
|
||||
else:
|
||||
value = get_token_attr_for_matcher(token.c, self.attr)
|
||||
|
||||
if self.predicate in ("IS_SUBSET", "IS_SUPERSET", "INTERSECTS"):
|
||||
if self.predicate in ("IN", "NOT_IN"):
|
||||
if isinstance(value, (str, int)):
|
||||
value = get_string_id(value)
|
||||
else:
|
||||
return False
|
||||
elif self.predicate in ("IS_SUBSET", "IS_SUPERSET", "INTERSECTS"):
|
||||
# ensure that all values are enclosed in a set
|
||||
if self.attr == MORPH:
|
||||
# break up MORPH into individual Feat=Val values
|
||||
value = set(get_string_id(v) for v in MorphAnalysis.from_id(self.vocab, value))
|
||||
elif isinstance(value, (str, int)):
|
||||
value = set((get_string_id(value),))
|
||||
elif isinstance(value, Iterable) and all(isinstance(v, (str, int)) for v in value):
|
||||
value = set(get_string_id(v) for v in value)
|
||||
else:
|
||||
# treat a single value as a list
|
||||
if isinstance(value, (str, int)):
|
||||
value = set([get_string_id(value)])
|
||||
else:
|
||||
value = set(get_string_id(v) for v in value)
|
||||
return False
|
||||
|
||||
if self.predicate == "IN":
|
||||
return value in self.value
|
||||
elif self.predicate == "NOT_IN":
|
||||
|
|
|
|||
384
spacy/matcher/polyleven.c
Normal file
384
spacy/matcher/polyleven.c
Normal file
|
|
@ -0,0 +1,384 @@
|
|||
/*
|
||||
* Adapted from Polyleven (https://ceptord.net/)
|
||||
*
|
||||
* Source: https://github.com/fujimotos/polyleven/blob/c3f95a080626c5652f0151a2e449963288ccae84/polyleven.c
|
||||
*
|
||||
* Copyright (c) 2021 Fujimoto Seiji <fujimoto@ceptord.net>
|
||||
* Copyright (c) 2021 Max Bachmann <kontakt@maxbachmann.de>
|
||||
* Copyright (c) 2022 Nick Mazuk
|
||||
* Copyright (c) 2022 Michael Weiss <code@mweiss.ch>
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
* of this software and associated documentation files (the "Software"), to deal
|
||||
* in the Software without restriction, including without limitation the rights
|
||||
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
* copies of the Software, and to permit persons to whom the Software is
|
||||
* furnished to do so, subject to the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be included in all
|
||||
* copies or substantial portions of the Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
* SOFTWARE.
|
||||
*/
|
||||
|
||||
#include <Python.h>
|
||||
#include <stdint.h>
|
||||
|
||||
#define MIN(a,b) ((a) < (b) ? (a) : (b))
|
||||
#define MAX(a,b) ((a) > (b) ? (a) : (b))
|
||||
#define CDIV(a,b) ((a) / (b) + ((a) % (b) > 0))
|
||||
#define BIT(i,n) (((i) >> (n)) & 1)
|
||||
#define FLIP(i,n) ((i) ^ ((uint64_t) 1 << (n)))
|
||||
#define ISASCII(kd) ((kd) == PyUnicode_1BYTE_KIND)
|
||||
|
||||
/*
|
||||
* Bare bone of PyUnicode
|
||||
*/
|
||||
struct strbuf {
|
||||
void *ptr;
|
||||
int kind;
|
||||
int64_t len;
|
||||
};
|
||||
|
||||
static void strbuf_init(struct strbuf *s, PyObject *o)
|
||||
{
|
||||
s->ptr = PyUnicode_DATA(o);
|
||||
s->kind = PyUnicode_KIND(o);
|
||||
s->len = PyUnicode_GET_LENGTH(o);
|
||||
}
|
||||
|
||||
#define strbuf_read(s, i) PyUnicode_READ((s)->kind, (s)->ptr, (i))
|
||||
|
||||
/*
|
||||
* An encoded mbleven model table.
|
||||
*
|
||||
* Each 8-bit integer represents an edit sequence, with using two
|
||||
* bits for a single operation.
|
||||
*
|
||||
* 01 = DELETE, 10 = INSERT, 11 = REPLACE
|
||||
*
|
||||
* For example, 13 is '1101' in binary notation, so it means
|
||||
* DELETE + REPLACE.
|
||||
*/
|
||||
static const uint8_t MBLEVEN_MATRIX[] = {
|
||||
3, 0, 0, 0, 0, 0, 0, 0,
|
||||
1, 0, 0, 0, 0, 0, 0, 0,
|
||||
15, 9, 6, 0, 0, 0, 0, 0,
|
||||
13, 7, 0, 0, 0, 0, 0, 0,
|
||||
5, 0, 0, 0, 0, 0, 0, 0,
|
||||
63, 39, 45, 57, 54, 30, 27, 0,
|
||||
61, 55, 31, 37, 25, 22, 0, 0,
|
||||
53, 29, 23, 0, 0, 0, 0, 0,
|
||||
21, 0, 0, 0, 0, 0, 0, 0,
|
||||
};
|
||||
|
||||
#define MBLEVEN_MATRIX_GET(k, d) ((((k) + (k) * (k)) / 2 - 1) + (d)) * 8
|
||||
|
||||
static int64_t mbleven_ascii(char *s1, int64_t len1,
|
||||
char *s2, int64_t len2, int k)
|
||||
{
|
||||
int pos;
|
||||
uint8_t m;
|
||||
int64_t i, j, c, r;
|
||||
|
||||
pos = MBLEVEN_MATRIX_GET(k, len1 - len2);
|
||||
r = k + 1;
|
||||
|
||||
while (MBLEVEN_MATRIX[pos]) {
|
||||
m = MBLEVEN_MATRIX[pos++];
|
||||
i = j = c = 0;
|
||||
while (i < len1 && j < len2) {
|
||||
if (s1[i] != s2[j]) {
|
||||
c++;
|
||||
if (!m) break;
|
||||
if (m & 1) i++;
|
||||
if (m & 2) j++;
|
||||
m >>= 2;
|
||||
} else {
|
||||
i++;
|
||||
j++;
|
||||
}
|
||||
}
|
||||
c += (len1 - i) + (len2 - j);
|
||||
r = MIN(r, c);
|
||||
if (r < 2) {
|
||||
return r;
|
||||
}
|
||||
}
|
||||
return r;
|
||||
}
|
||||
|
||||
static int64_t mbleven(PyObject *o1, PyObject *o2, int64_t k)
|
||||
{
|
||||
int pos;
|
||||
uint8_t m;
|
||||
int64_t i, j, c, r;
|
||||
struct strbuf s1, s2;
|
||||
|
||||
strbuf_init(&s1, o1);
|
||||
strbuf_init(&s2, o2);
|
||||
|
||||
if (s1.len < s2.len)
|
||||
return mbleven(o2, o1, k);
|
||||
|
||||
if (k > 3)
|
||||
return -1;
|
||||
|
||||
if (k < s1.len - s2.len)
|
||||
return k + 1;
|
||||
|
||||
if (ISASCII(s1.kind) && ISASCII(s2.kind))
|
||||
return mbleven_ascii(s1.ptr, s1.len, s2.ptr, s2.len, k);
|
||||
|
||||
pos = MBLEVEN_MATRIX_GET(k, s1.len - s2.len);
|
||||
r = k + 1;
|
||||
|
||||
while (MBLEVEN_MATRIX[pos]) {
|
||||
m = MBLEVEN_MATRIX[pos++];
|
||||
i = j = c = 0;
|
||||
while (i < s1.len && j < s2.len) {
|
||||
if (strbuf_read(&s1, i) != strbuf_read(&s2, j)) {
|
||||
c++;
|
||||
if (!m) break;
|
||||
if (m & 1) i++;
|
||||
if (m & 2) j++;
|
||||
m >>= 2;
|
||||
} else {
|
||||
i++;
|
||||
j++;
|
||||
}
|
||||
}
|
||||
c += (s1.len - i) + (s2.len - j);
|
||||
r = MIN(r, c);
|
||||
if (r < 2) {
|
||||
return r;
|
||||
}
|
||||
}
|
||||
return r;
|
||||
}
|
||||
|
||||
/*
|
||||
* Data structure to store Peq (equality bit-vector).
|
||||
*/
|
||||
struct blockmap_entry {
|
||||
uint32_t key[128];
|
||||
uint64_t val[128];
|
||||
};
|
||||
|
||||
struct blockmap {
|
||||
int64_t nr;
|
||||
struct blockmap_entry *list;
|
||||
};
|
||||
|
||||
#define blockmap_key(c) ((c) | 0x80000000U)
|
||||
#define blockmap_hash(c) ((c) % 128)
|
||||
|
||||
static int blockmap_init(struct blockmap *map, struct strbuf *s)
|
||||
{
|
||||
int64_t i;
|
||||
struct blockmap_entry *be;
|
||||
uint32_t c, k;
|
||||
uint8_t h;
|
||||
|
||||
map->nr = CDIV(s->len, 64);
|
||||
map->list = calloc(1, map->nr * sizeof(struct blockmap_entry));
|
||||
if (map->list == NULL) {
|
||||
PyErr_NoMemory();
|
||||
return -1;
|
||||
}
|
||||
|
||||
for (i = 0; i < s->len; i++) {
|
||||
be = &(map->list[i / 64]);
|
||||
c = strbuf_read(s, i);
|
||||
h = blockmap_hash(c);
|
||||
k = blockmap_key(c);
|
||||
|
||||
while (be->key[h] && be->key[h] != k)
|
||||
h = blockmap_hash(h + 1);
|
||||
be->key[h] = k;
|
||||
be->val[h] |= (uint64_t) 1 << (i % 64);
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
static void blockmap_clear(struct blockmap *map)
|
||||
{
|
||||
if (map->list)
|
||||
free(map->list);
|
||||
map->list = NULL;
|
||||
map->nr = 0;
|
||||
}
|
||||
|
||||
static uint64_t blockmap_get(struct blockmap *map, int block, uint32_t c)
|
||||
{
|
||||
struct blockmap_entry *be;
|
||||
uint8_t h;
|
||||
uint32_t k;
|
||||
|
||||
h = blockmap_hash(c);
|
||||
k = blockmap_key(c);
|
||||
|
||||
be = &(map->list[block]);
|
||||
while (be->key[h] && be->key[h] != k)
|
||||
h = blockmap_hash(h + 1);
|
||||
return be->key[h] == k ? be->val[h] : 0;
|
||||
}
|
||||
|
||||
/*
|
||||
* Myers' bit-parallel algorithm
|
||||
*
|
||||
* See: G. Myers. "A fast bit-vector algorithm for approximate string
|
||||
* matching based on dynamic programming." Journal of the ACM, 1999.
|
||||
*/
|
||||
static int64_t myers1999_block(struct strbuf *s1, struct strbuf *s2,
|
||||
struct blockmap *map)
|
||||
{
|
||||
uint64_t Eq, Xv, Xh, Ph, Mh, Pv, Mv, Last;
|
||||
uint64_t *Mhc, *Phc;
|
||||
int64_t i, b, hsize, vsize, Score;
|
||||
uint8_t Pb, Mb;
|
||||
|
||||
hsize = CDIV(s1->len, 64);
|
||||
vsize = CDIV(s2->len, 64);
|
||||
Score = s2->len;
|
||||
|
||||
Phc = malloc(hsize * 2 * sizeof(uint64_t));
|
||||
if (Phc == NULL) {
|
||||
PyErr_NoMemory();
|
||||
return -1;
|
||||
}
|
||||
Mhc = Phc + hsize;
|
||||
memset(Phc, -1, hsize * sizeof(uint64_t));
|
||||
memset(Mhc, 0, hsize * sizeof(uint64_t));
|
||||
Last = (uint64_t)1 << ((s2->len - 1) % 64);
|
||||
|
||||
for (b = 0; b < vsize; b++) {
|
||||
Mv = 0;
|
||||
Pv = (uint64_t) -1;
|
||||
Score = s2->len;
|
||||
|
||||
for (i = 0; i < s1->len; i++) {
|
||||
Eq = blockmap_get(map, b, strbuf_read(s1, i));
|
||||
|
||||
Pb = BIT(Phc[i / 64], i % 64);
|
||||
Mb = BIT(Mhc[i / 64], i % 64);
|
||||
|
||||
Xv = Eq | Mv;
|
||||
Xh = ((((Eq | Mb) & Pv) + Pv) ^ Pv) | Eq | Mb;
|
||||
|
||||
Ph = Mv | ~ (Xh | Pv);
|
||||
Mh = Pv & Xh;
|
||||
|
||||
if (Ph & Last) Score++;
|
||||
if (Mh & Last) Score--;
|
||||
|
||||
if ((Ph >> 63) ^ Pb)
|
||||
Phc[i / 64] = FLIP(Phc[i / 64], i % 64);
|
||||
|
||||
if ((Mh >> 63) ^ Mb)
|
||||
Mhc[i / 64] = FLIP(Mhc[i / 64], i % 64);
|
||||
|
||||
Ph = (Ph << 1) | Pb;
|
||||
Mh = (Mh << 1) | Mb;
|
||||
|
||||
Pv = Mh | ~ (Xv | Ph);
|
||||
Mv = Ph & Xv;
|
||||
}
|
||||
}
|
||||
free(Phc);
|
||||
return Score;
|
||||
}
|
||||
|
||||
static int64_t myers1999_simple(uint8_t *s1, int64_t len1, uint8_t *s2, int64_t len2)
|
||||
{
|
||||
uint64_t Peq[256];
|
||||
uint64_t Eq, Xv, Xh, Ph, Mh, Pv, Mv, Last;
|
||||
int64_t i;
|
||||
int64_t Score = len2;
|
||||
|
||||
memset(Peq, 0, sizeof(Peq));
|
||||
|
||||
for (i = 0; i < len2; i++)
|
||||
Peq[s2[i]] |= (uint64_t) 1 << i;
|
||||
|
||||
Mv = 0;
|
||||
Pv = (uint64_t) -1;
|
||||
Last = (uint64_t) 1 << (len2 - 1);
|
||||
|
||||
for (i = 0; i < len1; i++) {
|
||||
Eq = Peq[s1[i]];
|
||||
|
||||
Xv = Eq | Mv;
|
||||
Xh = (((Eq & Pv) + Pv) ^ Pv) | Eq;
|
||||
|
||||
Ph = Mv | ~ (Xh | Pv);
|
||||
Mh = Pv & Xh;
|
||||
|
||||
if (Ph & Last) Score++;
|
||||
if (Mh & Last) Score--;
|
||||
|
||||
Ph = (Ph << 1) | 1;
|
||||
Mh = (Mh << 1);
|
||||
|
||||
Pv = Mh | ~ (Xv | Ph);
|
||||
Mv = Ph & Xv;
|
||||
}
|
||||
return Score;
|
||||
}
|
||||
|
||||
static int64_t myers1999(PyObject *o1, PyObject *o2)
|
||||
{
|
||||
struct strbuf s1, s2;
|
||||
struct blockmap map;
|
||||
int64_t ret;
|
||||
|
||||
strbuf_init(&s1, o1);
|
||||
strbuf_init(&s2, o2);
|
||||
|
||||
if (s1.len < s2.len)
|
||||
return myers1999(o2, o1);
|
||||
|
||||
if (ISASCII(s1.kind) && ISASCII(s2.kind) && s2.len < 65)
|
||||
return myers1999_simple(s1.ptr, s1.len, s2.ptr, s2.len);
|
||||
|
||||
if (blockmap_init(&map, &s2))
|
||||
return -1;
|
||||
|
||||
ret = myers1999_block(&s1, &s2, &map);
|
||||
blockmap_clear(&map);
|
||||
return ret;
|
||||
}
|
||||
|
||||
/*
|
||||
* Interface functions
|
||||
*/
|
||||
static int64_t polyleven(PyObject *o1, PyObject *o2, int64_t k)
|
||||
{
|
||||
int64_t len1, len2;
|
||||
|
||||
len1 = PyUnicode_GET_LENGTH(o1);
|
||||
len2 = PyUnicode_GET_LENGTH(o2);
|
||||
|
||||
if (len1 < len2)
|
||||
return polyleven(o2, o1, k);
|
||||
|
||||
if (k == 0)
|
||||
return PyUnicode_Compare(o1, o2) ? 1 : 0;
|
||||
|
||||
if (0 < k && k < len1 - len2)
|
||||
return k + 1;
|
||||
|
||||
if (len2 == 0)
|
||||
return len1;
|
||||
|
||||
if (0 < k && k < 4)
|
||||
return mbleven(o1, o2, k);
|
||||
|
||||
return myers1999(o1, o2);
|
||||
}
|
||||
|
|
@ -89,11 +89,14 @@ def pipes_with_nvtx_range(
|
|||
types.MethodType(nvtx_range_wrapper_for_pipe_method, pipe), func
|
||||
)
|
||||
|
||||
# Try to preserve the original function signature.
|
||||
# We need to preserve the original function signature so that
|
||||
# the original parameters are passed to pydantic for validation downstream.
|
||||
try:
|
||||
wrapped_func.__signature__ = inspect.signature(func) # type: ignore
|
||||
except:
|
||||
pass
|
||||
# Can fail for Cython methods that do not have bindings.
|
||||
warnings.warn(Warnings.W122.format(method=name, pipe=pipe.name))
|
||||
continue
|
||||
|
||||
try:
|
||||
setattr(
|
||||
|
|
|
|||
|
|
@ -1,7 +1,6 @@
|
|||
from typing import cast, Any, Callable, Dict, Iterable, List, Optional
|
||||
from typing import Sequence, Tuple, Union
|
||||
from typing import Tuple
|
||||
from collections import Counter
|
||||
from copy import deepcopy
|
||||
from itertools import islice
|
||||
import numpy as np
|
||||
|
||||
|
|
@ -149,9 +148,7 @@ class EditTreeLemmatizer(TrainablePipe):
|
|||
if not any(len(doc) for doc in docs):
|
||||
# Handle cases where there are no tokens in any docs.
|
||||
n_labels = len(self.cfg["labels"])
|
||||
guesses: List[Ints2d] = [
|
||||
self.model.ops.alloc((0, n_labels), dtype="i") for doc in docs
|
||||
]
|
||||
guesses: List[Ints2d] = [self.model.ops.alloc2i(0, n_labels) for _ in docs]
|
||||
assert len(guesses) == n_docs
|
||||
return guesses
|
||||
scores = self.model.predict(docs)
|
||||
|
|
|
|||
|
|
@ -1,6 +1,5 @@
|
|||
import warnings
|
||||
from typing import Optional, Union, List, Dict, Tuple, Iterable, Any, Callable, Sequence
|
||||
from typing import cast
|
||||
import warnings
|
||||
from collections import defaultdict
|
||||
from pathlib import Path
|
||||
import srsly
|
||||
|
|
@ -317,7 +316,7 @@ class EntityRuler(Pipe):
|
|||
phrase_pattern["id"] = ent_id
|
||||
phrase_patterns.append(phrase_pattern)
|
||||
for entry in token_patterns + phrase_patterns: # type: ignore[operator]
|
||||
label = entry["label"]
|
||||
label = entry["label"] # type: ignore
|
||||
if "id" in entry:
|
||||
ent_label = label
|
||||
label = self._create_label(label, entry["id"])
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
# cython: infer_types=True, profile=True
|
||||
# cython: infer_types=True, profile=True, binding=True
|
||||
from typing import Optional, Tuple, Iterable, Iterator, Callable, Union, Dict
|
||||
import srsly
|
||||
import warnings
|
||||
|
|
|
|||
|
|
@ -133,6 +133,9 @@ def make_spancat(
|
|||
spans_key (str): Key of the doc.spans dict to save the spans under. During
|
||||
initialization and training, the component will look for spans on the
|
||||
reference document under the same key.
|
||||
scorer (Optional[Callable]): The scoring method. Defaults to
|
||||
Scorer.score_spans for the Doc.spans[spans_key] with overlapping
|
||||
spans allowed.
|
||||
threshold (float): Minimum probability to consider a prediction positive.
|
||||
Spans with a positive prediction will be saved on the Doc. Defaults to
|
||||
0.5.
|
||||
|
|
|
|||
|
|
@ -96,8 +96,8 @@ def make_multilabel_textcat(
|
|||
model: Model[List[Doc], List[Floats2d]],
|
||||
threshold: float,
|
||||
scorer: Optional[Callable],
|
||||
) -> "TextCategorizer":
|
||||
"""Create a TextCategorizer component. The text categorizer predicts categories
|
||||
) -> "MultiLabel_TextCategorizer":
|
||||
"""Create a MultiLabel_TextCategorizer component. The text categorizer predicts categories
|
||||
over a whole document. It can learn one or more labels, and the labels are considered
|
||||
to be non-mutually exclusive, which means that there can be zero or more labels
|
||||
per doc).
|
||||
|
|
@ -105,6 +105,7 @@ def make_multilabel_textcat(
|
|||
model (Model[List[Doc], List[Floats2d]]): A model instance that predicts
|
||||
scores for each category.
|
||||
threshold (float): Cutoff to consider a prediction "positive".
|
||||
scorer (Optional[Callable]): The scoring method.
|
||||
"""
|
||||
return MultiLabel_TextCategorizer(
|
||||
nlp.vocab, model, name, threshold=threshold, scorer=scorer
|
||||
|
|
@ -147,6 +148,7 @@ class MultiLabel_TextCategorizer(TextCategorizer):
|
|||
name (str): The component instance name, used to add entries to the
|
||||
losses during training.
|
||||
threshold (float): Cutoff to consider a prediction "positive".
|
||||
scorer (Optional[Callable]): The scoring method.
|
||||
|
||||
DOCS: https://spacy.io/api/textcategorizer#init
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
# cython: infer_types=True, profile=True
|
||||
# cython: infer_types=True, profile=True, binding=True
|
||||
from typing import Iterable, Iterator, Optional, Dict, Tuple, Callable
|
||||
import srsly
|
||||
from thinc.api import set_dropout_rate, Model, Optimizer
|
||||
|
|
|
|||
|
|
@ -181,12 +181,12 @@ class TokenPatternNumber(BaseModel):
|
|||
IS_SUBSET: Optional[List[StrictInt]] = Field(None, alias="is_subset")
|
||||
IS_SUPERSET: Optional[List[StrictInt]] = Field(None, alias="is_superset")
|
||||
INTERSECTS: Optional[List[StrictInt]] = Field(None, alias="intersects")
|
||||
EQ: Union[StrictInt, StrictFloat] = Field(None, alias="==")
|
||||
NEQ: Union[StrictInt, StrictFloat] = Field(None, alias="!=")
|
||||
GEQ: Union[StrictInt, StrictFloat] = Field(None, alias=">=")
|
||||
LEQ: Union[StrictInt, StrictFloat] = Field(None, alias="<=")
|
||||
GT: Union[StrictInt, StrictFloat] = Field(None, alias=">")
|
||||
LT: Union[StrictInt, StrictFloat] = Field(None, alias="<")
|
||||
EQ: Optional[Union[StrictInt, StrictFloat]] = Field(None, alias="==")
|
||||
NEQ: Optional[Union[StrictInt, StrictFloat]] = Field(None, alias="!=")
|
||||
GEQ: Optional[Union[StrictInt, StrictFloat]] = Field(None, alias=">=")
|
||||
LEQ: Optional[Union[StrictInt, StrictFloat]] = Field(None, alias="<=")
|
||||
GT: Optional[Union[StrictInt, StrictFloat]] = Field(None, alias=">")
|
||||
LT: Optional[Union[StrictInt, StrictFloat]] = Field(None, alias="<")
|
||||
|
||||
class Config:
|
||||
extra = "forbid"
|
||||
|
|
@ -207,7 +207,7 @@ class TokenPatternOperatorSimple(str, Enum):
|
|||
|
||||
|
||||
class TokenPatternOperatorMinMax(ConstrainedStr):
|
||||
regex = re.compile("^({\d+}|{\d+,\d*}|{\d*,\d+})$")
|
||||
regex = re.compile(r"^({\d+}|{\d+,\d*}|{\d*,\d+})$")
|
||||
|
||||
|
||||
TokenPatternOperator = Union[TokenPatternOperatorSimple, TokenPatternOperatorMinMax]
|
||||
|
|
@ -430,7 +430,7 @@ class ProjectConfigAssetURL(BaseModel):
|
|||
# fmt: off
|
||||
dest: StrictStr = Field(..., title="Destination of downloaded asset")
|
||||
url: Optional[StrictStr] = Field(None, title="URL of asset")
|
||||
checksum: str = Field(None, title="MD5 hash of file", regex=r"([a-fA-F\d]{32})")
|
||||
checksum: Optional[str] = Field(None, title="MD5 hash of file", regex=r"([a-fA-F\d]{32})")
|
||||
description: StrictStr = Field("", title="Description of asset")
|
||||
# fmt: on
|
||||
|
||||
|
|
@ -438,7 +438,7 @@ class ProjectConfigAssetURL(BaseModel):
|
|||
class ProjectConfigAssetGit(BaseModel):
|
||||
# fmt: off
|
||||
git: ProjectConfigAssetGitItem = Field(..., title="Git repo information")
|
||||
checksum: str = Field(None, title="MD5 hash of file", regex=r"([a-fA-F\d]{32})")
|
||||
checksum: Optional[str] = Field(None, title="MD5 hash of file", regex=r"([a-fA-F\d]{32})")
|
||||
description: Optional[StrictStr] = Field(None, title="Description of asset")
|
||||
# fmt: on
|
||||
|
||||
|
|
@ -508,12 +508,20 @@ class DocJSONSchema(BaseModel):
|
|||
None, title="Indices of sentences' start and end indices"
|
||||
)
|
||||
text: StrictStr = Field(..., title="Document text")
|
||||
spans: Dict[StrictStr, List[Dict[StrictStr, Union[StrictStr, StrictInt]]]] = Field(
|
||||
None, title="Span information - end/start indices, label, KB ID"
|
||||
)
|
||||
spans: Optional[
|
||||
Dict[StrictStr, List[Dict[StrictStr, Union[StrictStr, StrictInt]]]]
|
||||
] = Field(None, title="Span information - end/start indices, label, KB ID")
|
||||
tokens: List[Dict[StrictStr, Union[StrictStr, StrictInt]]] = Field(
|
||||
..., title="Token information - ID, start, annotations"
|
||||
)
|
||||
_: Optional[Dict[StrictStr, Any]] = Field(
|
||||
None, title="Any custom data stored in the document's _ attribute"
|
||||
underscore_doc: Optional[Dict[StrictStr, Any]] = Field(
|
||||
None,
|
||||
title="Any custom data stored in the document's _ attribute",
|
||||
alias="_",
|
||||
)
|
||||
underscore_token: Optional[Dict[StrictStr, List[Dict[StrictStr, Any]]]] = Field(
|
||||
None, title="Any custom data stored in the token's _ attribute"
|
||||
)
|
||||
underscore_span: Optional[Dict[StrictStr, List[Dict[StrictStr, Any]]]] = Field(
|
||||
None, title="Any custom data stored in the span's _ attribute"
|
||||
)
|
||||
|
|
|
|||
|
|
@ -256,11 +256,21 @@ def ko_tokenizer_tokenizer():
|
|||
return nlp.tokenizer
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def la_tokenizer():
|
||||
return get_lang_class("la")().tokenizer
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def lb_tokenizer():
|
||||
return get_lang_class("lb")().tokenizer
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def lg_tokenizer():
|
||||
return get_lang_class("lg")().tokenizer
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def lt_tokenizer():
|
||||
return get_lang_class("lt")().tokenizer
|
||||
|
|
@ -333,6 +343,14 @@ def ru_lemmatizer():
|
|||
return get_lang_class("ru")().add_pipe("lemmatizer")
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def ru_lookup_lemmatizer():
|
||||
pytest.importorskip("pymorphy2")
|
||||
return get_lang_class("ru")().add_pipe(
|
||||
"lemmatizer", config={"mode": "pymorphy2_lookup"}
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def sa_tokenizer():
|
||||
return get_lang_class("sa")().tokenizer
|
||||
|
|
@ -412,6 +430,15 @@ def uk_lemmatizer():
|
|||
return get_lang_class("uk")().add_pipe("lemmatizer")
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def uk_lookup_lemmatizer():
|
||||
pytest.importorskip("pymorphy2")
|
||||
pytest.importorskip("pymorphy2_dicts_uk")
|
||||
return get_lang_class("uk")().add_pipe(
|
||||
"lemmatizer", config={"mode": "pymorphy2_lookup"}
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def ur_tokenizer():
|
||||
return get_lang_class("ur")().tokenizer
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ import weakref
|
|||
import numpy
|
||||
from numpy.testing import assert_array_equal
|
||||
import pytest
|
||||
import warnings
|
||||
from thinc.api import NumpyOps, get_current_ops
|
||||
|
||||
from spacy.attrs import DEP, ENT_IOB, ENT_TYPE, HEAD, IS_ALPHA, MORPH, POS
|
||||
|
|
@ -81,6 +82,21 @@ def test_issue2396(en_vocab):
|
|||
assert (span.get_lca_matrix() == matrix).all()
|
||||
|
||||
|
||||
@pytest.mark.issue(11499)
|
||||
def test_init_args_unmodified(en_vocab):
|
||||
words = ["A", "sentence"]
|
||||
ents = ["B-TYPE1", ""]
|
||||
sent_starts = [True, False]
|
||||
Doc(
|
||||
vocab=en_vocab,
|
||||
words=words,
|
||||
ents=ents,
|
||||
sent_starts=sent_starts,
|
||||
)
|
||||
assert ents == ["B-TYPE1", ""]
|
||||
assert sent_starts == [True, False]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("text", ["-0.23", "+123,456", "±1"])
|
||||
@pytest.mark.parametrize("lang_cls", [English, MultiLanguage])
|
||||
@pytest.mark.issue(2782)
|
||||
|
|
@ -529,9 +545,9 @@ def test_doc_from_array_sent_starts(en_vocab):
|
|||
# no warning using default attrs
|
||||
attrs = doc._get_array_attrs()
|
||||
arr = doc.to_array(attrs)
|
||||
with pytest.warns(None) as record:
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter("error")
|
||||
new_doc.from_array(attrs, arr)
|
||||
assert len(record) == 0
|
||||
# only SENT_START uses SENT_START
|
||||
attrs = [SENT_START]
|
||||
arr = doc.to_array(attrs)
|
||||
|
|
|
|||
|
|
@ -1,12 +1,15 @@
|
|||
import pytest
|
||||
import spacy
|
||||
from spacy import schemas
|
||||
from spacy.tokens import Doc, Span
|
||||
from spacy.tokens import Doc, Span, Token
|
||||
import srsly
|
||||
from .test_underscore import clean_underscore # noqa: F401
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def doc(en_vocab):
|
||||
words = ["c", "d", "e"]
|
||||
spaces = [True, True, True]
|
||||
pos = ["VERB", "NOUN", "NOUN"]
|
||||
tags = ["VBP", "NN", "NN"]
|
||||
heads = [0, 0, 1]
|
||||
|
|
@ -17,6 +20,7 @@ def doc(en_vocab):
|
|||
return Doc(
|
||||
en_vocab,
|
||||
words=words,
|
||||
spaces=spaces,
|
||||
pos=pos,
|
||||
tags=tags,
|
||||
heads=heads,
|
||||
|
|
@ -45,6 +49,47 @@ def doc_without_deps(en_vocab):
|
|||
)
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def doc_json():
|
||||
return {
|
||||
"text": "c d e ",
|
||||
"ents": [{"start": 2, "end": 3, "label": "ORG"}],
|
||||
"sents": [{"start": 0, "end": 5}],
|
||||
"tokens": [
|
||||
{
|
||||
"id": 0,
|
||||
"start": 0,
|
||||
"end": 1,
|
||||
"tag": "VBP",
|
||||
"pos": "VERB",
|
||||
"morph": "Feat1=A",
|
||||
"dep": "ROOT",
|
||||
"head": 0,
|
||||
},
|
||||
{
|
||||
"id": 1,
|
||||
"start": 2,
|
||||
"end": 3,
|
||||
"tag": "NN",
|
||||
"pos": "NOUN",
|
||||
"morph": "Feat1=B",
|
||||
"dep": "dobj",
|
||||
"head": 0,
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"start": 4,
|
||||
"end": 5,
|
||||
"tag": "NN",
|
||||
"pos": "NOUN",
|
||||
"morph": "Feat1=A|Feat2=D",
|
||||
"dep": "dobj",
|
||||
"head": 1,
|
||||
},
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def test_doc_to_json(doc):
|
||||
json_doc = doc.to_json()
|
||||
assert json_doc["text"] == "c d e "
|
||||
|
|
@ -56,7 +101,8 @@ def test_doc_to_json(doc):
|
|||
assert json_doc["ents"][0]["start"] == 2 # character offset!
|
||||
assert json_doc["ents"][0]["end"] == 3 # character offset!
|
||||
assert json_doc["ents"][0]["label"] == "ORG"
|
||||
assert not schemas.validate(schemas.DocJSONSchema, json_doc)
|
||||
assert len(schemas.validate(schemas.DocJSONSchema, json_doc)) == 0
|
||||
assert srsly.json_loads(srsly.json_dumps(json_doc)) == json_doc
|
||||
|
||||
|
||||
def test_doc_to_json_underscore(doc):
|
||||
|
|
@ -64,11 +110,99 @@ def test_doc_to_json_underscore(doc):
|
|||
Doc.set_extension("json_test2", default=False)
|
||||
doc._.json_test1 = "hello world"
|
||||
doc._.json_test2 = [1, 2, 3]
|
||||
|
||||
json_doc = doc.to_json(underscore=["json_test1", "json_test2"])
|
||||
assert "_" in json_doc
|
||||
assert json_doc["_"]["json_test1"] == "hello world"
|
||||
assert json_doc["_"]["json_test2"] == [1, 2, 3]
|
||||
assert not schemas.validate(schemas.DocJSONSchema, json_doc)
|
||||
assert len(schemas.validate(schemas.DocJSONSchema, json_doc)) == 0
|
||||
assert srsly.json_loads(srsly.json_dumps(json_doc)) == json_doc
|
||||
|
||||
|
||||
def test_doc_to_json_with_token_span_attributes(doc):
|
||||
Doc.set_extension("json_test1", default=False)
|
||||
Doc.set_extension("json_test2", default=False)
|
||||
Token.set_extension("token_test", default=False)
|
||||
Span.set_extension("span_test", default=False)
|
||||
|
||||
doc._.json_test1 = "hello world"
|
||||
doc._.json_test2 = [1, 2, 3]
|
||||
doc[0:1]._.span_test = "span_attribute"
|
||||
doc[0:2]._.span_test = "span_attribute_2"
|
||||
doc[0]._.token_test = 117
|
||||
doc[1]._.token_test = 118
|
||||
doc.spans["span_group"] = [doc[0:1]]
|
||||
json_doc = doc.to_json(
|
||||
underscore=["json_test1", "json_test2", "token_test", "span_test"]
|
||||
)
|
||||
|
||||
assert "_" in json_doc
|
||||
assert json_doc["_"]["json_test1"] == "hello world"
|
||||
assert json_doc["_"]["json_test2"] == [1, 2, 3]
|
||||
assert "underscore_token" in json_doc
|
||||
assert "underscore_span" in json_doc
|
||||
assert json_doc["underscore_token"]["token_test"][0]["value"] == 117
|
||||
assert json_doc["underscore_token"]["token_test"][1]["value"] == 118
|
||||
assert json_doc["underscore_span"]["span_test"][0]["value"] == "span_attribute"
|
||||
assert json_doc["underscore_span"]["span_test"][1]["value"] == "span_attribute_2"
|
||||
assert len(schemas.validate(schemas.DocJSONSchema, json_doc)) == 0
|
||||
assert srsly.json_loads(srsly.json_dumps(json_doc)) == json_doc
|
||||
|
||||
|
||||
def test_doc_to_json_with_custom_user_data(doc):
|
||||
Doc.set_extension("json_test", default=False)
|
||||
Token.set_extension("token_test", default=False)
|
||||
Span.set_extension("span_test", default=False)
|
||||
|
||||
doc._.json_test = "hello world"
|
||||
doc[0:1]._.span_test = "span_attribute"
|
||||
doc[0]._.token_test = 117
|
||||
json_doc = doc.to_json(underscore=["json_test", "token_test", "span_test"])
|
||||
doc.user_data["user_data_test"] = 10
|
||||
doc.user_data[("user_data_test2", True)] = 10
|
||||
|
||||
assert "_" in json_doc
|
||||
assert json_doc["_"]["json_test"] == "hello world"
|
||||
assert "underscore_token" in json_doc
|
||||
assert "underscore_span" in json_doc
|
||||
assert json_doc["underscore_token"]["token_test"][0]["value"] == 117
|
||||
assert json_doc["underscore_span"]["span_test"][0]["value"] == "span_attribute"
|
||||
assert len(schemas.validate(schemas.DocJSONSchema, json_doc)) == 0
|
||||
assert srsly.json_loads(srsly.json_dumps(json_doc)) == json_doc
|
||||
|
||||
|
||||
def test_doc_to_json_with_token_span_same_identifier(doc):
|
||||
Doc.set_extension("my_ext", default=False)
|
||||
Token.set_extension("my_ext", default=False)
|
||||
Span.set_extension("my_ext", default=False)
|
||||
|
||||
doc._.my_ext = "hello world"
|
||||
doc[0:1]._.my_ext = "span_attribute"
|
||||
doc[0]._.my_ext = 117
|
||||
json_doc = doc.to_json(underscore=["my_ext"])
|
||||
|
||||
assert "_" in json_doc
|
||||
assert json_doc["_"]["my_ext"] == "hello world"
|
||||
assert "underscore_token" in json_doc
|
||||
assert "underscore_span" in json_doc
|
||||
assert json_doc["underscore_token"]["my_ext"][0]["value"] == 117
|
||||
assert json_doc["underscore_span"]["my_ext"][0]["value"] == "span_attribute"
|
||||
assert len(schemas.validate(schemas.DocJSONSchema, json_doc)) == 0
|
||||
assert srsly.json_loads(srsly.json_dumps(json_doc)) == json_doc
|
||||
|
||||
|
||||
def test_doc_to_json_with_token_attributes_missing(doc):
|
||||
Token.set_extension("token_test", default=False)
|
||||
Span.set_extension("span_test", default=False)
|
||||
|
||||
doc[0:1]._.span_test = "span_attribute"
|
||||
doc[0]._.token_test = 117
|
||||
json_doc = doc.to_json(underscore=["span_test"])
|
||||
|
||||
assert "underscore_span" in json_doc
|
||||
assert json_doc["underscore_span"]["span_test"][0]["value"] == "span_attribute"
|
||||
assert "underscore_token" not in json_doc
|
||||
assert len(schemas.validate(schemas.DocJSONSchema, json_doc)) == 0
|
||||
|
||||
|
||||
def test_doc_to_json_underscore_error_attr(doc):
|
||||
|
|
@ -94,11 +228,29 @@ def test_doc_to_json_span(doc):
|
|||
assert len(json_doc["spans"]) == 1
|
||||
assert len(json_doc["spans"]["test"]) == 2
|
||||
assert json_doc["spans"]["test"][0]["start"] == 0
|
||||
assert not schemas.validate(schemas.DocJSONSchema, json_doc)
|
||||
assert len(schemas.validate(schemas.DocJSONSchema, json_doc)) == 0
|
||||
|
||||
|
||||
def test_json_to_doc(doc):
|
||||
new_doc = Doc(doc.vocab).from_json(doc.to_json(), validate=True)
|
||||
json_doc = doc.to_json()
|
||||
json_doc = srsly.json_loads(srsly.json_dumps(json_doc))
|
||||
new_doc = Doc(doc.vocab).from_json(json_doc, validate=True)
|
||||
assert new_doc.text == doc.text == "c d e "
|
||||
assert len(new_doc) == len(doc) == 3
|
||||
assert new_doc[0].pos == doc[0].pos
|
||||
assert new_doc[0].tag == doc[0].tag
|
||||
assert new_doc[0].dep == doc[0].dep
|
||||
assert new_doc[0].head.idx == doc[0].head.idx
|
||||
assert new_doc[0].lemma == doc[0].lemma
|
||||
assert len(new_doc.ents) == 1
|
||||
assert new_doc.ents[0].start == 1
|
||||
assert new_doc.ents[0].end == 2
|
||||
assert new_doc.ents[0].label_ == "ORG"
|
||||
assert doc.to_bytes() == new_doc.to_bytes()
|
||||
|
||||
|
||||
def test_json_to_doc_compat(doc, doc_json):
|
||||
new_doc = Doc(doc.vocab).from_json(doc_json, validate=True)
|
||||
new_tokens = [token for token in new_doc]
|
||||
assert new_doc.text == doc.text == "c d e "
|
||||
assert len(new_tokens) == len([token for token in doc]) == 3
|
||||
|
|
@ -114,11 +266,8 @@ def test_json_to_doc(doc):
|
|||
|
||||
|
||||
def test_json_to_doc_underscore(doc):
|
||||
if not Doc.has_extension("json_test1"):
|
||||
Doc.set_extension("json_test1", default=False)
|
||||
if not Doc.has_extension("json_test2"):
|
||||
Doc.set_extension("json_test2", default=False)
|
||||
|
||||
Doc.set_extension("json_test1", default=False)
|
||||
Doc.set_extension("json_test2", default=False)
|
||||
doc._.json_test1 = "hello world"
|
||||
doc._.json_test2 = [1, 2, 3]
|
||||
json_doc = doc.to_json(underscore=["json_test1", "json_test2"])
|
||||
|
|
@ -126,6 +275,38 @@ def test_json_to_doc_underscore(doc):
|
|||
assert all([new_doc.has_extension(f"json_test{i}") for i in range(1, 3)])
|
||||
assert new_doc._.json_test1 == "hello world"
|
||||
assert new_doc._.json_test2 == [1, 2, 3]
|
||||
assert doc.to_bytes() == new_doc.to_bytes()
|
||||
|
||||
|
||||
def test_json_to_doc_with_token_span_attributes(doc):
|
||||
Doc.set_extension("json_test1", default=False)
|
||||
Doc.set_extension("json_test2", default=False)
|
||||
Token.set_extension("token_test", default=False)
|
||||
Span.set_extension("span_test", default=False)
|
||||
doc._.json_test1 = "hello world"
|
||||
doc._.json_test2 = [1, 2, 3]
|
||||
doc[0:1]._.span_test = "span_attribute"
|
||||
doc[0:2]._.span_test = "span_attribute_2"
|
||||
doc[0]._.token_test = 117
|
||||
doc[1]._.token_test = 118
|
||||
|
||||
json_doc = doc.to_json(
|
||||
underscore=["json_test1", "json_test2", "token_test", "span_test"]
|
||||
)
|
||||
json_doc = srsly.json_loads(srsly.json_dumps(json_doc))
|
||||
new_doc = Doc(doc.vocab).from_json(json_doc, validate=True)
|
||||
|
||||
assert all([new_doc.has_extension(f"json_test{i}") for i in range(1, 3)])
|
||||
assert new_doc._.json_test1 == "hello world"
|
||||
assert new_doc._.json_test2 == [1, 2, 3]
|
||||
assert new_doc[0]._.token_test == 117
|
||||
assert new_doc[1]._.token_test == 118
|
||||
assert new_doc[0:1]._.span_test == "span_attribute"
|
||||
assert new_doc[0:2]._.span_test == "span_attribute_2"
|
||||
assert new_doc.user_data == doc.user_data
|
||||
assert new_doc.to_bytes(exclude=["user_data"]) == doc.to_bytes(
|
||||
exclude=["user_data"]
|
||||
)
|
||||
|
||||
|
||||
def test_json_to_doc_spans(doc):
|
||||
|
|
|
|||
0
spacy/tests/lang/la/__init__.py
Normal file
0
spacy/tests/lang/la/__init__.py
Normal file
8
spacy/tests/lang/la/test_exception.py
Normal file
8
spacy/tests/lang/la/test_exception.py
Normal file
|
|
@ -0,0 +1,8 @@
|
|||
import pytest
|
||||
|
||||
|
||||
def test_la_tokenizer_handles_exc_in_text(la_tokenizer):
|
||||
text = "scio te omnia facturum, ut nobiscum quam primum sis"
|
||||
tokens = la_tokenizer(text)
|
||||
assert len(tokens) == 11
|
||||
assert tokens[6].text == "nobis"
|
||||
35
spacy/tests/lang/la/test_text.py
Normal file
35
spacy/tests/lang/la/test_text.py
Normal file
|
|
@ -0,0 +1,35 @@
|
|||
import pytest
|
||||
from spacy.lang.la.lex_attrs import like_num
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"text,match",
|
||||
[
|
||||
("IIII", True),
|
||||
("VI", True),
|
||||
("vi", True),
|
||||
("IV", True),
|
||||
("iv", True),
|
||||
("IX", True),
|
||||
("ix", True),
|
||||
("MMXXII", True),
|
||||
("0", True),
|
||||
("1", True),
|
||||
("quattuor", True),
|
||||
("decem", True),
|
||||
("tertius", True),
|
||||
("canis", False),
|
||||
("MMXX11", False),
|
||||
(",", False),
|
||||
],
|
||||
)
|
||||
def test_lex_attrs_like_number(la_tokenizer, text, match):
|
||||
tokens = la_tokenizer(text)
|
||||
assert len(tokens) == 1
|
||||
assert tokens[0].like_num == match
|
||||
|
||||
|
||||
@pytest.mark.parametrize("word", ["quinque"])
|
||||
def test_la_lex_attrs_capitals(word):
|
||||
assert like_num(word)
|
||||
assert like_num(word.upper())
|
||||
0
spacy/tests/lang/lg/__init__.py
Normal file
0
spacy/tests/lang/lg/__init__.py
Normal file
15
spacy/tests/lang/lg/test_tokenizer.py
Normal file
15
spacy/tests/lang/lg/test_tokenizer.py
Normal file
|
|
@ -0,0 +1,15 @@
|
|||
import pytest
|
||||
|
||||
LG_BASIC_TOKENIZATION_TESTS = [
|
||||
(
|
||||
"Abooluganda ab’emmamba ababiri",
|
||||
["Abooluganda", "ab’emmamba", "ababiri"],
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("text,expected_tokens", LG_BASIC_TOKENIZATION_TESTS)
|
||||
def test_lg_tokenizer_basic(lg_tokenizer, text, expected_tokens):
|
||||
tokens = lg_tokenizer(text)
|
||||
token_list = [token.text for token in tokens if not token.is_space]
|
||||
assert expected_tokens == token_list
|
||||
|
|
@ -1,5 +1,6 @@
|
|||
from spacy.tokens import Doc
|
||||
import pytest
|
||||
from spacy.tokens import Doc
|
||||
from spacy.util import filter_spans
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
|
|
@ -207,3 +208,18 @@ def test_chunking(nl_sample, nl_reference_chunking):
|
|||
"""
|
||||
chunks = [s.text.lower() for s in nl_sample.noun_chunks]
|
||||
assert chunks == nl_reference_chunking
|
||||
|
||||
|
||||
@pytest.mark.issue(10846)
|
||||
def test_no_overlapping_chunks(nl_vocab):
|
||||
# fmt: off
|
||||
doc = Doc(
|
||||
nl_vocab,
|
||||
words=["Dit", "programma", "wordt", "beschouwd", "als", "'s", "werelds", "eerste", "computerprogramma"],
|
||||
deps=["det", "nsubj:pass", "aux:pass", "ROOT", "mark", "det", "fixed", "amod", "xcomp"],
|
||||
heads=[1, 3, 3, 3, 8, 8, 5, 8, 3],
|
||||
pos=["DET", "NOUN", "AUX", "VERB", "SCONJ", "DET", "NOUN", "ADJ", "NOUN"],
|
||||
)
|
||||
# fmt: on
|
||||
chunks = list(doc.noun_chunks)
|
||||
assert filter_spans(chunks) == chunks
|
||||
|
|
|
|||
|
|
@ -2,6 +2,9 @@ import pytest
|
|||
from spacy.tokens import Doc
|
||||
|
||||
|
||||
pytestmark = pytest.mark.filterwarnings("ignore::DeprecationWarning")
|
||||
|
||||
|
||||
def test_ru_doc_lemmatization(ru_lemmatizer):
|
||||
words = ["мама", "мыла", "раму"]
|
||||
pos = ["NOUN", "VERB", "NOUN"]
|
||||
|
|
@ -75,3 +78,17 @@ def test_ru_lemmatizer_punct(ru_lemmatizer):
|
|||
assert ru_lemmatizer.pymorphy2_lemmatize(doc[0]) == ['"']
|
||||
doc = Doc(ru_lemmatizer.vocab, words=["»"], pos=["PUNCT"])
|
||||
assert ru_lemmatizer.pymorphy2_lemmatize(doc[0]) == ['"']
|
||||
|
||||
|
||||
def test_ru_doc_lookup_lemmatization(ru_lookup_lemmatizer):
|
||||
words = ["мама", "мыла", "раму"]
|
||||
pos = ["NOUN", "VERB", "NOUN"]
|
||||
morphs = [
|
||||
"Animacy=Anim|Case=Nom|Gender=Fem|Number=Sing",
|
||||
"Aspect=Imp|Gender=Fem|Mood=Ind|Number=Sing|Tense=Past|VerbForm=Fin|Voice=Act",
|
||||
"Animacy=Anim|Case=Acc|Gender=Fem|Number=Sing",
|
||||
]
|
||||
doc = Doc(ru_lookup_lemmatizer.vocab, words=words, pos=pos, morphs=morphs)
|
||||
doc = ru_lookup_lemmatizer(doc)
|
||||
lemmas = [token.lemma_ for token in doc]
|
||||
assert lemmas == ["мама", "мыла", "раму"]
|
||||
|
|
|
|||
|
|
@ -1,7 +1,19 @@
|
|||
import pytest
|
||||
from spacy.tokens import Doc
|
||||
|
||||
|
||||
pytestmark = pytest.mark.filterwarnings("ignore::DeprecationWarning")
|
||||
|
||||
|
||||
def test_uk_lemmatizer(uk_lemmatizer):
|
||||
"""Check that the default uk lemmatizer runs."""
|
||||
doc = Doc(uk_lemmatizer.vocab, words=["a", "b", "c"])
|
||||
uk_lemmatizer(doc)
|
||||
assert [token.lemma for token in doc]
|
||||
|
||||
|
||||
def test_uk_lookup_lemmatizer(uk_lookup_lemmatizer):
|
||||
"""Check that the lookup uk lemmatizer runs."""
|
||||
doc = Doc(uk_lookup_lemmatizer.vocab, words=["a", "b", "c"])
|
||||
uk_lookup_lemmatizer(doc)
|
||||
assert [token.lemma for token in doc]
|
||||
|
|
|
|||
|
|
@ -316,6 +316,20 @@ def test_dependency_matcher_precedence_ops(en_vocab, op, num_matches):
|
|||
("the", "brown", "$--", 0),
|
||||
("brown", "the", "$--", 1),
|
||||
("brown", "brown", "$--", 0),
|
||||
("quick", "fox", "<++", 1),
|
||||
("quick", "over", "<++", 0),
|
||||
("over", "jumped", "<++", 0),
|
||||
("the", "fox", "<++", 2),
|
||||
("brown", "fox", "<--", 0),
|
||||
("fox", "jumped", "<--", 0),
|
||||
("fox", "over", "<--", 1),
|
||||
("jumped", "over", ">++", 1),
|
||||
("fox", "lazy", ">++", 0),
|
||||
("over", "the", ">++", 0),
|
||||
("brown", "fox", ">--", 0),
|
||||
("fox", "brown", ">--", 1),
|
||||
("jumped", "fox", ">--", 1),
|
||||
("fox", "the", ">--", 2),
|
||||
],
|
||||
)
|
||||
def test_dependency_matcher_ops(en_vocab, doc, left, right, op, num_matches):
|
||||
|
|
|
|||
44
spacy/tests/matcher/test_levenshtein.py
Normal file
44
spacy/tests/matcher/test_levenshtein.py
Normal file
|
|
@ -0,0 +1,44 @@
|
|||
import pytest
|
||||
from spacy.matcher import levenshtein
|
||||
|
||||
|
||||
# empty string plus 10 random ASCII, 10 random unicode, and 2 random long tests
|
||||
# from polyleven
|
||||
@pytest.mark.parametrize(
|
||||
"dist,a,b",
|
||||
[
|
||||
(0, "", ""),
|
||||
(4, "bbcb", "caba"),
|
||||
(3, "abcb", "cacc"),
|
||||
(3, "aa", "ccc"),
|
||||
(1, "cca", "ccac"),
|
||||
(1, "aba", "aa"),
|
||||
(4, "bcbb", "abac"),
|
||||
(3, "acbc", "bba"),
|
||||
(3, "cbba", "a"),
|
||||
(2, "bcc", "ba"),
|
||||
(4, "aaa", "ccbb"),
|
||||
(3, "うあい", "いいうい"),
|
||||
(2, "あううい", "うあい"),
|
||||
(3, "いういい", "うううあ"),
|
||||
(2, "うい", "あいあ"),
|
||||
(2, "いあい", "いう"),
|
||||
(1, "いい", "あいい"),
|
||||
(3, "あうあ", "いいああ"),
|
||||
(4, "いあうう", "ううああ"),
|
||||
(3, "いあいい", "ういああ"),
|
||||
(3, "いいああ", "ううあう"),
|
||||
(
|
||||
166,
|
||||
"TCTGGGCACGGATTCGTCAGATTCCATGTCCATATTTGAGGCTCTTGCAGGCAAAATTTGGGCATGTGAACTCCTTATAGTCCCCGTGC",
|
||||
"ATATGGATTGGGGGCATTCAAAGATACGGTTTCCCTTTCTTCAGTTTCGCGCGGCGCACGTCCGGGTGCGAGCCAGTTCGTCTTACTCACATTGTCGACTTCACGAATCGCGCATGATGTGCTTAGCCTGTACTTACGAACGAACTTTCGGTCCAAATACATTCTATCAACACCGAGGTATCCGTGCCACACGCCGAAGCTCGACCGTGTTCGTTGAGAGGTGGAAATGGTAAAAGATGAACATAGTC",
|
||||
),
|
||||
(
|
||||
111,
|
||||
"GGTTCGGCCGAATTCATAGAGCGTGGTAGTCGACGGTATCCCGCCTGGTAGGGGCCCCTTCTACCTAGCGGAAGTTTGTCAGTACTCTATAACACGAGGGCCTCTCACACCCTAGATCGTCCAGCCACTCGAAGATCGCAGCACCCTTACAGAAAGGCATTAATGTTTCTCCTAGCACTTGTGCAATGGTGAAGGAGTGATG",
|
||||
"CGTAACACTTCGCGCTACTGGGCTGCAACGTCTTGGGCATACATGCAAGATTATCTAATGCAAGCTTGAGCCCCGCTTGCGGAATTTCCCTAATCGGGGTCCCTTCCTGTTACGATAAGGACGCGTGCACT",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_levenshtein(dist, a, b):
|
||||
assert levenshtein(a, b) == dist
|
||||
|
|
@ -368,6 +368,16 @@ def test_matcher_intersect_value_operator(en_vocab):
|
|||
doc[0]._.ext = ["A", "B"]
|
||||
assert len(matcher(doc)) == 1
|
||||
|
||||
# INTERSECTS matches nothing for iterables that aren't all str or int
|
||||
matcher = Matcher(en_vocab)
|
||||
pattern = [{"_": {"ext": {"INTERSECTS": ["Abx", "C"]}}}]
|
||||
matcher.add("M", [pattern])
|
||||
doc = Doc(en_vocab, words=["a", "b", "c"])
|
||||
doc[0]._.ext = [["Abx"], "B"]
|
||||
assert len(matcher(doc)) == 0
|
||||
doc[0]._.ext = ["Abx", "B"]
|
||||
assert len(matcher(doc)) == 1
|
||||
|
||||
# INTERSECTS with an empty pattern list matches nothing
|
||||
matcher = Matcher(en_vocab)
|
||||
pattern = [{"_": {"ext": {"INTERSECTS": []}}}]
|
||||
|
|
@ -476,14 +486,22 @@ def test_matcher_extension_set_membership(en_vocab):
|
|||
assert len(matches) == 0
|
||||
|
||||
|
||||
@pytest.mark.xfail(reason="IN predicate must handle sequence values in extensions")
|
||||
def test_matcher_extension_in_set_predicate(en_vocab):
|
||||
matcher = Matcher(en_vocab)
|
||||
Token.set_extension("ext", default=[])
|
||||
pattern = [{"_": {"ext": {"IN": ["A", "C"]}}}]
|
||||
matcher.add("M", [pattern])
|
||||
doc = Doc(en_vocab, words=["a", "b", "c"])
|
||||
|
||||
# The IN predicate expects an exact match between the
|
||||
# extension value and one of the pattern's values.
|
||||
doc[0]._.ext = ["A", "B"]
|
||||
assert len(matcher(doc)) == 0
|
||||
|
||||
doc[0]._.ext = ["A"]
|
||||
assert len(matcher(doc)) == 0
|
||||
|
||||
doc[0]._.ext = "A"
|
||||
assert len(matcher(doc)) == 1
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
import pytest
|
||||
import warnings
|
||||
import srsly
|
||||
from mock import Mock
|
||||
|
||||
|
|
@ -344,13 +345,13 @@ def test_phrase_matcher_validation(en_vocab):
|
|||
matcher.add("TEST1", [doc1])
|
||||
with pytest.warns(UserWarning):
|
||||
matcher.add("TEST2", [doc2])
|
||||
with pytest.warns(None) as record:
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter("error")
|
||||
matcher.add("TEST3", [doc3])
|
||||
assert not record.list
|
||||
matcher = PhraseMatcher(en_vocab, attr="POS", validate=True)
|
||||
with pytest.warns(None) as record:
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter("error")
|
||||
matcher.add("TEST4", [doc2])
|
||||
assert not record.list
|
||||
|
||||
|
||||
def test_attr_validation(en_vocab):
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@ def test_build_dependencies():
|
|||
"types-dataclasses",
|
||||
"types-mock",
|
||||
"types-requests",
|
||||
"types-setuptools",
|
||||
]
|
||||
# ignore language-specific packages that shouldn't be installed by all
|
||||
libs_ignore_setup = [
|
||||
|
|
|
|||
|
|
@ -1048,6 +1048,10 @@ def test_no_gold_ents(patterns):
|
|||
for eg in train_examples:
|
||||
eg.predicted = ruler(eg.predicted)
|
||||
|
||||
# Entity ruler is no longer needed (initialization below wipes out the
|
||||
# patterns and causes warnings)
|
||||
nlp.remove_pipe("entity_ruler")
|
||||
|
||||
def create_kb(vocab):
|
||||
# create artificial KB
|
||||
mykb = KnowledgeBase(vocab, entity_vector_length=vector_length)
|
||||
|
|
|
|||
|
|
@ -605,10 +605,35 @@ def test_update_with_annotates():
|
|||
assert results[component] == ""
|
||||
|
||||
|
||||
def test_load_disable_enable() -> None:
|
||||
"""
|
||||
Tests spacy.load() with dis-/enabling components.
|
||||
"""
|
||||
@pytest.mark.issue(11443)
|
||||
def test_enable_disable_conflict_with_config():
|
||||
"""Test conflict between enable/disable w.r.t. `nlp.disabled` set in the config."""
|
||||
nlp = English()
|
||||
nlp.add_pipe("tagger")
|
||||
nlp.add_pipe("senter")
|
||||
nlp.add_pipe("sentencizer")
|
||||
|
||||
with make_tempdir() as tmp_dir:
|
||||
nlp.to_disk(tmp_dir)
|
||||
# Expected to fail, as config and arguments conflict.
|
||||
with pytest.raises(ValueError):
|
||||
spacy.load(
|
||||
tmp_dir, enable=["tagger"], config={"nlp": {"disabled": ["senter"]}}
|
||||
)
|
||||
# Expected to succeed without warning due to the lack of a conflicting config option.
|
||||
spacy.load(tmp_dir, enable=["tagger"])
|
||||
# Expected to succeed with a warning, as disable=[] should override the config setting.
|
||||
with pytest.warns(UserWarning):
|
||||
spacy.load(
|
||||
tmp_dir,
|
||||
enable=["tagger"],
|
||||
disable=[],
|
||||
config={"nlp": {"disabled": ["senter"]}},
|
||||
)
|
||||
|
||||
|
||||
def test_load_disable_enable():
|
||||
"""Tests spacy.load() with dis-/enabling components."""
|
||||
|
||||
base_nlp = English()
|
||||
for pipe in ("sentencizer", "tagger", "parser"):
|
||||
|
|
@ -618,6 +643,7 @@ def test_load_disable_enable() -> None:
|
|||
base_nlp.to_disk(tmp_dir)
|
||||
to_disable = ["parser", "tagger"]
|
||||
to_enable = ["tagger", "parser"]
|
||||
single_str = "tagger"
|
||||
|
||||
# Setting only `disable`.
|
||||
nlp = spacy.load(tmp_dir, disable=to_disable)
|
||||
|
|
@ -632,6 +658,16 @@ def test_load_disable_enable() -> None:
|
|||
]
|
||||
)
|
||||
|
||||
# Loading with a string representing one component
|
||||
nlp = spacy.load(tmp_dir, exclude=single_str)
|
||||
assert single_str not in nlp.component_names
|
||||
|
||||
nlp = spacy.load(tmp_dir, disable=single_str)
|
||||
assert single_str in nlp.component_names
|
||||
assert single_str not in nlp.pipe_names
|
||||
assert nlp._disabled == {single_str}
|
||||
assert nlp.disabled == [single_str]
|
||||
|
||||
# Testing consistent enable/disable combination.
|
||||
nlp = spacy.load(
|
||||
tmp_dir,
|
||||
|
|
|
|||
|
|
@ -404,10 +404,11 @@ def test_serialize_pipeline_disable_enable():
|
|||
assert nlp3.component_names == ["ner", "tagger"]
|
||||
with make_tempdir() as d:
|
||||
nlp3.to_disk(d)
|
||||
nlp4 = spacy.load(d, disable=["ner"])
|
||||
assert nlp4.pipe_names == []
|
||||
with pytest.warns(UserWarning):
|
||||
nlp4 = spacy.load(d, disable=["ner"])
|
||||
assert nlp4.pipe_names == ["tagger"]
|
||||
assert nlp4.component_names == ["ner", "tagger"]
|
||||
assert nlp4.disabled == ["ner", "tagger"]
|
||||
assert nlp4.disabled == ["ner"]
|
||||
with make_tempdir() as d:
|
||||
nlp.to_disk(d)
|
||||
nlp5 = spacy.load(d, exclude=["tagger"])
|
||||
|
|
|
|||
|
|
@ -659,3 +659,36 @@ def test_multiprocessing_gpu_warning(nlp2, texts):
|
|||
# Trigger multi-processing.
|
||||
for _ in docs:
|
||||
pass
|
||||
|
||||
|
||||
def test_dot_in_factory_names(nlp):
|
||||
Language.component("my_evil_component", func=evil_component)
|
||||
nlp.add_pipe("my_evil_component")
|
||||
|
||||
with pytest.raises(ValueError, match="not permitted"):
|
||||
Language.component("my.evil.component.v1", func=evil_component)
|
||||
|
||||
with pytest.raises(ValueError, match="not permitted"):
|
||||
Language.factory("my.evil.component.v1", func=evil_component)
|
||||
|
||||
|
||||
def test_component_return():
|
||||
"""Test that an error is raised if components return a type other than a
|
||||
doc."""
|
||||
nlp = English()
|
||||
|
||||
@Language.component("test_component_good_pipe")
|
||||
def good_pipe(doc):
|
||||
return doc
|
||||
|
||||
nlp.add_pipe("test_component_good_pipe")
|
||||
nlp("text")
|
||||
nlp.remove_pipe("test_component_good_pipe")
|
||||
|
||||
@Language.component("test_component_bad_pipe")
|
||||
def bad_pipe(doc):
|
||||
return doc.text
|
||||
|
||||
nlp.add_pipe("test_component_bad_pipe")
|
||||
with pytest.raises(ValueError, match="instead of a Doc"):
|
||||
nlp("text")
|
||||
|
|
|
|||
|
|
@ -10,7 +10,8 @@ from spacy.ml._precomputable_affine import _backprop_precomputable_affine_paddin
|
|||
from spacy.util import dot_to_object, SimpleFrozenList, import_file
|
||||
from spacy.util import to_ternary_int
|
||||
from thinc.api import Config, Optimizer, ConfigValidationError
|
||||
from thinc.api import set_current_ops
|
||||
from thinc.api import get_current_ops, set_current_ops, NumpyOps, CupyOps, MPSOps
|
||||
from thinc.compat import has_cupy_gpu, has_torch_mps_gpu
|
||||
from spacy.training.batchers import minibatch_by_words
|
||||
from spacy.lang.en import English
|
||||
from spacy.lang.nl import Dutch
|
||||
|
|
@ -18,7 +19,6 @@ from spacy.language import DEFAULT_CONFIG_PATH
|
|||
from spacy.schemas import ConfigSchemaTraining, TokenPattern, TokenPatternSchema
|
||||
from pydantic import ValidationError
|
||||
|
||||
from thinc.api import get_current_ops, NumpyOps, CupyOps
|
||||
|
||||
from .util import get_random_doc, make_tempdir
|
||||
|
||||
|
|
@ -111,26 +111,25 @@ def test_PrecomputableAffine(nO=4, nI=5, nF=3, nP=2):
|
|||
|
||||
def test_prefer_gpu():
|
||||
current_ops = get_current_ops()
|
||||
try:
|
||||
import cupy # noqa: F401
|
||||
|
||||
prefer_gpu()
|
||||
if has_cupy_gpu:
|
||||
assert prefer_gpu()
|
||||
assert isinstance(get_current_ops(), CupyOps)
|
||||
except ImportError:
|
||||
elif has_torch_mps_gpu:
|
||||
assert prefer_gpu()
|
||||
assert isinstance(get_current_ops(), MPSOps)
|
||||
else:
|
||||
assert not prefer_gpu()
|
||||
set_current_ops(current_ops)
|
||||
|
||||
|
||||
def test_require_gpu():
|
||||
current_ops = get_current_ops()
|
||||
try:
|
||||
import cupy # noqa: F401
|
||||
|
||||
if has_cupy_gpu:
|
||||
require_gpu()
|
||||
assert isinstance(get_current_ops(), CupyOps)
|
||||
except ImportError:
|
||||
with pytest.raises(ValueError):
|
||||
require_gpu()
|
||||
elif has_torch_mps_gpu:
|
||||
require_gpu()
|
||||
assert isinstance(get_current_ops(), MPSOps)
|
||||
set_current_ops(current_ops)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -31,7 +31,7 @@ def doc(nlp):
|
|||
words = ["Sarah", "'s", "sister", "flew", "to", "Silicon", "Valley", "via", "London", "."]
|
||||
tags = ["NNP", "POS", "NN", "VBD", "IN", "NNP", "NNP", "IN", "NNP", "."]
|
||||
pos = ["PROPN", "PART", "NOUN", "VERB", "ADP", "PROPN", "PROPN", "ADP", "PROPN", "PUNCT"]
|
||||
ents = ["B-PERSON", "I-PERSON", "O", "O", "O", "B-LOC", "I-LOC", "O", "B-GPE", "O"]
|
||||
ents = ["B-PERSON", "I-PERSON", "O", "", "O", "B-LOC", "I-LOC", "O", "B-GPE", "O"]
|
||||
cats = {"TRAVEL": 1.0, "BAKING": 0.0}
|
||||
# fmt: on
|
||||
doc = Doc(nlp.vocab, words=words, tags=tags, pos=pos, ents=ents)
|
||||
|
|
@ -106,6 +106,7 @@ def test_lowercase_augmenter(nlp, doc):
|
|||
assert [(e.start, e.end, e.label) for e in eg.reference.ents] == ents
|
||||
for ref_ent, orig_ent in zip(eg.reference.ents, doc.ents):
|
||||
assert ref_ent.text == orig_ent.text.lower()
|
||||
assert [t.ent_iob for t in doc] == [t.ent_iob for t in eg.reference]
|
||||
assert [t.pos_ for t in eg.reference] == [t.pos_ for t in doc]
|
||||
|
||||
# check that augmentation works when lowercasing leads to different
|
||||
|
|
@ -166,7 +167,7 @@ def test_make_whitespace_variant(nlp):
|
|||
lemmas = ["they", "fly", "to", "New", "York", "City", ".", "\n", "then", "they", "drive", "to", "Washington", ",", "D.C."]
|
||||
heads = [1, 1, 1, 4, 5, 2, 1, 10, 10, 10, 10, 10, 11, 12, 12]
|
||||
deps = ["nsubj", "ROOT", "prep", "compound", "compound", "pobj", "punct", "dep", "advmod", "nsubj", "ROOT", "prep", "pobj", "punct", "appos"]
|
||||
ents = ["O", "O", "O", "B-GPE", "I-GPE", "I-GPE", "O", "O", "O", "O", "O", "O", "B-GPE", "O", "B-GPE"]
|
||||
ents = ["O", "", "O", "B-GPE", "I-GPE", "I-GPE", "O", "O", "O", "O", "O", "O", "B-GPE", "O", "B-GPE"]
|
||||
# fmt: on
|
||||
doc = Doc(
|
||||
nlp.vocab,
|
||||
|
|
@ -215,6 +216,8 @@ def test_make_whitespace_variant(nlp):
|
|||
assert mod_ex2.reference[j].head.i == j - 1
|
||||
# entities are well-formed
|
||||
assert len(doc.ents) == len(mod_ex.reference.ents)
|
||||
# there is one token with missing entity information
|
||||
assert any(t.ent_iob == 0 for t in mod_ex.reference)
|
||||
for ent in mod_ex.reference.ents:
|
||||
assert not ent[0].is_space
|
||||
assert not ent[-1].is_space
|
||||
|
|
|
|||
30
spacy/tests/training/test_logger.py
Normal file
30
spacy/tests/training/test_logger.py
Normal file
|
|
@ -0,0 +1,30 @@
|
|||
import pytest
|
||||
import spacy
|
||||
|
||||
from spacy.training import loggers
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def nlp():
|
||||
nlp = spacy.blank("en")
|
||||
nlp.add_pipe("ner")
|
||||
return nlp
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def info():
|
||||
return {
|
||||
"losses": {"ner": 100},
|
||||
"other_scores": {"ENTS_F": 0.85, "ENTS_P": 0.90, "ENTS_R": 0.80},
|
||||
"epoch": 100,
|
||||
"step": 125,
|
||||
"score": 85,
|
||||
}
|
||||
|
||||
|
||||
def test_console_logger(nlp, info):
|
||||
console_logger = loggers.console_logger(
|
||||
progress_bar=True, console_output=True, output_file=None
|
||||
)
|
||||
log_step, finalize = console_logger(nlp)
|
||||
log_step(info)
|
||||
|
|
@ -431,3 +431,41 @@ def test_Example_aligned_whitespace(en_vocab):
|
|||
|
||||
example = Example(predicted, reference)
|
||||
assert example.get_aligned("TAG", as_string=True) == tags
|
||||
|
||||
|
||||
@pytest.mark.issue("11260")
|
||||
def test_issue11260():
|
||||
annots = {
|
||||
"words": ["I", "like", "New", "York", "."],
|
||||
"spans": {
|
||||
"cities": [(7, 15, "LOC", "")],
|
||||
"people": [(0, 1, "PERSON", "")],
|
||||
},
|
||||
}
|
||||
vocab = Vocab()
|
||||
predicted = Doc(vocab, words=annots["words"])
|
||||
example = Example.from_dict(predicted, annots)
|
||||
assert len(example.reference.spans["cities"]) == 1
|
||||
assert len(example.reference.spans["people"]) == 1
|
||||
|
||||
output_dict = example.to_dict()
|
||||
assert "spans" in output_dict["doc_annotation"]
|
||||
assert output_dict["doc_annotation"]["spans"]["cities"] == annots["spans"]["cities"]
|
||||
assert output_dict["doc_annotation"]["spans"]["people"] == annots["spans"]["people"]
|
||||
|
||||
output_example = Example.from_dict(predicted, output_dict)
|
||||
|
||||
assert len(output_example.reference.spans["cities"]) == len(
|
||||
example.reference.spans["cities"]
|
||||
)
|
||||
assert len(output_example.reference.spans["people"]) == len(
|
||||
example.reference.spans["people"]
|
||||
)
|
||||
for span in example.reference.spans["cities"]:
|
||||
assert span.label_ == "LOC"
|
||||
assert span.text == "New York"
|
||||
assert span.start_char == 7
|
||||
for span in example.reference.spans["people"]:
|
||||
assert span.label_ == "PERSON"
|
||||
assert span.text == "I"
|
||||
assert span.start_char == 0
|
||||
|
|
|
|||
|
|
@ -72,7 +72,7 @@ class Doc:
|
|||
lemmas: Optional[List[str]] = ...,
|
||||
heads: Optional[List[int]] = ...,
|
||||
deps: Optional[List[str]] = ...,
|
||||
sent_starts: Optional[List[Union[bool, None]]] = ...,
|
||||
sent_starts: Optional[List[Union[bool, int, None]]] = ...,
|
||||
ents: Optional[List[str]] = ...,
|
||||
) -> None: ...
|
||||
@property
|
||||
|
|
|
|||
|
|
@ -217,9 +217,9 @@ cdef class Doc:
|
|||
head in the doc. Defaults to None.
|
||||
deps (Optional[List[str]]): A list of unicode strings, of the same
|
||||
length as words, to assign as token.dep. Defaults to None.
|
||||
sent_starts (Optional[List[Union[bool, None]]]): A list of values, of
|
||||
the same length as words, to assign as token.is_sent_start. Will be
|
||||
overridden by heads if heads is provided. Defaults to None.
|
||||
sent_starts (Optional[List[Union[bool, int, None]]]): A list of values,
|
||||
of the same length as words, to assign as token.is_sent_start. Will
|
||||
be overridden by heads if heads is provided. Defaults to None.
|
||||
ents (Optional[List[str]]): A list of unicode strings, of the same
|
||||
length as words, as IOB tags to assign as token.ent_iob and
|
||||
token.ent_type. Defaults to None.
|
||||
|
|
@ -285,6 +285,7 @@ cdef class Doc:
|
|||
heads = [0] * len(deps)
|
||||
if heads and not deps:
|
||||
raise ValueError(Errors.E1017)
|
||||
sent_starts = list(sent_starts) if sent_starts is not None else None
|
||||
if sent_starts is not None:
|
||||
for i in range(len(sent_starts)):
|
||||
if sent_starts[i] is True:
|
||||
|
|
@ -300,12 +301,11 @@ cdef class Doc:
|
|||
ent_iobs = None
|
||||
ent_types = None
|
||||
if ents is not None:
|
||||
ents = [ent if ent != "" else None for ent in ents]
|
||||
iob_strings = Token.iob_strings()
|
||||
# make valid IOB2 out of IOB1 or IOB2
|
||||
for i, ent in enumerate(ents):
|
||||
if ent is "":
|
||||
ents[i] = None
|
||||
elif ent is not None and not isinstance(ent, str):
|
||||
if ent is not None and not isinstance(ent, str):
|
||||
raise ValueError(Errors.E177.format(tag=ent))
|
||||
if i < len(ents) - 1:
|
||||
# OI -> OB
|
||||
|
|
@ -1602,13 +1602,26 @@ cdef class Doc:
|
|||
ents.append(char_span)
|
||||
self.ents = ents
|
||||
|
||||
# Add custom attributes. Note that only Doc extensions are currently considered, Token and Span extensions are
|
||||
# not yet supported.
|
||||
# Add custom attributes for the whole Doc object.
|
||||
for attr in doc_json.get("_", {}):
|
||||
if not Doc.has_extension(attr):
|
||||
Doc.set_extension(attr)
|
||||
self._.set(attr, doc_json["_"][attr])
|
||||
|
||||
for token_attr in doc_json.get("underscore_token", {}):
|
||||
if not Token.has_extension(token_attr):
|
||||
Token.set_extension(token_attr)
|
||||
for token_data in doc_json["underscore_token"][token_attr]:
|
||||
start = token_by_char(self.c, self.length, token_data["start"])
|
||||
value = token_data["value"]
|
||||
self[start]._.set(token_attr, value)
|
||||
|
||||
for span_attr in doc_json.get("underscore_span", {}):
|
||||
if not Span.has_extension(span_attr):
|
||||
Span.set_extension(span_attr)
|
||||
for span_data in doc_json["underscore_span"][span_attr]:
|
||||
value = span_data["value"]
|
||||
self.char_span(span_data["start"], span_data["end"])._.set(span_attr, value)
|
||||
return self
|
||||
|
||||
def to_json(self, underscore=None):
|
||||
|
|
@ -1650,20 +1663,44 @@ cdef class Doc:
|
|||
for span_group in self.spans:
|
||||
data["spans"][span_group] = []
|
||||
for span in self.spans[span_group]:
|
||||
span_data = {
|
||||
"start": span.start_char, "end": span.end_char, "label": span.label_, "kb_id": span.kb_id_
|
||||
}
|
||||
span_data = {"start": span.start_char, "end": span.end_char, "label": span.label_, "kb_id": span.kb_id_}
|
||||
data["spans"][span_group].append(span_data)
|
||||
|
||||
if underscore:
|
||||
data["_"] = {}
|
||||
user_keys = set()
|
||||
if self.user_data:
|
||||
for data_key, value in self.user_data.copy().items():
|
||||
if type(data_key) == tuple and len(data_key) >= 4 and data_key[0] == "._.":
|
||||
attr = data_key[1]
|
||||
start = data_key[2]
|
||||
end = data_key[3]
|
||||
if attr in underscore:
|
||||
user_keys.add(attr)
|
||||
if not srsly.is_json_serializable(value):
|
||||
raise ValueError(Errors.E107.format(attr=attr, value=repr(value)))
|
||||
# Check if doc attribute
|
||||
if start is None:
|
||||
if "_" not in data:
|
||||
data["_"] = {}
|
||||
data["_"][attr] = value
|
||||
# Check if token attribute
|
||||
elif end is None:
|
||||
if "underscore_token" not in data:
|
||||
data["underscore_token"] = {}
|
||||
if attr not in data["underscore_token"]:
|
||||
data["underscore_token"][attr] = []
|
||||
data["underscore_token"][attr].append({"start": start, "value": value})
|
||||
# Else span attribute
|
||||
else:
|
||||
if "underscore_span" not in data:
|
||||
data["underscore_span"] = {}
|
||||
if attr not in data["underscore_span"]:
|
||||
data["underscore_span"][attr] = []
|
||||
data["underscore_span"][attr].append({"start": start, "end": end, "value": value})
|
||||
|
||||
for attr in underscore:
|
||||
if not self.has_extension(attr):
|
||||
if attr not in user_keys:
|
||||
raise ValueError(Errors.E106.format(attr=attr, opts=underscore))
|
||||
value = self._.get(attr)
|
||||
if not srsly.is_json_serializable(value):
|
||||
raise ValueError(Errors.E107.format(attr=attr, value=repr(value)))
|
||||
data["_"][attr] = value
|
||||
return data
|
||||
|
||||
def to_utf8_array(self, int nr_char=-1):
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ from functools import partial
|
|||
|
||||
from ..util import registry
|
||||
from .example import Example
|
||||
from .iob_utils import split_bilu_label
|
||||
from .iob_utils import split_bilu_label, _doc_to_biluo_tags_with_partial
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from ..language import Language # noqa: F401
|
||||
|
|
@ -62,6 +62,9 @@ def combined_augmenter(
|
|||
if orth_variants and random.random() < orth_level:
|
||||
raw_text = example.text
|
||||
orig_dict = example.to_dict()
|
||||
orig_dict["doc_annotation"]["entities"] = _doc_to_biluo_tags_with_partial(
|
||||
example.reference
|
||||
)
|
||||
variant_text, variant_token_annot = make_orth_variants(
|
||||
nlp,
|
||||
raw_text,
|
||||
|
|
@ -128,6 +131,9 @@ def lower_casing_augmenter(
|
|||
|
||||
def make_lowercase_variant(nlp: "Language", example: Example):
|
||||
example_dict = example.to_dict()
|
||||
example_dict["doc_annotation"]["entities"] = _doc_to_biluo_tags_with_partial(
|
||||
example.reference
|
||||
)
|
||||
doc = nlp.make_doc(example.text.lower())
|
||||
example_dict["token_annotation"]["ORTH"] = [t.lower_ for t in example.reference]
|
||||
return example.from_dict(doc, example_dict)
|
||||
|
|
@ -146,6 +152,9 @@ def orth_variants_augmenter(
|
|||
else:
|
||||
raw_text = example.text
|
||||
orig_dict = example.to_dict()
|
||||
orig_dict["doc_annotation"]["entities"] = _doc_to_biluo_tags_with_partial(
|
||||
example.reference
|
||||
)
|
||||
variant_text, variant_token_annot = make_orth_variants(
|
||||
nlp,
|
||||
raw_text,
|
||||
|
|
@ -248,6 +257,9 @@ def make_whitespace_variant(
|
|||
RETURNS (Example): Example with one additional space token.
|
||||
"""
|
||||
example_dict = example.to_dict()
|
||||
example_dict["doc_annotation"]["entities"] = _doc_to_biluo_tags_with_partial(
|
||||
example.reference
|
||||
)
|
||||
doc_dict = example_dict.get("doc_annotation", {})
|
||||
token_dict = example_dict.get("token_annotation", {})
|
||||
# returned unmodified if:
|
||||
|
|
|
|||
|
|
@ -361,6 +361,7 @@ cdef class Example:
|
|||
"doc_annotation": {
|
||||
"cats": dict(self.reference.cats),
|
||||
"entities": doc_to_biluo_tags(self.reference),
|
||||
"spans": self._spans_to_dict(),
|
||||
"links": self._links_to_dict()
|
||||
},
|
||||
"token_annotation": {
|
||||
|
|
@ -376,6 +377,18 @@ cdef class Example:
|
|||
}
|
||||
}
|
||||
|
||||
def _spans_to_dict(self):
|
||||
span_dict = {}
|
||||
for key in self.reference.spans:
|
||||
span_tuples = []
|
||||
for span in self.reference.spans[key]:
|
||||
span_tuple = (span.start_char, span.end_char, span.label_, span.kb_id_)
|
||||
span_tuples.append(span_tuple)
|
||||
span_dict[key] = span_tuples
|
||||
|
||||
return span_dict
|
||||
|
||||
|
||||
def _links_to_dict(self):
|
||||
links = {}
|
||||
for ent in self.reference.ents:
|
||||
|
|
|
|||
|
|
@ -337,3 +337,5 @@ def ensure_shape(vectors_loc):
|
|||
# store all the results in a list in memory
|
||||
lines2 = open_file(vectors_loc)
|
||||
yield from lines2
|
||||
lines2.close()
|
||||
lines.close()
|
||||
|
|
|
|||
|
|
@ -60,6 +60,14 @@ def doc_to_biluo_tags(doc: Doc, missing: str = "O"):
|
|||
)
|
||||
|
||||
|
||||
def _doc_to_biluo_tags_with_partial(doc: Doc) -> List[str]:
|
||||
ents = doc_to_biluo_tags(doc, missing="-")
|
||||
for i, token in enumerate(doc):
|
||||
if token.ent_iob == 2:
|
||||
ents[i] = "O"
|
||||
return ents
|
||||
|
||||
|
||||
def offsets_to_biluo_tags(
|
||||
doc: Doc, entities: Iterable[Tuple[int, int, Union[str, int]]], missing: str = "O"
|
||||
) -> List[str]:
|
||||
|
|
|
|||
|
|
@ -1,10 +1,13 @@
|
|||
from typing import TYPE_CHECKING, Dict, Any, Tuple, Callable, List, Optional, IO
|
||||
from typing import TYPE_CHECKING, Dict, Any, Tuple, Callable, List, Optional, IO, Union
|
||||
from wasabi import Printer
|
||||
from pathlib import Path
|
||||
import tqdm
|
||||
import sys
|
||||
import srsly
|
||||
|
||||
from ..util import registry
|
||||
from ..errors import Errors
|
||||
from .. import util
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from ..language import Language # noqa: F401
|
||||
|
|
@ -23,13 +26,44 @@ def setup_table(
|
|||
return final_cols, final_widths, ["r" for _ in final_widths]
|
||||
|
||||
|
||||
@registry.loggers("spacy.ConsoleLogger.v1")
|
||||
def console_logger(progress_bar: bool = False):
|
||||
@registry.loggers("spacy.ConsoleLogger.v2")
|
||||
def console_logger(
|
||||
progress_bar: bool = False,
|
||||
console_output: bool = True,
|
||||
output_file: Optional[Union[str, Path]] = None,
|
||||
):
|
||||
"""The ConsoleLogger.v2 prints out training logs in the console and/or saves them to a jsonl file.
|
||||
progress_bar (bool): Whether the logger should print the progress bar.
|
||||
console_output (bool): Whether the logger should print the logs on the console.
|
||||
output_file (Optional[Union[str, Path]]): The file to save the training logs to.
|
||||
"""
|
||||
_log_exist = False
|
||||
if output_file:
|
||||
output_file = util.ensure_path(output_file) # type: ignore
|
||||
if output_file.exists(): # type: ignore
|
||||
_log_exist = True
|
||||
if not output_file.parents[0].exists(): # type: ignore
|
||||
output_file.parents[0].mkdir(parents=True) # type: ignore
|
||||
|
||||
def setup_printer(
|
||||
nlp: "Language", stdout: IO = sys.stdout, stderr: IO = sys.stderr
|
||||
) -> Tuple[Callable[[Optional[Dict[str, Any]]], None], Callable[[], None]]:
|
||||
write = lambda text: print(text, file=stdout, flush=True)
|
||||
msg = Printer(no_print=True)
|
||||
|
||||
nonlocal output_file
|
||||
output_stream = None
|
||||
if _log_exist:
|
||||
write(
|
||||
msg.warn(
|
||||
f"Saving logs is disabled because {output_file} already exists."
|
||||
)
|
||||
)
|
||||
output_file = None
|
||||
elif output_file:
|
||||
write(msg.info(f"Saving results to {output_file}"))
|
||||
output_stream = open(output_file, "w", encoding="utf-8")
|
||||
|
||||
# ensure that only trainable components are logged
|
||||
logged_pipes = [
|
||||
name
|
||||
|
|
@ -40,13 +74,15 @@ def console_logger(progress_bar: bool = False):
|
|||
score_weights = nlp.config["training"]["score_weights"]
|
||||
score_cols = [col for col, value in score_weights.items() if value is not None]
|
||||
loss_cols = [f"Loss {pipe}" for pipe in logged_pipes]
|
||||
spacing = 2
|
||||
table_header, table_widths, table_aligns = setup_table(
|
||||
cols=["E", "#"] + loss_cols + score_cols + ["Score"],
|
||||
widths=[3, 6] + [8 for _ in loss_cols] + [6 for _ in score_cols] + [6],
|
||||
)
|
||||
write(msg.row(table_header, widths=table_widths, spacing=spacing))
|
||||
write(msg.row(["-" * width for width in table_widths], spacing=spacing))
|
||||
|
||||
if console_output:
|
||||
spacing = 2
|
||||
table_header, table_widths, table_aligns = setup_table(
|
||||
cols=["E", "#"] + loss_cols + score_cols + ["Score"],
|
||||
widths=[3, 6] + [8 for _ in loss_cols] + [6 for _ in score_cols] + [6],
|
||||
)
|
||||
write(msg.row(table_header, widths=table_widths, spacing=spacing))
|
||||
write(msg.row(["-" * width for width in table_widths], spacing=spacing))
|
||||
progress = None
|
||||
|
||||
def log_step(info: Optional[Dict[str, Any]]) -> None:
|
||||
|
|
@ -57,12 +93,15 @@ def console_logger(progress_bar: bool = False):
|
|||
if progress is not None:
|
||||
progress.update(1)
|
||||
return
|
||||
losses = [
|
||||
"{0:.2f}".format(float(info["losses"][pipe_name]))
|
||||
for pipe_name in logged_pipes
|
||||
]
|
||||
|
||||
losses = []
|
||||
log_losses = {}
|
||||
for pipe_name in logged_pipes:
|
||||
losses.append("{0:.2f}".format(float(info["losses"][pipe_name])))
|
||||
log_losses[pipe_name] = float(info["losses"][pipe_name])
|
||||
|
||||
scores = []
|
||||
log_scores = {}
|
||||
for col in score_cols:
|
||||
score = info["other_scores"].get(col, 0.0)
|
||||
try:
|
||||
|
|
@ -73,6 +112,7 @@ def console_logger(progress_bar: bool = False):
|
|||
if col != "speed":
|
||||
score *= 100
|
||||
scores.append("{0:.2f}".format(score))
|
||||
log_scores[str(col)] = score
|
||||
|
||||
data = (
|
||||
[info["epoch"], info["step"]]
|
||||
|
|
@ -80,20 +120,36 @@ def console_logger(progress_bar: bool = False):
|
|||
+ scores
|
||||
+ ["{0:.2f}".format(float(info["score"]))]
|
||||
)
|
||||
|
||||
if output_stream:
|
||||
# Write to log file per log_step
|
||||
log_data = {
|
||||
"epoch": info["epoch"],
|
||||
"step": info["step"],
|
||||
"losses": log_losses,
|
||||
"scores": log_scores,
|
||||
"score": float(info["score"]),
|
||||
}
|
||||
output_stream.write(srsly.json_dumps(log_data) + "\n")
|
||||
|
||||
if progress is not None:
|
||||
progress.close()
|
||||
write(
|
||||
msg.row(data, widths=table_widths, aligns=table_aligns, spacing=spacing)
|
||||
)
|
||||
if progress_bar:
|
||||
# Set disable=None, so that it disables on non-TTY
|
||||
progress = tqdm.tqdm(
|
||||
total=eval_frequency, disable=None, leave=False, file=stderr
|
||||
if console_output:
|
||||
write(
|
||||
msg.row(
|
||||
data, widths=table_widths, aligns=table_aligns, spacing=spacing
|
||||
)
|
||||
)
|
||||
progress.set_description(f"Epoch {info['epoch']+1}")
|
||||
if progress_bar:
|
||||
# Set disable=None, so that it disables on non-TTY
|
||||
progress = tqdm.tqdm(
|
||||
total=eval_frequency, disable=None, leave=False, file=stderr
|
||||
)
|
||||
progress.set_description(f"Epoch {info['epoch']+1}")
|
||||
|
||||
def finalize() -> None:
|
||||
pass
|
||||
if output_stream:
|
||||
output_stream.close()
|
||||
|
||||
return log_step, finalize
|
||||
|
||||
|
|
|
|||
|
|
@ -67,7 +67,6 @@ LEXEME_NORM_LANGS = ["cs", "da", "de", "el", "en", "id", "lb", "mk", "pt", "ru",
|
|||
CONFIG_SECTION_ORDER = ["paths", "variables", "system", "nlp", "components", "corpora", "training", "pretraining", "initialize"]
|
||||
# fmt: on
|
||||
|
||||
|
||||
logger = logging.getLogger("spacy")
|
||||
logger_stream_handler = logging.StreamHandler()
|
||||
logger_stream_handler.setFormatter(
|
||||
|
|
@ -394,13 +393,17 @@ def get_module_path(module: ModuleType) -> Path:
|
|||
return file_path.parent
|
||||
|
||||
|
||||
# Default value for passed enable/disable values.
|
||||
_DEFAULT_EMPTY_PIPES = SimpleFrozenList()
|
||||
|
||||
|
||||
def load_model(
|
||||
name: Union[str, Path],
|
||||
*,
|
||||
vocab: Union["Vocab", bool] = True,
|
||||
disable: Iterable[str] = SimpleFrozenList(),
|
||||
enable: Iterable[str] = SimpleFrozenList(),
|
||||
exclude: Iterable[str] = SimpleFrozenList(),
|
||||
disable: Union[str, Iterable[str]] = _DEFAULT_EMPTY_PIPES,
|
||||
enable: Union[str, Iterable[str]] = _DEFAULT_EMPTY_PIPES,
|
||||
exclude: Union[str, Iterable[str]] = _DEFAULT_EMPTY_PIPES,
|
||||
config: Union[Dict[str, Any], Config] = SimpleFrozenDict(),
|
||||
) -> "Language":
|
||||
"""Load a model from a package or data path.
|
||||
|
|
@ -408,9 +411,9 @@ def load_model(
|
|||
name (str): Package name or model path.
|
||||
vocab (Vocab / True): Optional vocab to pass in on initialization. If True,
|
||||
a new Vocab object will be created.
|
||||
disable (Iterable[str]): Names of pipeline components to disable.
|
||||
enable (Iterable[str]): Names of pipeline components to enable. All others will be disabled.
|
||||
exclude (Iterable[str]): Names of pipeline components to exclude.
|
||||
disable (Union[str, Iterable[str]]): Name(s) of pipeline component(s) to disable.
|
||||
enable (Union[str, Iterable[str]]): Name(s) of pipeline component(s) to enable. All others will be disabled.
|
||||
exclude (Union[str, Iterable[str]]): Name(s) of pipeline component(s) to exclude.
|
||||
config (Dict[str, Any] / Config): Config overrides as nested dict or dict
|
||||
keyed by section values in dot notation.
|
||||
RETURNS (Language): The loaded nlp object.
|
||||
|
|
@ -440,9 +443,9 @@ def load_model_from_package(
|
|||
name: str,
|
||||
*,
|
||||
vocab: Union["Vocab", bool] = True,
|
||||
disable: Iterable[str] = SimpleFrozenList(),
|
||||
enable: Iterable[str] = SimpleFrozenList(),
|
||||
exclude: Iterable[str] = SimpleFrozenList(),
|
||||
disable: Union[str, Iterable[str]] = SimpleFrozenList(),
|
||||
enable: Union[str, Iterable[str]] = SimpleFrozenList(),
|
||||
exclude: Union[str, Iterable[str]] = SimpleFrozenList(),
|
||||
config: Union[Dict[str, Any], Config] = SimpleFrozenDict(),
|
||||
) -> "Language":
|
||||
"""Load a model from an installed package.
|
||||
|
|
@ -450,12 +453,12 @@ def load_model_from_package(
|
|||
name (str): The package name.
|
||||
vocab (Vocab / True): Optional vocab to pass in on initialization. If True,
|
||||
a new Vocab object will be created.
|
||||
disable (Iterable[str]): Names of pipeline components to disable. Disabled
|
||||
disable (Union[str, Iterable[str]]): Name(s) of pipeline component(s) to disable. Disabled
|
||||
pipes will be loaded but they won't be run unless you explicitly
|
||||
enable them by calling nlp.enable_pipe.
|
||||
enable (Iterable[str]): Names of pipeline components to enable. All other
|
||||
enable (Union[str, Iterable[str]]): Name(s) of pipeline component(s) to enable. All other
|
||||
pipes will be disabled (and can be enabled using `nlp.enable_pipe`).
|
||||
exclude (Iterable[str]): Names of pipeline components to exclude. Excluded
|
||||
exclude (Union[str, Iterable[str]]): Name(s) of pipeline component(s) to exclude. Excluded
|
||||
components won't be loaded.
|
||||
config (Dict[str, Any] / Config): Config overrides as nested dict or dict
|
||||
keyed by section values in dot notation.
|
||||
|
|
@ -470,9 +473,9 @@ def load_model_from_path(
|
|||
*,
|
||||
meta: Optional[Dict[str, Any]] = None,
|
||||
vocab: Union["Vocab", bool] = True,
|
||||
disable: Iterable[str] = SimpleFrozenList(),
|
||||
enable: Iterable[str] = SimpleFrozenList(),
|
||||
exclude: Iterable[str] = SimpleFrozenList(),
|
||||
disable: Union[str, Iterable[str]] = _DEFAULT_EMPTY_PIPES,
|
||||
enable: Union[str, Iterable[str]] = _DEFAULT_EMPTY_PIPES,
|
||||
exclude: Union[str, Iterable[str]] = _DEFAULT_EMPTY_PIPES,
|
||||
config: Union[Dict[str, Any], Config] = SimpleFrozenDict(),
|
||||
) -> "Language":
|
||||
"""Load a model from a data directory path. Creates Language class with
|
||||
|
|
@ -482,12 +485,12 @@ def load_model_from_path(
|
|||
meta (Dict[str, Any]): Optional model meta.
|
||||
vocab (Vocab / True): Optional vocab to pass in on initialization. If True,
|
||||
a new Vocab object will be created.
|
||||
disable (Iterable[str]): Names of pipeline components to disable. Disabled
|
||||
disable (Union[str, Iterable[str]]): Name(s) of pipeline component(s) to disable. Disabled
|
||||
pipes will be loaded but they won't be run unless you explicitly
|
||||
enable them by calling nlp.enable_pipe.
|
||||
enable (Iterable[str]): Names of pipeline components to enable. All other
|
||||
enable (Union[str, Iterable[str]]): Name(s) of pipeline component(s) to enable. All other
|
||||
pipes will be disabled (and can be enabled using `nlp.enable_pipe`).
|
||||
exclude (Iterable[str]): Names of pipeline components to exclude. Excluded
|
||||
exclude (Union[str, Iterable[str]]): Name(s) of pipeline component(s) to exclude. Excluded
|
||||
components won't be loaded.
|
||||
config (Dict[str, Any] / Config): Config overrides as nested dict or dict
|
||||
keyed by section values in dot notation.
|
||||
|
|
@ -516,9 +519,9 @@ def load_model_from_config(
|
|||
*,
|
||||
meta: Dict[str, Any] = SimpleFrozenDict(),
|
||||
vocab: Union["Vocab", bool] = True,
|
||||
disable: Iterable[str] = SimpleFrozenList(),
|
||||
enable: Iterable[str] = SimpleFrozenList(),
|
||||
exclude: Iterable[str] = SimpleFrozenList(),
|
||||
disable: Union[str, Iterable[str]] = _DEFAULT_EMPTY_PIPES,
|
||||
enable: Union[str, Iterable[str]] = _DEFAULT_EMPTY_PIPES,
|
||||
exclude: Union[str, Iterable[str]] = _DEFAULT_EMPTY_PIPES,
|
||||
auto_fill: bool = False,
|
||||
validate: bool = True,
|
||||
) -> "Language":
|
||||
|
|
@ -529,12 +532,12 @@ def load_model_from_config(
|
|||
meta (Dict[str, Any]): Optional model meta.
|
||||
vocab (Vocab / True): Optional vocab to pass in on initialization. If True,
|
||||
a new Vocab object will be created.
|
||||
disable (Iterable[str]): Names of pipeline components to disable. Disabled
|
||||
disable (Union[str, Iterable[str]]): Name(s) of pipeline component(s) to disable. Disabled
|
||||
pipes will be loaded but they won't be run unless you explicitly
|
||||
enable them by calling nlp.enable_pipe.
|
||||
enable (Iterable[str]): Names of pipeline components to enable. All other
|
||||
enable (Union[str, Iterable[str]]): Name(s) of pipeline component(s) to enable. All other
|
||||
pipes will be disabled (and can be enabled using `nlp.enable_pipe`).
|
||||
exclude (Iterable[str]): Names of pipeline components to exclude. Excluded
|
||||
exclude (Union[str, Iterable[str]]): Name(s) of pipeline component(s) to exclude. Excluded
|
||||
components won't be loaded.
|
||||
auto_fill (bool): Whether to auto-fill config with missing defaults.
|
||||
validate (bool): Whether to show config validation errors.
|
||||
|
|
@ -616,9 +619,9 @@ def load_model_from_init_py(
|
|||
init_file: Union[Path, str],
|
||||
*,
|
||||
vocab: Union["Vocab", bool] = True,
|
||||
disable: Iterable[str] = SimpleFrozenList(),
|
||||
enable: Iterable[str] = SimpleFrozenList(),
|
||||
exclude: Iterable[str] = SimpleFrozenList(),
|
||||
disable: Union[str, Iterable[str]] = SimpleFrozenList(),
|
||||
enable: Union[str, Iterable[str]] = SimpleFrozenList(),
|
||||
exclude: Union[str, Iterable[str]] = SimpleFrozenList(),
|
||||
config: Union[Dict[str, Any], Config] = SimpleFrozenDict(),
|
||||
) -> "Language":
|
||||
"""Helper function to use in the `load()` method of a model package's
|
||||
|
|
@ -626,12 +629,12 @@ def load_model_from_init_py(
|
|||
|
||||
vocab (Vocab / True): Optional vocab to pass in on initialization. If True,
|
||||
a new Vocab object will be created.
|
||||
disable (Iterable[str]): Names of pipeline components to disable. Disabled
|
||||
disable (Union[str, Iterable[str]]): Name(s) of pipeline component(s) to disable. Disabled
|
||||
pipes will be loaded but they won't be run unless you explicitly
|
||||
enable them by calling nlp.enable_pipe.
|
||||
enable (Iterable[str]): Names of pipeline components to enable. All other
|
||||
enable (Union[str, Iterable[str]]): Name(s) of pipeline component(s) to enable. All other
|
||||
pipes will be disabled (and can be enabled using `nlp.enable_pipe`).
|
||||
exclude (Iterable[str]): Names of pipeline components to exclude. Excluded
|
||||
exclude (Union[str, Iterable[str]]): Name(s) of pipeline component(s) to exclude. Excluded
|
||||
components won't be loaded.
|
||||
config (Dict[str, Any] / Config): Config overrides as nested dict or dict
|
||||
keyed by section values in dot notation.
|
||||
|
|
@ -795,6 +798,15 @@ def get_model_lower_version(constraint: str) -> Optional[str]:
|
|||
return None
|
||||
|
||||
|
||||
def is_prerelease_version(version: str) -> bool:
|
||||
"""Check whether a version is a prerelease version.
|
||||
|
||||
version (str): The version, e.g. "3.0.0.dev1".
|
||||
RETURNS (bool): Whether the version is a prerelease version.
|
||||
"""
|
||||
return Version(version).is_prerelease
|
||||
|
||||
|
||||
def get_base_version(version: str) -> str:
|
||||
"""Generate the base version without any prerelease identifiers.
|
||||
|
||||
|
|
|
|||
|
|
@ -77,14 +77,15 @@ $ python -m spacy info [--markdown] [--silent] [--exclude]
|
|||
$ python -m spacy info [model] [--markdown] [--silent] [--exclude]
|
||||
```
|
||||
|
||||
| Name | Description |
|
||||
| ------------------------------------------------ | --------------------------------------------------------------------------------------------- |
|
||||
| `model` | A trained pipeline, i.e. package name or path (optional). ~~Optional[str] \(option)~~ |
|
||||
| `--markdown`, `-md` | Print information as Markdown. ~~bool (flag)~~ |
|
||||
| `--silent`, `-s` <Tag variant="new">2.0.12</Tag> | Don't print anything, just return the values. ~~bool (flag)~~ |
|
||||
| `--exclude`, `-e` | Comma-separated keys to exclude from the print-out. Defaults to `"labels"`. ~~Optional[str]~~ |
|
||||
| `--help`, `-h` | Show help message and available arguments. ~~bool (flag)~~ |
|
||||
| **PRINTS** | Information about your spaCy installation. |
|
||||
| Name | Description |
|
||||
| ------------------------------------------------ | ----------------------------------------------------------------------------------------------------------------------- |
|
||||
| `model` | A trained pipeline, i.e. package name or path (optional). ~~Optional[str] \(option)~~ |
|
||||
| `--markdown`, `-md` | Print information as Markdown. ~~bool (flag)~~ |
|
||||
| `--silent`, `-s` <Tag variant="new">2.0.12</Tag> | Don't print anything, just return the values. ~~bool (flag)~~ |
|
||||
| `--exclude`, `-e` | Comma-separated keys to exclude from the print-out. Defaults to `"labels"`. ~~Optional[str]~~ |
|
||||
| `--url`, `-u` <Tag variant="new">3.5.0</Tag> | Print the URL to download the most recent compatible version of the pipeline. Requires a pipeline name. ~~bool (flag)~~ |
|
||||
| `--help`, `-h` | Show help message and available arguments. ~~bool (flag)~~ |
|
||||
| **PRINTS** | Information about your spaCy installation. |
|
||||
|
||||
## validate {#validate new="2" tag="command"}
|
||||
|
||||
|
|
@ -1481,7 +1482,7 @@ You'll also need to add the assets you want to track with
|
|||
</Infobox>
|
||||
|
||||
```cli
|
||||
$ python -m spacy project dvc [project_dir] [workflow] [--force] [--verbose]
|
||||
$ python -m spacy project dvc [project_dir] [workflow] [--force] [--verbose] [--quiet]
|
||||
```
|
||||
|
||||
> #### Example
|
||||
|
|
@ -1498,6 +1499,7 @@ $ python -m spacy project dvc [project_dir] [workflow] [--force] [--verbose]
|
|||
| `workflow` | Name of workflow defined in `project.yml`. Defaults to first workflow if not set. ~~Optional[str] \(option)~~ |
|
||||
| `--force`, `-F` | Force-updating config file. ~~bool (flag)~~ |
|
||||
| `--verbose`, `-V` | Print more output generated by DVC. ~~bool (flag)~~ |
|
||||
| `--quiet`, `-q` | Print no output generated by DVC. ~~bool (flag)~~ |
|
||||
| `--help`, `-h` | Show help message and available arguments. ~~bool (flag)~~ |
|
||||
| **CREATES** | A `dvc.yaml` file in the project directory, based on the steps defined in the given workflow. |
|
||||
|
||||
|
|
|
|||
|
|
@ -395,12 +395,13 @@ file to keep track of your settings and hyperparameters and your own
|
|||
> "pos": List[str],
|
||||
> "morphs": List[str],
|
||||
> "sent_starts": List[Optional[bool]],
|
||||
> "deps": List[string],
|
||||
> "deps": List[str],
|
||||
> "heads": List[int],
|
||||
> "entities": List[str],
|
||||
> "entities": List[(int, int, str)],
|
||||
> "cats": Dict[str, float],
|
||||
> "links": Dict[(int, int), dict],
|
||||
> "spans": Dict[str, List[Tuple]],
|
||||
> }
|
||||
> ```
|
||||
|
||||
|
|
@ -417,9 +418,10 @@ file to keep track of your settings and hyperparameters and your own
|
|||
| `deps` | List of string values indicating the [dependency relation](/usage/linguistic-features#dependency-parse) of a token to its head. ~~List[str]~~ |
|
||||
| `heads` | List of integer values indicating the dependency head of each token, referring to the absolute index of each token in the text. ~~List[int]~~ |
|
||||
| `entities` | **Option 1:** List of [BILUO tags](/usage/linguistic-features#accessing-ner) per token of the format `"{action}-{label}"`, or `None` for unannotated tokens. ~~List[str]~~ |
|
||||
| `entities` | **Option 2:** List of `"(start, end, label)"` tuples defining all entities in the text. ~~List[Tuple[int, int, str]]~~ |
|
||||
| `entities` | **Option 2:** List of `(start_char, end_char, label)` tuples defining all entities in the text. ~~List[Tuple[int, int, str]]~~ |
|
||||
| `cats` | Dictionary of `label`/`value` pairs indicating how relevant a certain [text category](/api/textcategorizer) is for the text. ~~Dict[str, float]~~ |
|
||||
| `links` | Dictionary of `offset`/`dict` pairs defining [named entity links](/usage/linguistic-features#entity-linking). The character offsets are linked to a dictionary of relevant knowledge base IDs. ~~Dict[Tuple[int, int], Dict]~~ |
|
||||
| `spans` | Dictionary of `spans_key`/`List[Tuple]` pairs defining the spans for each spans key as `(start_char, end_char, label, kb_id)` tuples. ~~Dict[str, List[Tuple[int, int, str, str]]~~ |
|
||||
|
||||
<Infobox title="Notes and caveats">
|
||||
|
||||
|
|
|
|||
|
|
@ -62,7 +62,7 @@ of relations, see the usage guide on
|
|||
|
||||
</Infobox>
|
||||
|
||||
### Operators
|
||||
### Operators {#operators}
|
||||
|
||||
The following operators are supported by the `DependencyMatcher`, most of which
|
||||
come directly from
|
||||
|
|
@ -82,6 +82,11 @@ come directly from
|
|||
| `A $- B` | `B` is a left immediate sibling of `A`, i.e. `A` and `B` have the same parent and `A.i == B.i + 1`. |
|
||||
| `A $++ B` | `B` is a right sibling of `A`, i.e. `A` and `B` have the same parent and `A.i < B.i`. |
|
||||
| `A $-- B` | `B` is a left sibling of `A`, i.e. `A` and `B` have the same parent and `A.i > B.i`. |
|
||||
| `A >++ B` | `B` is a right child of `A`, i.e. `A` is a parent of `B` and `A.i < B.i` _(not in Semgrex)_. |
|
||||
| `A >-- B` | `B` is a left child of `A`, i.e. `A` is a parent of `B` and `A.i > B.i` _(not in Semgrex)_. |
|
||||
| `A <++ B` | `B` is a right parent of `A`, i.e. `A` is a child of `B` and `A.i < B.i` _(not in Semgrex)_. |
|
||||
| `A <-- B` | `B` is a left parent of `A`, i.e. `A` is a child of `B` and `A.i > B.i` _(not in Semgrex)_. |
|
||||
|
||||
|
||||
## DependencyMatcher.\_\_init\_\_ {#init tag="method"}
|
||||
|
||||
|
|
|
|||
|
|
@ -31,21 +31,21 @@ Construct a `Doc` object. The most common way to get a `Doc` object is via the
|
|||
> doc = Doc(nlp.vocab, words=words, spaces=spaces)
|
||||
> ```
|
||||
|
||||
| Name | Description |
|
||||
| ---------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| `vocab` | A storage container for lexical types. ~~Vocab~~ |
|
||||
| `words` | A list of strings or integer hash values to add to the document as words. ~~Optional[List[Union[str,int]]]~~ |
|
||||
| `spaces` | A list of boolean values indicating whether each word has a subsequent space. Must have the same length as `words`, if specified. Defaults to a sequence of `True`. ~~Optional[List[bool]]~~ |
|
||||
| _keyword-only_ | |
|
||||
| `user\_data` | Optional extra data to attach to the Doc. ~~Dict~~ |
|
||||
| `tags` <Tag variant="new">3</Tag> | A list of strings, of the same length as `words`, to assign as `token.tag` for each word. Defaults to `None`. ~~Optional[List[str]]~~ |
|
||||
| `pos` <Tag variant="new">3</Tag> | A list of strings, of the same length as `words`, to assign as `token.pos` for each word. Defaults to `None`. ~~Optional[List[str]]~~ |
|
||||
| `morphs` <Tag variant="new">3</Tag> | A list of strings, of the same length as `words`, to assign as `token.morph` for each word. Defaults to `None`. ~~Optional[List[str]]~~ |
|
||||
| `lemmas` <Tag variant="new">3</Tag> | A list of strings, of the same length as `words`, to assign as `token.lemma` for each word. Defaults to `None`. ~~Optional[List[str]]~~ |
|
||||
| `heads` <Tag variant="new">3</Tag> | A list of values, of the same length as `words`, to assign as the head for each word. Head indices are the absolute position of the head in the `Doc`. Defaults to `None`. ~~Optional[List[int]]~~ |
|
||||
| `deps` <Tag variant="new">3</Tag> | A list of strings, of the same length as `words`, to assign as `token.dep` for each word. Defaults to `None`. ~~Optional[List[str]]~~ |
|
||||
| `sent_starts` <Tag variant="new">3</Tag> | A list of values, of the same length as `words`, to assign as `token.is_sent_start`. Will be overridden by heads if `heads` is provided. Defaults to `None`. ~~Optional[List[Optional[bool]]]~~ |
|
||||
| `ents` <Tag variant="new">3</Tag> | A list of strings, of the same length of `words`, to assign the token-based IOB tag. Defaults to `None`. ~~Optional[List[str]]~~ |
|
||||
| Name | Description |
|
||||
| ---------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| `vocab` | A storage container for lexical types. ~~Vocab~~ |
|
||||
| `words` | A list of strings or integer hash values to add to the document as words. ~~Optional[List[Union[str,int]]]~~ |
|
||||
| `spaces` | A list of boolean values indicating whether each word has a subsequent space. Must have the same length as `words`, if specified. Defaults to a sequence of `True`. ~~Optional[List[bool]]~~ |
|
||||
| _keyword-only_ | |
|
||||
| `user\_data` | Optional extra data to attach to the Doc. ~~Dict~~ |
|
||||
| `tags` <Tag variant="new">3</Tag> | A list of strings, of the same length as `words`, to assign as `token.tag` for each word. Defaults to `None`. ~~Optional[List[str]]~~ |
|
||||
| `pos` <Tag variant="new">3</Tag> | A list of strings, of the same length as `words`, to assign as `token.pos` for each word. Defaults to `None`. ~~Optional[List[str]]~~ |
|
||||
| `morphs` <Tag variant="new">3</Tag> | A list of strings, of the same length as `words`, to assign as `token.morph` for each word. Defaults to `None`. ~~Optional[List[str]]~~ |
|
||||
| `lemmas` <Tag variant="new">3</Tag> | A list of strings, of the same length as `words`, to assign as `token.lemma` for each word. Defaults to `None`. ~~Optional[List[str]]~~ |
|
||||
| `heads` <Tag variant="new">3</Tag> | A list of values, of the same length as `words`, to assign as the head for each word. Head indices are the absolute position of the head in the `Doc`. Defaults to `None`. ~~Optional[List[int]]~~ |
|
||||
| `deps` <Tag variant="new">3</Tag> | A list of strings, of the same length as `words`, to assign as `token.dep` for each word. Defaults to `None`. ~~Optional[List[str]]~~ |
|
||||
| `sent_starts` <Tag variant="new">3</Tag> | A list of values, of the same length as `words`, to assign as `token.is_sent_start`. Will be overridden by heads if `heads` is provided. Defaults to `None`. ~~Optional[List[Union[bool, int, None]]]~~ |
|
||||
| `ents` <Tag variant="new">3</Tag> | A list of strings, of the same length of `words`, to assign the token-based IOB tag. Defaults to `None`. ~~Optional[List[str]]~~ |
|
||||
|
||||
## Doc.\_\_getitem\_\_ {#getitem tag="method"}
|
||||
|
||||
|
|
|
|||
|
|
@ -288,10 +288,14 @@ Calculate alignment tables between two tokenizations.
|
|||
|
||||
### Alignment attributes {#alignment-attributes"}
|
||||
|
||||
| Name | Description |
|
||||
| ----- | --------------------------------------------------------------------- |
|
||||
| `x2y` | The `Ragged` object holding the alignment from `x` to `y`. ~~Ragged~~ |
|
||||
| `y2x` | The `Ragged` object holding the alignment from `y` to `x`. ~~Ragged~~ |
|
||||
Alignment attributes are managed using `AlignmentArray`, which is a
|
||||
simplified version of Thinc's [Ragged](https://thinc.ai/docs/api-types#ragged)
|
||||
type that only supports the `data` and `length` attributes.
|
||||
|
||||
| Name | Description |
|
||||
| ----- | ------------------------------------------------------------------------------------- |
|
||||
| `x2y` | The `AlignmentArray` object holding the alignment from `x` to `y`. ~~AlignmentArray~~ |
|
||||
| `y2x` | The `AlignmentArray` object holding the alignment from `y` to `x`. ~~AlignmentArray~~ |
|
||||
|
||||
<Infobox title="Important note" variant="warning">
|
||||
|
||||
|
|
@ -311,10 +315,10 @@ tokenizations add up to the same string. For example, you'll be able to align
|
|||
> spacy_tokens = ["obama", "'s", "podcast"]
|
||||
> alignment = Alignment.from_strings(bert_tokens, spacy_tokens)
|
||||
> a2b = alignment.x2y
|
||||
> assert list(a2b.dataXd) == [0, 1, 1, 2]
|
||||
> assert list(a2b.data) == [0, 1, 1, 2]
|
||||
> ```
|
||||
>
|
||||
> If `a2b.dataXd[1] == a2b.dataXd[2] == 1`, that means that `A[1]` (`"'"`) and
|
||||
> If `a2b.data[1] == a2b.data[2] == 1`, that means that `A[1]` (`"'"`) and
|
||||
> `A[2]` (`"s"`) both align to `B[1]` (`"'s"`).
|
||||
|
||||
### Alignment.from_strings {#classmethod tag="function"}
|
||||
|
|
|
|||
|
|
@ -63,17 +63,18 @@ spaCy loads a model under the hood based on its
|
|||
> nlp = Language.from_config(config)
|
||||
> ```
|
||||
|
||||
| Name | Description |
|
||||
| -------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
|
||||
| `config` | The loaded config. ~~Union[Dict[str, Any], Config]~~ |
|
||||
| _keyword-only_ | |
|
||||
| `vocab` | A `Vocab` object. If `True`, a vocab is created using the default language data settings. ~~Vocab~~ |
|
||||
| `disable` | Names of pipeline components to [disable](/usage/processing-pipelines#disabling). Disabled pipes will be loaded but they won't be run unless you explicitly enable them by calling [`nlp.enable_pipe`](/api/language#enable_pipe). ~~List[str]~~ |
|
||||
| `exclude` | Names of pipeline components to [exclude](/usage/processing-pipelines#disabling). Excluded components won't be loaded. ~~List[str]~~ |
|
||||
| `meta` | [Meta data](/api/data-formats#meta) overrides. ~~Dict[str, Any]~~ |
|
||||
| `auto_fill` | Whether to automatically fill in missing values in the config, based on defaults and function argument annotations. Defaults to `True`. ~~bool~~ |
|
||||
| `validate` | Whether to validate the component config and arguments against the types expected by the factory. Defaults to `True`. ~~bool~~ |
|
||||
| **RETURNS** | The initialized object. ~~Language~~ |
|
||||
| Name | Description |
|
||||
| ------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| `config` | The loaded config. ~~Union[Dict[str, Any], Config]~~ |
|
||||
| _keyword-only_ | |
|
||||
| `vocab` | A `Vocab` object. If `True`, a vocab is created using the default language data settings. ~~Vocab~~ |
|
||||
| `disable` | Name(s) of pipeline component(s) to [disable](/usage/processing-pipelines#disabling). Disabled pipes will be loaded but they won't be run unless you explicitly enable them by calling [`nlp.enable_pipe`](/api/language#enable_pipe). ~~Union[str, Iterable[str]]~~ |
|
||||
| `enable` <Tag variant="new">3.4</Tag> | Name(s) of pipeline component(s) to [enable](/usage/processing-pipelines#disabling). All other pipes will be disabled, but can be enabled again using [`nlp.enable_pipe`](/api/language#enable_pipe). ~~Union[str, Iterable[str]]~~ |
|
||||
| `exclude` | Name(s) of pipeline component(s) to [exclude](/usage/processing-pipelines#disabling). Excluded components won't be loaded. ~~Union[str, Iterable[str]]~~ |
|
||||
| `meta` | [Meta data](/api/data-formats#meta) overrides. ~~Dict[str, Any]~~ |
|
||||
| `auto_fill` | Whether to automatically fill in missing values in the config, based on defaults and function argument annotations. Defaults to `True`. ~~bool~~ |
|
||||
| `validate` | Whether to validate the component config and arguments against the types expected by the factory. Defaults to `True`. ~~bool~~ |
|
||||
| **RETURNS** | The initialized object. ~~Language~~ |
|
||||
|
||||
## Language.component {#component tag="classmethod" new="3"}
|
||||
|
||||
|
|
@ -701,8 +702,8 @@ As of spaCy v3.0, the `disable_pipes` method has been renamed to `select_pipes`:
|
|||
| Name | Description |
|
||||
| -------------- | ------------------------------------------------------------------------------------------------------ |
|
||||
| _keyword-only_ | |
|
||||
| `disable` | Name(s) of pipeline components to disable. ~~Optional[Union[str, Iterable[str]]]~~ |
|
||||
| `enable` | Name(s) of pipeline components that will not be disabled. ~~Optional[Union[str, Iterable[str]]]~~ |
|
||||
| `disable` | Name(s) of pipeline component(s) to disable. ~~Optional[Union[str, Iterable[str]]]~~ |
|
||||
| `enable` | Name(s) of pipeline component(s) that will not be disabled. ~~Optional[Union[str, Iterable[str]]]~~ |
|
||||
| **RETURNS** | The disabled pipes that can be restored by calling the object's `.restore()` method. ~~DisabledPipes~~ |
|
||||
|
||||
## Language.get_factory_meta {#get_factory_meta tag="classmethod" new="3"}
|
||||
|
|
|
|||
|
|
@ -248,6 +248,59 @@ added to an existing vectors table. See more details in
|
|||
|
||||
## Loggers {#loggers}
|
||||
|
||||
These functions are available from `@spacy.registry.loggers`.
|
||||
|
||||
### spacy.ConsoleLogger.v1 {#ConsoleLogger_v1}
|
||||
|
||||
> #### Example config
|
||||
>
|
||||
> ```ini
|
||||
> [training.logger]
|
||||
> @loggers = "spacy.ConsoleLogger.v1"
|
||||
> progress_bar = true
|
||||
> ```
|
||||
|
||||
Writes the results of a training step to the console in a tabular format.
|
||||
|
||||
<Accordion title="Example console output" spaced>
|
||||
|
||||
```cli
|
||||
$ python -m spacy train config.cfg
|
||||
```
|
||||
|
||||
```
|
||||
ℹ Using CPU
|
||||
ℹ Loading config and nlp from: config.cfg
|
||||
ℹ Pipeline: ['tok2vec', 'tagger']
|
||||
ℹ Start training
|
||||
ℹ Training. Initial learn rate: 0.0
|
||||
|
||||
E # LOSS TOK2VEC LOSS TAGGER TAG_ACC SCORE
|
||||
--- ------ ------------ ----------- ------- ------
|
||||
0 0 0.00 86.20 0.22 0.00
|
||||
0 200 3.08 18968.78 34.00 0.34
|
||||
0 400 31.81 22539.06 33.64 0.34
|
||||
0 600 92.13 22794.91 43.80 0.44
|
||||
0 800 183.62 21541.39 56.05 0.56
|
||||
0 1000 352.49 25461.82 65.15 0.65
|
||||
0 1200 422.87 23708.82 71.84 0.72
|
||||
0 1400 601.92 24994.79 76.57 0.77
|
||||
0 1600 662.57 22268.02 80.20 0.80
|
||||
0 1800 1101.50 28413.77 82.56 0.83
|
||||
0 2000 1253.43 28736.36 85.00 0.85
|
||||
0 2200 1411.02 28237.53 87.42 0.87
|
||||
0 2400 1605.35 28439.95 88.70 0.89
|
||||
```
|
||||
|
||||
Note that the cumulative loss keeps increasing within one epoch, but should
|
||||
start decreasing across epochs.
|
||||
|
||||
</Accordion>
|
||||
|
||||
| Name | Description |
|
||||
| -------------- | --------------------------------------------------------- |
|
||||
| `progress_bar` | Whether the logger should print the progress bar ~~bool~~ |
|
||||
|
||||
Logging utilities for spaCy are implemented in the
|
||||
[`spacy-loggers`](https://github.com/explosion/spacy-loggers) repo, and the
|
||||
functions are typically available from `@spacy.registry.loggers`.
|
||||
|
|
|
|||
|
|
@ -45,16 +45,16 @@ specified separately using the new `exclude` keyword argument.
|
|||
> nlp = spacy.load("en_core_web_sm", exclude=["parser", "tagger"])
|
||||
> ```
|
||||
|
||||
| Name | Description |
|
||||
| ------------------------------------ | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| `name` | Pipeline to load, i.e. package name or path. ~~Union[str, Path]~~ |
|
||||
| _keyword-only_ | |
|
||||
| `vocab` | Optional shared vocab to pass in on initialization. If `True` (default), a new `Vocab` object will be created. ~~Union[Vocab, bool]~~ |
|
||||
| `disable` | Names of pipeline components to [disable](/usage/processing-pipelines#disabling). Disabled pipes will be loaded but they won't be run unless you explicitly enable them by calling [nlp.enable_pipe](/api/language#enable_pipe). ~~List[str]~~ |
|
||||
| `enable` | Names of pipeline components to [enable](/usage/processing-pipelines#disabling). All other pipes will be disabled. ~~List[str]~~ |
|
||||
| `exclude` <Tag variant="new">3</Tag> | Names of pipeline components to [exclude](/usage/processing-pipelines#disabling). Excluded components won't be loaded. ~~List[str]~~ |
|
||||
| `config` <Tag variant="new">3</Tag> | Optional config overrides, either as nested dict or dict keyed by section value in dot notation, e.g. `"components.name.value"`. ~~Union[Dict[str, Any], Config]~~ |
|
||||
| **RETURNS** | A `Language` object with the loaded pipeline. ~~Language~~ |
|
||||
| Name | Description |
|
||||
| ------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
|
||||
| `name` | Pipeline to load, i.e. package name or path. ~~Union[str, Path]~~ |
|
||||
| _keyword-only_ | |
|
||||
| `vocab` | Optional shared vocab to pass in on initialization. If `True` (default), a new `Vocab` object will be created. ~~Union[Vocab, bool]~~ |
|
||||
| `disable` | Name(s) of pipeline component(s) to [disable](/usage/processing-pipelines#disabling). Disabled pipes will be loaded but they won't be run unless you explicitly enable them by calling [nlp.enable_pipe](/api/language#enable_pipe). ~~Union[str, Iterable[str]]~~ |
|
||||
| `enable` <Tag variant="new">3.4</Tag> | Name(s) of pipeline component(s) to [enable](/usage/processing-pipelines#disabling). All other pipes will be disabled. ~~Union[str, Iterable[str]]~~ |
|
||||
| `exclude` <Tag variant="new">3</Tag> | Name(s) of pipeline component(s) to [exclude](/usage/processing-pipelines#disabling). Excluded components won't be loaded. ~~Union[str, Iterable[str]]~~ |
|
||||
| `config` <Tag variant="new">3</Tag> | Optional config overrides, either as nested dict or dict keyed by section value in dot notation, e.g. `"components.name.value"`. ~~Union[Dict[str, Any], Config]~~ |
|
||||
| **RETURNS** | A `Language` object with the loaded pipeline. ~~Language~~ |
|
||||
|
||||
Essentially, `spacy.load()` is a convenience wrapper that reads the pipeline's
|
||||
[`config.cfg`](/api/data-formats#config), uses the language and pipeline
|
||||
|
|
@ -275,8 +275,8 @@ Render a dependency parse tree or named entity visualization.
|
|||
|
||||
### displacy.parse_deps {#displacy.parse_deps tag="method" new="2"}
|
||||
|
||||
Generate dependency parse in `{'words': [], 'arcs': []}` format.
|
||||
For use with the `manual=True` argument in `displacy.render`.
|
||||
Generate dependency parse in `{'words': [], 'arcs': []}` format. For use with
|
||||
the `manual=True` argument in `displacy.render`.
|
||||
|
||||
> #### Example
|
||||
>
|
||||
|
|
@ -297,8 +297,8 @@ For use with the `manual=True` argument in `displacy.render`.
|
|||
|
||||
### displacy.parse_ents {#displacy.parse_ents tag="method" new="2"}
|
||||
|
||||
Generate named entities in `[{start: i, end: i, label: 'label'}]` format.
|
||||
For use with the `manual=True` argument in `displacy.render`.
|
||||
Generate named entities in `[{start: i, end: i, label: 'label'}]` format. For
|
||||
use with the `manual=True` argument in `displacy.render`.
|
||||
|
||||
> #### Example
|
||||
>
|
||||
|
|
@ -319,8 +319,8 @@ For use with the `manual=True` argument in `displacy.render`.
|
|||
|
||||
### displacy.parse_spans {#displacy.parse_spans tag="method" new="2"}
|
||||
|
||||
Generate spans in `[{start_token: i, end_token: i, label: 'label'}]` format.
|
||||
For use with the `manual=True` argument in `displacy.render`.
|
||||
Generate spans in `[{start_token: i, end_token: i, label: 'label'}]` format. For
|
||||
use with the `manual=True` argument in `displacy.render`.
|
||||
|
||||
> #### Example
|
||||
>
|
||||
|
|
@ -451,7 +451,7 @@ factories.
|
|||
| Registry name | Description |
|
||||
| ----------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| `architectures` | Registry for functions that create [model architectures](/api/architectures). Can be used to register custom model architectures and reference them in the `config.cfg`. |
|
||||
| `augmenters` | Registry for functions that create [data augmentation](#augmenters) callbacks for corpora and other training data iterators. |
|
||||
| `augmenters` | Registry for functions that create [data augmentation](#augmenters) callbacks for corpora and other training data iterators. |
|
||||
| `batchers` | Registry for training and evaluation [data batchers](#batchers). |
|
||||
| `callbacks` | Registry for custom callbacks to [modify the `nlp` object](/usage/training#custom-code-nlp-callbacks) before training. |
|
||||
| `displacy_colors` | Registry for custom color scheme for the [`displacy` NER visualizer](/usage/visualizers). Automatically reads from [entry points](/usage/saving-loading#entry-points). |
|
||||
|
|
@ -505,7 +505,7 @@ finished. To log each training step, a
|
|||
and the accuracy scores on the development set.
|
||||
|
||||
The built-in, default logger is the ConsoleLogger, which prints results to the
|
||||
console in tabular format. The
|
||||
console in tabular format and saves them to a `jsonl` file. The
|
||||
[spacy-loggers](https://github.com/explosion/spacy-loggers) package, included as
|
||||
a dependency of spaCy, enables other loggers, such as one that sends results to
|
||||
a [Weights & Biases](https://www.wandb.com/) dashboard.
|
||||
|
|
@ -513,16 +513,20 @@ a [Weights & Biases](https://www.wandb.com/) dashboard.
|
|||
Instead of using one of the built-in loggers, you can
|
||||
[implement your own](/usage/training#custom-logging).
|
||||
|
||||
#### spacy.ConsoleLogger.v1 {#ConsoleLogger tag="registered function"}
|
||||
#### spacy.ConsoleLogger.v2 {#ConsoleLogger tag="registered function"}
|
||||
|
||||
> #### Example config
|
||||
>
|
||||
> ```ini
|
||||
> [training.logger]
|
||||
> @loggers = "spacy.ConsoleLogger.v1"
|
||||
> @loggers = "spacy.ConsoleLogger.v2"
|
||||
> progress_bar = true
|
||||
> console_output = true
|
||||
> output_file = "training_log.jsonl"
|
||||
> ```
|
||||
|
||||
Writes the results of a training step to the console in a tabular format.
|
||||
Writes the results of a training step to the console in a tabular format and
|
||||
saves them to a `jsonl` file.
|
||||
|
||||
<Accordion title="Example console output" spaced>
|
||||
|
||||
|
|
@ -536,22 +540,23 @@ $ python -m spacy train config.cfg
|
|||
ℹ Pipeline: ['tok2vec', 'tagger']
|
||||
ℹ Start training
|
||||
ℹ Training. Initial learn rate: 0.0
|
||||
ℹ Saving results to training_log.jsonl
|
||||
|
||||
E # LOSS TOK2VEC LOSS TAGGER TAG_ACC SCORE
|
||||
--- ------ ------------ ----------- ------- ------
|
||||
1 0 0.00 86.20 0.22 0.00
|
||||
1 200 3.08 18968.78 34.00 0.34
|
||||
1 400 31.81 22539.06 33.64 0.34
|
||||
1 600 92.13 22794.91 43.80 0.44
|
||||
1 800 183.62 21541.39 56.05 0.56
|
||||
1 1000 352.49 25461.82 65.15 0.65
|
||||
1 1200 422.87 23708.82 71.84 0.72
|
||||
1 1400 601.92 24994.79 76.57 0.77
|
||||
1 1600 662.57 22268.02 80.20 0.80
|
||||
1 1800 1101.50 28413.77 82.56 0.83
|
||||
1 2000 1253.43 28736.36 85.00 0.85
|
||||
1 2200 1411.02 28237.53 87.42 0.87
|
||||
1 2400 1605.35 28439.95 88.70 0.89
|
||||
0 0 0.00 86.20 0.22 0.00
|
||||
0 200 3.08 18968.78 34.00 0.34
|
||||
0 400 31.81 22539.06 33.64 0.34
|
||||
0 600 92.13 22794.91 43.80 0.44
|
||||
0 800 183.62 21541.39 56.05 0.56
|
||||
0 1000 352.49 25461.82 65.15 0.65
|
||||
0 1200 422.87 23708.82 71.84 0.72
|
||||
0 1400 601.92 24994.79 76.57 0.77
|
||||
0 1600 662.57 22268.02 80.20 0.80
|
||||
0 1800 1101.50 28413.77 82.56 0.83
|
||||
0 2000 1253.43 28736.36 85.00 0.85
|
||||
0 2200 1411.02 28237.53 87.42 0.87
|
||||
0 2400 1605.35 28439.95 88.70 0.89
|
||||
```
|
||||
|
||||
Note that the cumulative loss keeps increasing within one epoch, but should
|
||||
|
|
@ -559,6 +564,12 @@ start decreasing across epochs.
|
|||
|
||||
</Accordion>
|
||||
|
||||
| Name | Description |
|
||||
| ---------------- | --------------------------------------------------------------------- |
|
||||
| `progress_bar` | Whether the logger should print the progress bar ~~bool~~ |
|
||||
| `console_output` | Whether the logger should print the logs on the console. ~~bool~~ |
|
||||
| `output_file` | The file to save the training logs to. ~~Optional[Union[str, Path]]~~ |
|
||||
|
||||
## Readers {#readers}
|
||||
|
||||
### File readers {#file-readers source="github.com/explosion/srsly" new="3"}
|
||||
|
|
@ -1059,15 +1070,16 @@ and create a `Language` object. The model data will then be loaded in via
|
|||
> nlp = util.load_model("/path/to/data")
|
||||
> ```
|
||||
|
||||
| Name | Description |
|
||||
| ------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
|
||||
| `name` | Package name or path. ~~str~~ |
|
||||
| _keyword-only_ | |
|
||||
| `vocab` | Optional shared vocab to pass in on initialization. If `True` (default), a new `Vocab` object will be created. ~~Union[Vocab, bool]~~ |
|
||||
| `disable` | Names of pipeline components to [disable](/usage/processing-pipelines#disabling). Disabled pipes will be loaded but they won't be run unless you explicitly enable them by calling [`nlp.enable_pipe`](/api/language#enable_pipe). ~~List[str]~~ |
|
||||
| `exclude` <Tag variant="new">3</Tag> | Names of pipeline components to [exclude](/usage/processing-pipelines#disabling). Excluded components won't be loaded. ~~List[str]~~ |
|
||||
| `config` <Tag variant="new">3</Tag> | Config overrides as nested dict or flat dict keyed by section values in dot notation, e.g. `"nlp.pipeline"`. ~~Union[Dict[str, Any], Config]~~ |
|
||||
| **RETURNS** | `Language` class with the loaded pipeline. ~~Language~~ |
|
||||
| Name | Description |
|
||||
| ------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| `name` | Package name or path. ~~str~~ |
|
||||
| _keyword-only_ | |
|
||||
| `vocab` | Optional shared vocab to pass in on initialization. If `True` (default), a new `Vocab` object will be created. ~~Union[Vocab, bool]~~ |
|
||||
| `disable` | Name(s) of pipeline component(s) to [disable](/usage/processing-pipelines#disabling). Disabled pipes will be loaded but they won't be run unless you explicitly enable them by calling [`nlp.enable_pipe`](/api/language#enable_pipe). ~~Union[str, Iterable[str]]~~ |
|
||||
| `enable` <Tag variant="new">3.4</Tag> | Name(s) of pipeline component(s) to [enable](/usage/processing-pipelines#disabling). All other pipes will be disabled, but can be enabled again using [`nlp.enable_pipe`](/api/language#enable_pipe). ~~Union[str, Iterable[str]]~~ |
|
||||
| `exclude` | Name(s) of pipeline component(s) to [exclude](/usage/processing-pipelines#disabling). Excluded components won't be loaded. ~~Union[str, Iterable[str]]~~ |
|
||||
| `config` <Tag variant="new">3</Tag> | Config overrides as nested dict or flat dict keyed by section values in dot notation, e.g. `"nlp.pipeline"`. ~~Union[Dict[str, Any], Config]~~ |
|
||||
| **RETURNS** | `Language` class with the loaded pipeline. ~~Language~~ |
|
||||
|
||||
### util.load_model_from_init_py {#util.load_model_from_init_py tag="function" new="2"}
|
||||
|
||||
|
|
@ -1083,15 +1095,16 @@ A helper function to use in the `load()` method of a pipeline package's
|
|||
> return load_model_from_init_py(__file__, **overrides)
|
||||
> ```
|
||||
|
||||
| Name | Description |
|
||||
| ------------------------------------ | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| `init_file` | Path to package's `__init__.py`, i.e. `__file__`. ~~Union[str, Path]~~ |
|
||||
| _keyword-only_ | |
|
||||
| `vocab` <Tag variant="new">3</Tag> | Optional shared vocab to pass in on initialization. If `True` (default), a new `Vocab` object will be created. ~~Union[Vocab, bool]~~ |
|
||||
| `disable` | Names of pipeline components to [disable](/usage/processing-pipelines#disabling). Disabled pipes will be loaded but they won't be run unless you explicitly enable them by calling [nlp.enable_pipe](/api/language#enable_pipe). ~~List[str]~~ |
|
||||
| `exclude` <Tag variant="new">3</Tag> | Names of pipeline components to [exclude](/usage/processing-pipelines#disabling). Excluded components won't be loaded. ~~List[str]~~ |
|
||||
| `config` <Tag variant="new">3</Tag> | Config overrides as nested dict or flat dict keyed by section values in dot notation, e.g. `"nlp.pipeline"`. ~~Union[Dict[str, Any], Config]~~ |
|
||||
| **RETURNS** | `Language` class with the loaded pipeline. ~~Language~~ |
|
||||
| Name | Description |
|
||||
| ------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| `init_file` | Path to package's `__init__.py`, i.e. `__file__`. ~~Union[str, Path]~~ |
|
||||
| _keyword-only_ | |
|
||||
| `vocab` <Tag variant="new">3</Tag> | Optional shared vocab to pass in on initialization. If `True` (default), a new `Vocab` object will be created. ~~Union[Vocab, bool]~~ |
|
||||
| `disable` | Name(s) of pipeline component(s) to [disable](/usage/processing-pipelines#disabling). Disabled pipes will be loaded but they won't be run unless you explicitly enable them by calling [`nlp.enable_pipe`](/api/language#enable_pipe). ~~Union[str, Iterable[str]]~~ |
|
||||
| `enable` <Tag variant="new">3.4</Tag> | Name(s) of pipeline component(s) to [enable](/usage/processing-pipelines#disabling). All other pipes will be disabled, but can be enabled again using [`nlp.enable_pipe`](/api/language#enable_pipe). ~~Union[str, Iterable[str]]~~ |
|
||||
| `exclude` <Tag variant="new">3</Tag> | Name(s) of pipeline component(s) to [exclude](/usage/processing-pipelines#disabling). Excluded components won't be loaded. ~~Union[str, Iterable[str]]~~ |
|
||||
| `config` <Tag variant="new">3</Tag> | Config overrides as nested dict or flat dict keyed by section values in dot notation, e.g. `"nlp.pipeline"`. ~~Union[Dict[str, Any], Config]~~ |
|
||||
| **RETURNS** | `Language` class with the loaded pipeline. ~~Language~~ |
|
||||
|
||||
### util.load_config {#util.load_config tag="function" new="3"}
|
||||
|
||||
|
|
|
|||
|
|
@ -1422,9 +1422,9 @@ other_tokens = ["i", "listened", "to", "obama", "'", "s", "podcasts", "."]
|
|||
spacy_tokens = ["i", "listened", "to", "obama", "'s", "podcasts", "."]
|
||||
align = Alignment.from_strings(other_tokens, spacy_tokens)
|
||||
print(f"a -> b, lengths: {align.x2y.lengths}") # array([1, 1, 1, 1, 1, 1, 1, 1])
|
||||
print(f"a -> b, mapping: {align.x2y.dataXd}") # array([0, 1, 2, 3, 4, 4, 5, 6]) : two tokens both refer to "'s"
|
||||
print(f"a -> b, mapping: {align.x2y.data}") # array([0, 1, 2, 3, 4, 4, 5, 6]) : two tokens both refer to "'s"
|
||||
print(f"b -> a, lengths: {align.y2x.lengths}") # array([1, 1, 1, 1, 2, 1, 1]) : the token "'s" refers to two tokens
|
||||
print(f"b -> a, mappings: {align.y2x.dataXd}") # array([0, 1, 2, 3, 4, 5, 6, 7])
|
||||
print(f"b -> a, mappings: {align.y2x.data}") # array([0, 1, 2, 3, 4, 5, 6, 7])
|
||||
```
|
||||
|
||||
Here are some insights from the alignment information generated in the example
|
||||
|
|
@ -1433,10 +1433,10 @@ above:
|
|||
- The one-to-one mappings for the first four tokens are identical, which means
|
||||
they map to each other. This makes sense because they're also identical in the
|
||||
input: `"i"`, `"listened"`, `"to"` and `"obama"`.
|
||||
- The value of `x2y.dataXd[6]` is `5`, which means that `other_tokens[6]`
|
||||
- The value of `x2y.data[6]` is `5`, which means that `other_tokens[6]`
|
||||
(`"podcasts"`) aligns to `spacy_tokens[5]` (also `"podcasts"`).
|
||||
- `x2y.dataXd[4]` and `x2y.dataXd[5]` are both `4`, which means that both tokens
|
||||
4 and 5 of `other_tokens` (`"'"` and `"s"`) align to token 4 of `spacy_tokens`
|
||||
- `x2y.data[4]` and `x2y.data[5]` are both `4`, which means that both tokens 4
|
||||
and 5 of `other_tokens` (`"'"` and `"s"`) align to token 4 of `spacy_tokens`
|
||||
(`"'s"`).
|
||||
|
||||
<Infobox title="Important note" variant="warning">
|
||||
|
|
|
|||
|
|
@ -365,15 +365,32 @@ pipeline package can be found.
|
|||
To download a trained pipeline directly using
|
||||
[pip](https://pypi.python.org/pypi/pip), point `pip install` to the URL or local
|
||||
path of the wheel file or archive. Installing the wheel is usually more
|
||||
efficient. To find the direct link to a package, head over to the
|
||||
[releases](https://github.com/explosion/spacy-models/releases), right click on
|
||||
the archive link and copy it to your clipboard.
|
||||
efficient.
|
||||
|
||||
> #### Pipeline Package URLs {#pipeline-urls}
|
||||
>
|
||||
> Pretrained pipeline distributions are hosted on
|
||||
> [Github Releases](https://github.com/explosion/spacy-models/releases), and you
|
||||
> can find download links there, as well as on the model page. You can also get
|
||||
> URLs directly from the command line by using `spacy info` with the `--url`
|
||||
> flag, which may be useful for automation.
|
||||
>
|
||||
> ```bash
|
||||
> spacy info en_core_web_sm --url
|
||||
> ```
|
||||
>
|
||||
> This command will print the URL for the latest version of a pipeline
|
||||
> compatible with the version of spaCy you're using. Note that in order to look
|
||||
> up the compatibility information an internet connection is required.
|
||||
|
||||
```bash
|
||||
# With external URL
|
||||
$ pip install https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.0.0/en_core_web_sm-3.0.0-py3-none-any.whl
|
||||
$ pip install https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.0.0/en_core_web_sm-3.0.0.tar.gz
|
||||
|
||||
# Using spacy info to get the external URL
|
||||
$ pip install $(spacy info en_core_web_sm --url)
|
||||
|
||||
# With local file
|
||||
$ pip install /Users/you/en_core_web_sm-3.0.0-py3-none-any.whl
|
||||
$ pip install /Users/you/en_core_web_sm-3.0.0.tar.gz
|
||||
|
|
@ -514,21 +531,16 @@ should be specifying them directly.
|
|||
Because pipeline packages are valid Python packages, you can add them to your
|
||||
application's `requirements.txt`. If you're running your own internal PyPi
|
||||
installation, you can upload the pipeline packages there. pip's
|
||||
[requirements file format](https://pip.pypa.io/en/latest/reference/pip_install/#requirements-file-format)
|
||||
supports both package names to download via a PyPi server, as well as direct
|
||||
URLs.
|
||||
[requirements file format](https://pip.pypa.io/en/latest/reference/requirements-file-format/)
|
||||
supports both package names to download via a PyPi server, as well as
|
||||
[direct URLs](#pipeline-urls).
|
||||
|
||||
```text
|
||||
### requirements.txt
|
||||
spacy>=3.0.0,<4.0.0
|
||||
https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.0.0/en_core_web_sm-3.0.0.tar.gz#egg=en_core_web_sm
|
||||
en_core_web_sm @ https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.4.0/en_core_web_sm-3.4.0-py3-none-any.whl
|
||||
```
|
||||
|
||||
Specifying `#egg=` with the package name tells pip which package to expect from
|
||||
the download URL. This way, the package won't be re-downloaded and overwritten
|
||||
if it's already installed - just like when you're downloading a package from
|
||||
PyPi.
|
||||
|
||||
All pipeline packages are versioned and specify their spaCy dependency. This
|
||||
ensures cross-compatibility and lets you specify exact version requirements for
|
||||
each pipeline. If you've [trained](/usage/training) your own pipeline, you can
|
||||
|
|
|
|||
|
|
@ -148,6 +148,13 @@ skipped. You can also set `--force` to force re-running a command, or `--dry` to
|
|||
perform a "dry run" and see what would happen (without actually running the
|
||||
script).
|
||||
|
||||
Since spaCy v3.4.2, `spacy projects run` checks your installed dependencies to
|
||||
verify that your environment is properly set up and aligns with the project's
|
||||
`requirements.txt`, if there is one. If missing or conflicting dependencies are
|
||||
detected, a corresponding warning is displayed. If you'd like to disable the
|
||||
dependency check, set `check_requirements: false` in your project's
|
||||
`project.yml`.
|
||||
|
||||
### 4. Run a workflow {#run-workfow}
|
||||
|
||||
> #### project.yml
|
||||
|
|
@ -226,26 +233,28 @@ pipelines.
|
|||
```yaml
|
||||
%%GITHUB_PROJECTS/pipelines/tagger_parser_ud/project.yml
|
||||
```
|
||||
|
||||
> #### Tip: Overriding variables on the CLI
|
||||
>
|
||||
> If you want to override one or more variables on the CLI and are not already specifying a
|
||||
> project directory, you need to add `.` as a placeholder:
|
||||
> If you want to override one or more variables on the CLI and are not already
|
||||
> specifying a project directory, you need to add `.` as a placeholder:
|
||||
>
|
||||
> ```
|
||||
> python -m spacy project run test . --vars.foo bar
|
||||
> ```
|
||||
|
||||
| Section | Description |
|
||||
| --------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
|
||||
| `title` | An optional project title used in `--help` message and [auto-generated docs](#custom-docs). |
|
||||
| `description` | An optional project description used in [auto-generated docs](#custom-docs). |
|
||||
| `vars` | A dictionary of variables that can be referenced in paths, URLs and scripts and overriden on the CLI, just like [`config.cfg` variables](/usage/training#config-interpolation). For example, `${vars.name}` will use the value of the variable `name`. Variables need to be defined in the section `vars`, but can be a nested dict, so you're able to reference `${vars.model.name}`. |
|
||||
| `env` | A dictionary of variables, mapped to the names of environment variables that will be read in when running the project. For example, `${env.name}` will use the value of the environment variable defined as `name`. |
|
||||
| `directories` | An optional list of [directories](#project-files) that should be created in the project for assets, training outputs, metrics etc. spaCy will make sure that these directories always exist. |
|
||||
| `assets` | A list of assets that can be fetched with the [`project assets`](/api/cli#project-assets) command. `url` defines a URL or local path, `dest` is the destination file relative to the project directory, and an optional `checksum` ensures that an error is raised if the file's checksum doesn't match. Instead of `url`, you can also provide a `git` block with the keys `repo`, `branch` and `path`, to download from a Git repo. |
|
||||
| `workflows` | A dictionary of workflow names, mapped to a list of command names, to execute in order. Workflows can be run with the [`project run`](/api/cli#project-run) command. |
|
||||
| `commands` | A list of named commands. A command can define an optional help message (shown in the CLI when the user adds `--help`) and the `script`, a list of commands to run. The `deps` and `outputs` let you define the created file the command depends on and produces, respectively. This lets spaCy determine whether a command needs to be re-run because its dependencies or outputs changed. Commands can be run as part of a workflow, or separately with the [`project run`](/api/cli#project-run) command. |
|
||||
| `spacy_version` | Optional spaCy version range like `>=3.0.0,<3.1.0` that the project is compatible with. If it's loaded with an incompatible version, an error is raised when the project is loaded. |
|
||||
| Section | Description |
|
||||
| --------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
|
||||
| `title` | An optional project title used in `--help` message and [auto-generated docs](#custom-docs). |
|
||||
| `description` | An optional project description used in [auto-generated docs](#custom-docs). |
|
||||
| `vars` | A dictionary of variables that can be referenced in paths, URLs and scripts and overriden on the CLI, just like [`config.cfg` variables](/usage/training#config-interpolation). For example, `${vars.name}` will use the value of the variable `name`. Variables need to be defined in the section `vars`, but can be a nested dict, so you're able to reference `${vars.model.name}`. |
|
||||
| `env` | A dictionary of variables, mapped to the names of environment variables that will be read in when running the project. For example, `${env.name}` will use the value of the environment variable defined as `name`. |
|
||||
| `directories` | An optional list of [directories](#project-files) that should be created in the project for assets, training outputs, metrics etc. spaCy will make sure that these directories always exist. |
|
||||
| `assets` | A list of assets that can be fetched with the [`project assets`](/api/cli#project-assets) command. `url` defines a URL or local path, `dest` is the destination file relative to the project directory, and an optional `checksum` ensures that an error is raised if the file's checksum doesn't match. Instead of `url`, you can also provide a `git` block with the keys `repo`, `branch` and `path`, to download from a Git repo. |
|
||||
| `workflows` | A dictionary of workflow names, mapped to a list of command names, to execute in order. Workflows can be run with the [`project run`](/api/cli#project-run) command. |
|
||||
| `commands` | A list of named commands. A command can define an optional help message (shown in the CLI when the user adds `--help`) and the `script`, a list of commands to run. The `deps` and `outputs` let you define the created file the command depends on and produces, respectively. This lets spaCy determine whether a command needs to be re-run because its dependencies or outputs changed. Commands can be run as part of a workflow, or separately with the [`project run`](/api/cli#project-run) command. |
|
||||
| `spacy_version` | Optional spaCy version range like `>=3.0.0,<3.1.0` that the project is compatible with. If it's loaded with an incompatible version, an error is raised when the project is loaded. |
|
||||
| `check_requirements` <Tag variant="new">3.4.2</Tag> | A flag determining whether to verify that the installed dependencies align with the project's `requirements.txt`. Defaults to `true`. |
|
||||
|
||||
### Data assets {#data-assets}
|
||||
|
||||
|
|
|
|||
|
|
@ -265,6 +265,11 @@
|
|||
"name": "Luxembourgish",
|
||||
"has_examples": true
|
||||
},
|
||||
{
|
||||
"code": "lg",
|
||||
"name": "Luganda",
|
||||
"has_examples": true
|
||||
},
|
||||
{
|
||||
"code": "lij",
|
||||
"name": "Ligurian",
|
||||
|
|
|
|||
|
|
@ -76,6 +76,7 @@ const MODEL_META = {
|
|||
benchmark_ner: 'NER accuracy',
|
||||
benchmark_speed: 'Speed',
|
||||
compat: 'Latest compatible package version for your spaCy installation',
|
||||
download_link: 'Download link for the pipeline',
|
||||
}
|
||||
|
||||
const LABEL_SCHEME_META = {
|
||||
|
|
@ -114,7 +115,11 @@ function formatVectors(data) {
|
|||
if (!data) return 'n/a'
|
||||
if (Object.values(data).every(n => n === 0)) return 'context vectors only'
|
||||
const { keys, vectors, width } = data
|
||||
return `${abbrNum(keys)} keys, ${abbrNum(vectors)} unique vectors (${width} dimensions)`
|
||||
if (keys >= 0) {
|
||||
return `${abbrNum(keys)} keys, ${abbrNum(vectors)} unique vectors (${width} dimensions)`
|
||||
} else {
|
||||
return `${abbrNum(vectors)} floret vectors (${width} dimensions)`
|
||||
}
|
||||
}
|
||||
|
||||
function formatAccuracy(data, lang) {
|
||||
|
|
@ -134,6 +139,13 @@ function formatAccuracy(data, lang) {
|
|||
.filter(item => item)
|
||||
}
|
||||
|
||||
function formatDownloadLink(lang, name, version) {
|
||||
const fullName = `${lang}_${name}-${version}`
|
||||
const filename = `${fullName}-py3-none-any.whl`
|
||||
const url = `https://github.com/explosion/spacy-models/releases/download/${fullName}/${filename}`
|
||||
return <Link to={url} hideIcon>{filename}</Link>
|
||||
}
|
||||
|
||||
function formatModelMeta(data) {
|
||||
return {
|
||||
fullName: `${data.lang}_${data.name}-${data.version}`,
|
||||
|
|
@ -150,6 +162,7 @@ function formatModelMeta(data) {
|
|||
labels: isEmptyObj(data.labels) ? null : data.labels,
|
||||
vectors: formatVectors(data.vectors),
|
||||
accuracy: formatAccuracy(data.performance, data.lang),
|
||||
download_link: formatDownloadLink(data.lang, data.name, data.version),
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -240,6 +253,7 @@ const Model = ({
|
|||
{ label: 'Components', content: components, help: MODEL_META.components },
|
||||
{ label: 'Pipeline', content: pipeline, help: MODEL_META.pipeline },
|
||||
{ label: 'Vectors', content: meta.vectors, help: MODEL_META.vecs },
|
||||
{ label: 'Download Link', content: meta.download_link, help: MODEL_META.download_link },
|
||||
{ label: 'Sources', content: sources, help: MODEL_META.sources },
|
||||
{ label: 'Author', content: author },
|
||||
{ label: 'License', content: license },
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ const DEFAULT_PLATFORM = 'x86'
|
|||
const DEFAULT_MODELS = ['en']
|
||||
const DEFAULT_OPT = 'efficiency'
|
||||
const DEFAULT_HARDWARE = 'cpu'
|
||||
const DEFAULT_CUDA = 'cuda113'
|
||||
const DEFAULT_CUDA = 'cuda-autodetect'
|
||||
const CUDA = {
|
||||
'8.0': 'cuda80',
|
||||
'9.0': 'cuda90',
|
||||
|
|
@ -17,15 +17,7 @@ const CUDA = {
|
|||
'9.2': 'cuda92',
|
||||
'10.0': 'cuda100',
|
||||
'10.1': 'cuda101',
|
||||
'10.2': 'cuda102',
|
||||
'11.0': 'cuda110',
|
||||
'11.1': 'cuda111',
|
||||
'11.2': 'cuda112',
|
||||
'11.3': 'cuda113',
|
||||
'11.4': 'cuda114',
|
||||
'11.5': 'cuda115',
|
||||
'11.6': 'cuda116',
|
||||
'11.7': 'cuda117',
|
||||
'10.2, 11.0+': 'cuda-autodetect',
|
||||
}
|
||||
const LANG_EXTRAS = ['ja'] // only for languages with models
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user