2020-07-22 14:42:59 +03:00
|
|
|
|
from typing import Optional, Any, Dict, Callable, Iterable, Union, List, Pattern
|
|
|
|
|
from typing import Tuple, Iterator
|
|
|
|
|
from dataclasses import dataclass
|
2017-05-25 04:10:54 +03:00
|
|
|
|
import random
|
2017-07-25 19:57:59 +03:00
|
|
|
|
import itertools
|
2017-10-16 20:22:40 +03:00
|
|
|
|
import weakref
|
2017-10-17 19:18:10 +03:00
|
|
|
|
import functools
|
2020-07-22 14:42:59 +03:00
|
|
|
|
from collections import Iterable as IterableInstance
|
2017-10-27 22:07:59 +03:00
|
|
|
|
from contextlib import contextmanager
|
💫 Better support for semi-supervised learning (#3035)
The new spacy pretrain command implemented BERT/ULMFit/etc-like transfer learning, using our Language Modelling with Approximate Outputs version of BERT's cloze task. Pretraining is convenient, but in some ways it's a bit of a strange solution. All we're doing is initialising the weights. At the same time, we're putting a lot of work into our optimisation so that it's less sensitive to initial conditions, and more likely to find good optima. I discuss this a bit in the pseudo-rehearsal blog post: https://explosion.ai/blog/pseudo-rehearsal-catastrophic-forgetting
Support semi-supervised learning in spacy train
One obvious way to improve these pretraining methods is to do multi-task learning, instead of just transfer learning. This has been shown to work very well: https://arxiv.org/pdf/1809.08370.pdf . This patch makes it easy to do this sort of thing.
Add a new argument to spacy train, --raw-text. This takes a jsonl file with unlabelled data that can be used in arbitrary ways to do semi-supervised learning.
Add a new method to the Language class and to pipeline components, .rehearse(). This is like .update(), but doesn't expect GoldParse objects. It takes a batch of Doc objects, and performs an update on some semi-supervised objective.
Move the BERT-LMAO objective out from spacy/cli/pretrain.py into spacy/_ml.py, so we can create a new pipeline component, ClozeMultitask. This can be specified as a parser or NER multitask in the spacy train command. Example usage:
python -m spacy train en ./tmp ~/data/en-core-web/train/nw.json ~/data/en-core-web/dev/nw.json --pipeline parser --raw-textt ~/data/unlabelled/reddit-100k.jsonl --vectors en_vectors_web_lg --parser-multitasks cloze
Implement rehearsal methods for pipeline components
The new --raw-text argument and nlp.rehearse() method also gives us a good place to implement the the idea in the pseudo-rehearsal blog post in the parser. This works as follows:
Add a new nlp.resume_training() method. This allocates copies of pre-trained models in the pipeline, setting things up for the rehearsal updates. It also returns an optimizer object. This also greatly reduces confusion around the nlp.begin_training() method, which randomises the weights, making it not suitable for adding new labels or otherwise fine-tuning a pre-trained model.
Implement rehearsal updates on the Parser class, making it available for the dependency parser and NER. During rehearsal, the initial model is used to supervise the model being trained. The current model is asked to match the predictions of the initial model on some data. This minimises catastrophic forgetting, by keeping the model's predictions close to the original. See the blog post for details.
Implement rehearsal updates for tagger
Implement rehearsal updates for text categoriz
2018-12-10 18:25:33 +03:00
|
|
|
|
from copy import copy, deepcopy
|
2020-02-27 20:42:27 +03:00
|
|
|
|
from pathlib import Path
|
2020-02-28 14:20:23 +03:00
|
|
|
|
import warnings
|
2020-07-22 14:42:59 +03:00
|
|
|
|
from thinc.api import get_current_ops, Config, require_gpu, Optimizer
|
💫 Replace ujson, msgpack and dill/pickle/cloudpickle with srsly (#3003)
Remove hacks and wrappers, keep code in sync across our libraries and move spaCy a few steps closer to only depending on packages with binary wheels 🎉
See here: https://github.com/explosion/srsly
Serialization is hard, especially across Python versions and multiple platforms. After dealing with many subtle bugs over the years (encodings, locales, large files) our libraries like spaCy and Prodigy have steadily grown a number of utility functions to wrap the multiple serialization formats we need to support (especially json, msgpack and pickle). These wrapping functions ended up duplicated across our codebases, so we wanted to put them in one place.
At the same time, we noticed that having a lot of small dependencies was making maintainence harder, and making installation slower. To solve this, we've made srsly standalone, by including the component packages directly within it. This way we can provide all the serialization utilities we need in a single binary wheel.
srsly currently includes forks of the following packages:
ujson
msgpack
msgpack-numpy
cloudpickle
* WIP: replace json/ujson with srsly
* Replace ujson in examples
Use regular json instead of srsly to make code easier to read and follow
* Update requirements
* Fix imports
* Fix typos
* Replace msgpack with srsly
* Fix warning
2018-12-03 03:28:22 +03:00
|
|
|
|
import srsly
|
2019-10-08 13:20:55 +03:00
|
|
|
|
import multiprocessing as mp
|
|
|
|
|
from itertools import chain, cycle
|
2017-05-18 12:25:19 +03:00
|
|
|
|
|
2020-02-12 13:50:42 +03:00
|
|
|
|
from .tokens.underscore import Underscore
|
2020-07-24 15:50:26 +03:00
|
|
|
|
from .vocab import Vocab, create_vocab
|
2020-05-22 18:42:06 +03:00
|
|
|
|
from .pipe_analysis import analyze_pipes, analyze_all_pipes, validate_attrs
|
2019-12-22 03:53:56 +03:00
|
|
|
|
from .gold import Example
|
2017-10-07 01:26:05 +03:00
|
|
|
|
from .scorer import Scorer
|
2020-02-27 20:42:27 +03:00
|
|
|
|
from .util import link_vectors_to_models, create_default_optimizer, registry
|
2020-07-26 14:18:43 +03:00
|
|
|
|
from .util import SimpleFrozenDict, combine_score_weights
|
2020-07-24 15:50:26 +03:00
|
|
|
|
from .lang.tokenizer_exceptions import URL_MATCH, BASE_EXCEPTIONS
|
2017-10-27 15:40:14 +03:00
|
|
|
|
from .lang.punctuation import TOKENIZER_PREFIXES, TOKENIZER_SUFFIXES
|
|
|
|
|
from .lang.punctuation import TOKENIZER_INFIXES
|
2020-07-22 23:18:46 +03:00
|
|
|
|
from .tokens import Doc
|
2020-07-24 15:50:26 +03:00
|
|
|
|
from .lookups import load_lookups
|
|
|
|
|
from .tokenizer import Tokenizer
|
|
|
|
|
from .lemmatizer import Lemmatizer
|
2020-04-28 14:37:37 +03:00
|
|
|
|
from .errors import Errors, Warnings
|
2020-07-22 14:42:59 +03:00
|
|
|
|
from .schemas import ConfigSchema
|
2020-07-02 18:10:27 +03:00
|
|
|
|
from .git_info import GIT_VERSION
|
2017-04-15 13:05:47 +03:00
|
|
|
|
from . import util
|
2017-10-07 01:26:05 +03:00
|
|
|
|
from . import about
|
2016-10-09 13:24:24 +03:00
|
|
|
|
|
2015-08-27 10:16:11 +03:00
|
|
|
|
|
2020-07-29 00:12:42 +03:00
|
|
|
|
# TODO: integrate pipeline analyis
|
2019-10-27 15:35:49 +03:00
|
|
|
|
ENABLE_PIPELINE_ANALYSIS = False
|
2020-07-22 14:42:59 +03:00
|
|
|
|
# This is the base config will all settings (training etc.)
|
|
|
|
|
DEFAULT_CONFIG_PATH = Path(__file__).parent / "default_config.cfg"
|
|
|
|
|
DEFAULT_CONFIG = Config().from_disk(DEFAULT_CONFIG_PATH)
|
2019-10-27 15:35:49 +03:00
|
|
|
|
|
|
|
|
|
|
2020-07-12 15:03:23 +03:00
|
|
|
|
class BaseDefaults:
|
2020-07-29 00:12:42 +03:00
|
|
|
|
"""Language data defaults, available via Language.Defaults. Can be
|
|
|
|
|
overwritten by language subclasses by defining their own subclasses of
|
|
|
|
|
Language.Defaults.
|
|
|
|
|
"""
|
2020-07-24 15:50:26 +03:00
|
|
|
|
config: Config = Config()
|
|
|
|
|
tokenizer_exceptions: Dict[str, List[dict]] = BASE_EXCEPTIONS
|
|
|
|
|
prefixes: Optional[List[Union[str, Pattern]]] = TOKENIZER_PREFIXES
|
|
|
|
|
suffixes: Optional[List[Union[str, Pattern]]] = TOKENIZER_SUFFIXES
|
|
|
|
|
infixes: Optional[List[Union[str, Pattern]]] = TOKENIZER_INFIXES
|
|
|
|
|
token_match: Optional[Pattern] = None
|
|
|
|
|
url_match: Optional[Pattern] = URL_MATCH
|
|
|
|
|
syntax_iterators: Dict[str, Callable] = {}
|
|
|
|
|
lex_attr_getters: Dict[int, Callable[[str], Any]] = {}
|
|
|
|
|
stop_words = set()
|
|
|
|
|
writing_system = {"direction": "ltr", "has_case": True, "has_letters": True}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@registry.tokenizers("spacy.Tokenizer.v1")
|
|
|
|
|
def create_tokenizer() -> Callable[["Language"], Tokenizer]:
|
2020-07-29 00:12:42 +03:00
|
|
|
|
"""Registered function to create a tokenizer. Returns a factory that takes
|
|
|
|
|
the nlp object and returns a Tokenizer instance using the language detaults.
|
|
|
|
|
"""
|
2020-07-24 15:50:26 +03:00
|
|
|
|
def tokenizer_factory(nlp: "Language") -> Tokenizer:
|
|
|
|
|
prefixes = nlp.Defaults.prefixes
|
|
|
|
|
suffixes = nlp.Defaults.suffixes
|
|
|
|
|
infixes = nlp.Defaults.infixes
|
|
|
|
|
prefix_search = util.compile_prefix_regex(prefixes).search if prefixes else None
|
|
|
|
|
suffix_search = util.compile_suffix_regex(suffixes).search if suffixes else None
|
|
|
|
|
infix_finditer = util.compile_infix_regex(infixes).finditer if infixes else None
|
|
|
|
|
return Tokenizer(
|
|
|
|
|
nlp.vocab,
|
|
|
|
|
rules=nlp.Defaults.tokenizer_exceptions,
|
|
|
|
|
prefix_search=prefix_search,
|
|
|
|
|
suffix_search=suffix_search,
|
|
|
|
|
infix_finditer=infix_finditer,
|
|
|
|
|
token_match=nlp.Defaults.token_match,
|
|
|
|
|
url_match=nlp.Defaults.url_match,
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
return tokenizer_factory
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@registry.lemmatizers("spacy.Lemmatizer.v1")
|
|
|
|
|
def create_lemmatizer() -> Callable[["Language"], "Lemmatizer"]:
|
2020-07-29 00:12:42 +03:00
|
|
|
|
"""Registered function to create a lemmatizer. Returns a factory that takes
|
|
|
|
|
the nlp object and returns a Lemmatizer instance with data loaded in from
|
|
|
|
|
spacy-lookups-data, if the package is installed.
|
|
|
|
|
"""
|
|
|
|
|
# TODO: Will be replaced when the lemmatizer becomes a pipeline component
|
2020-07-24 15:50:26 +03:00
|
|
|
|
tables = ["lemma_lookup", "lemma_rules", "lemma_exc", "lemma_index"]
|
|
|
|
|
|
|
|
|
|
def lemmatizer_factory(nlp: "Language") -> "Lemmatizer":
|
|
|
|
|
lookups = load_lookups(lang=nlp.lang, tables=tables, strict=False)
|
|
|
|
|
return Lemmatizer(lookups=lookups)
|
|
|
|
|
|
|
|
|
|
return lemmatizer_factory
|
2015-09-14 10:48:51 +03:00
|
|
|
|
|
2015-08-26 20:16:09 +03:00
|
|
|
|
|
2020-07-12 15:03:23 +03:00
|
|
|
|
class Language:
|
2017-05-19 00:57:38 +03:00
|
|
|
|
"""A text-processing pipeline. Usually you'll load this once per process,
|
|
|
|
|
and pass the instance around your application.
|
2017-05-19 19:47:24 +03:00
|
|
|
|
|
|
|
|
|
Defaults (class): Settings, data and factory methods for creating the `nlp`
|
|
|
|
|
object and processing pipeline.
|
2020-05-24 18:20:58 +03:00
|
|
|
|
lang (str): Two-letter language ID, i.e. ISO code.
|
💫 Tidy up and auto-format .py files (#2983)
<!--- Provide a general summary of your changes in the title. -->
## Description
- [x] Use [`black`](https://github.com/ambv/black) to auto-format all `.py` files.
- [x] Update flake8 config to exclude very large files (lemmatization tables etc.)
- [x] Update code to be compatible with flake8 rules
- [x] Fix various small bugs, inconsistencies and messy stuff in the language data
- [x] Update docs to explain new code style (`black`, `flake8`, when to use `# fmt: off` and `# fmt: on` and what `# noqa` means)
Once #2932 is merged, which auto-formats and tidies up the CLI, we'll be able to run `flake8 spacy` actually get meaningful results.
At the moment, the code style and linting isn't applied automatically, but I'm hoping that the new [GitHub Actions](https://github.com/features/actions) will let us auto-format pull requests and post comments with relevant linting information.
### Types of change
enhancement, code style
## Checklist
<!--- Before you submit the PR, go over this checklist and make sure you can
tick off all the boxes. [] -> [x] -->
- [x] I have submitted the spaCy Contributor Agreement.
- [x] I ran the tests, and all new and existing tests passed.
- [x] My changes don't require a change to the documentation, or if they do, I've added all required information.
2018-11-30 19:03:03 +03:00
|
|
|
|
|
2019-03-08 13:42:26 +03:00
|
|
|
|
DOCS: https://spacy.io/api/language
|
|
|
|
|
"""
|
2019-03-11 01:36:47 +03:00
|
|
|
|
|
2016-09-24 21:26:17 +03:00
|
|
|
|
Defaults = BaseDefaults
|
2020-07-22 14:42:59 +03:00
|
|
|
|
lang: str = None
|
|
|
|
|
default_config = DEFAULT_CONFIG
|
2015-08-25 16:37:17 +03:00
|
|
|
|
|
2020-07-24 15:50:26 +03:00
|
|
|
|
factories = SimpleFrozenDict(error=Errors.E957)
|
2020-07-22 14:42:59 +03:00
|
|
|
|
_factory_meta: Dict[str, "FactoryMeta"] = {} # meta by factory
|
2017-10-07 01:25:54 +03:00
|
|
|
|
|
💫 Tidy up and auto-format .py files (#2983)
<!--- Provide a general summary of your changes in the title. -->
## Description
- [x] Use [`black`](https://github.com/ambv/black) to auto-format all `.py` files.
- [x] Update flake8 config to exclude very large files (lemmatization tables etc.)
- [x] Update code to be compatible with flake8 rules
- [x] Fix various small bugs, inconsistencies and messy stuff in the language data
- [x] Update docs to explain new code style (`black`, `flake8`, when to use `# fmt: off` and `# fmt: on` and what `# noqa` means)
Once #2932 is merged, which auto-formats and tidies up the CLI, we'll be able to run `flake8 spacy` actually get meaningful results.
At the moment, the code style and linting isn't applied automatically, but I'm hoping that the new [GitHub Actions](https://github.com/features/actions) will let us auto-format pull requests and post comments with relevant linting information.
### Types of change
enhancement, code style
## Checklist
<!--- Before you submit the PR, go over this checklist and make sure you can
tick off all the boxes. [] -> [x] -->
- [x] I have submitted the spaCy Contributor Agreement.
- [x] I ran the tests, and all new and existing tests passed.
- [x] My changes don't require a change to the documentation, or if they do, I've added all required information.
2018-11-30 19:03:03 +03:00
|
|
|
|
def __init__(
|
2020-02-28 13:57:41 +03:00
|
|
|
|
self,
|
2020-07-22 14:42:59 +03:00
|
|
|
|
vocab: Union[Vocab, bool] = True,
|
2020-07-27 01:27:53 +03:00
|
|
|
|
*,
|
2020-07-22 14:42:59 +03:00
|
|
|
|
max_length: int = 10 ** 6,
|
|
|
|
|
meta: Dict[str, Any] = {},
|
|
|
|
|
create_tokenizer: Optional[Callable[["Language"], Callable[[str], Doc]]] = None,
|
2020-07-24 15:50:26 +03:00
|
|
|
|
create_lemmatizer: Optional[Callable[["Language"], Callable]] = None,
|
2020-02-28 13:57:41 +03:00
|
|
|
|
**kwargs,
|
2020-07-29 00:12:42 +03:00
|
|
|
|
) -> None:
|
2017-05-19 00:57:38 +03:00
|
|
|
|
"""Initialise a Language object.
|
|
|
|
|
|
2020-07-22 14:42:59 +03:00
|
|
|
|
vocab (Vocab): A `Vocab` object. If `True`, a vocab is created.
|
2017-05-19 00:57:38 +03:00
|
|
|
|
meta (dict): Custom meta data for the Language class. Is written to by
|
|
|
|
|
models to add model meta data.
|
2020-07-25 13:14:28 +03:00
|
|
|
|
max_length (int): Maximum number of characters in a single text. The
|
|
|
|
|
current models may run out memory on extremely long texts, due to
|
|
|
|
|
large internal allocations. You should segment these texts into
|
|
|
|
|
meaningful units, e.g. paragraphs, subsections etc, before passing
|
|
|
|
|
them to spaCy. Default maximum length is 1,000,000 charas (1mb). As
|
|
|
|
|
a rule of thumb, if all pipeline components are enabled, spaCy's
|
|
|
|
|
default models currently requires roughly 1GB of temporary memory per
|
2018-03-29 22:45:26 +03:00
|
|
|
|
100,000 characters in one text.
|
2020-07-25 13:14:28 +03:00
|
|
|
|
create_tokenizer (Callable): Function that takes the nlp object and
|
|
|
|
|
returns a tokenizer.
|
|
|
|
|
create_lemmatizer (Callable): Function that takes the nlp object and
|
|
|
|
|
returns a lemmatizer.
|
2020-07-29 00:12:42 +03:00
|
|
|
|
|
|
|
|
|
DOCS: https://spacy.io/api/language#init
|
2017-05-19 00:57:38 +03:00
|
|
|
|
"""
|
2020-07-22 14:42:59 +03:00
|
|
|
|
# We're only calling this to import all factories provided via entry
|
|
|
|
|
# points. The factory decorator applied to these functions takes care
|
|
|
|
|
# of the rest.
|
|
|
|
|
util.registry._entry_point_factories.get_all()
|
|
|
|
|
|
|
|
|
|
self._config = util.deep_merge_configs(self.default_config, DEFAULT_CONFIG)
|
2017-07-23 01:50:18 +03:00
|
|
|
|
self._meta = dict(meta)
|
2017-10-25 12:57:43 +03:00
|
|
|
|
self._path = None
|
2020-07-22 14:42:59 +03:00
|
|
|
|
self._optimizer = None
|
|
|
|
|
# Component meta and configs are only needed on the instance
|
|
|
|
|
self._pipe_meta: Dict[str, "FactoryMeta"] = {} # meta by component
|
|
|
|
|
self._pipe_configs: Dict[str, Config] = {} # config by component
|
|
|
|
|
|
2017-05-16 12:21:59 +03:00
|
|
|
|
if vocab is True:
|
2020-07-22 14:42:59 +03:00
|
|
|
|
vectors_name = meta.get("vectors", {}).get("name")
|
2020-07-24 15:50:26 +03:00
|
|
|
|
if not create_lemmatizer:
|
|
|
|
|
lemma_cfg = {"lemmatizer": self._config["nlp"]["lemmatizer"]}
|
|
|
|
|
create_lemmatizer = registry.make_from_config(lemma_cfg)["lemmatizer"]
|
|
|
|
|
vocab = create_vocab(
|
|
|
|
|
self.lang,
|
|
|
|
|
self.Defaults,
|
|
|
|
|
lemmatizer=create_lemmatizer(self),
|
2020-07-22 14:42:59 +03:00
|
|
|
|
vectors_name=vectors_name,
|
2020-07-25 13:14:28 +03:00
|
|
|
|
load_data=self._config["nlp"]["load_vocab_data"],
|
2020-07-22 14:42:59 +03:00
|
|
|
|
)
|
2019-08-01 18:13:01 +03:00
|
|
|
|
else:
|
|
|
|
|
if (self.lang and vocab.lang) and (self.lang != vocab.lang):
|
|
|
|
|
raise ValueError(Errors.E150.format(nlp=self.lang, vocab=vocab.lang))
|
2020-07-25 16:01:15 +03:00
|
|
|
|
self.vocab: Vocab = vocab
|
2020-07-22 14:42:59 +03:00
|
|
|
|
if self.lang is None:
|
|
|
|
|
self.lang = self.vocab.lang
|
2017-10-07 01:25:54 +03:00
|
|
|
|
self.pipeline = []
|
2018-03-29 22:45:26 +03:00
|
|
|
|
self.max_length = max_length
|
2020-07-22 14:42:59 +03:00
|
|
|
|
self.resolved = {}
|
|
|
|
|
# Create the default tokenizer from the default config
|
|
|
|
|
if not create_tokenizer:
|
|
|
|
|
tokenizer_cfg = {"tokenizer": self._config["nlp"]["tokenizer"]}
|
|
|
|
|
create_tokenizer = registry.make_from_config(tokenizer_cfg)["tokenizer"]
|
|
|
|
|
self.tokenizer = create_tokenizer(self)
|
|
|
|
|
|
|
|
|
|
def __init_subclass__(cls, **kwargs):
|
|
|
|
|
super().__init_subclass__(**kwargs)
|
2020-07-24 15:50:26 +03:00
|
|
|
|
cls.default_config = util.deep_merge_configs(
|
|
|
|
|
cls.Defaults.config, DEFAULT_CONFIG
|
|
|
|
|
)
|
|
|
|
|
cls.default_config["nlp"]["lang"] = cls.lang
|
2015-10-12 11:33:11 +03:00
|
|
|
|
|
2017-10-25 12:57:43 +03:00
|
|
|
|
@property
|
|
|
|
|
def path(self):
|
|
|
|
|
return self._path
|
|
|
|
|
|
2017-07-23 01:50:18 +03:00
|
|
|
|
@property
|
2020-07-22 14:42:59 +03:00
|
|
|
|
def meta(self) -> Dict[str, Any]:
|
2020-07-29 00:12:42 +03:00
|
|
|
|
"""Custom meta data of the language class. If a model is loaded, this
|
|
|
|
|
includes details from the model's meta.json.
|
|
|
|
|
|
|
|
|
|
RETURNS (Dict[str, Any]): The meta.
|
|
|
|
|
|
|
|
|
|
DOCS: https://spacy.io/api/language#meta
|
|
|
|
|
"""
|
2020-05-30 16:01:58 +03:00
|
|
|
|
spacy_version = util.get_model_version_range(about.__version__)
|
2019-08-01 18:13:01 +03:00
|
|
|
|
if self.vocab.lang:
|
|
|
|
|
self._meta.setdefault("lang", self.vocab.lang)
|
|
|
|
|
else:
|
|
|
|
|
self._meta.setdefault("lang", self.lang)
|
💫 Tidy up and auto-format .py files (#2983)
<!--- Provide a general summary of your changes in the title. -->
## Description
- [x] Use [`black`](https://github.com/ambv/black) to auto-format all `.py` files.
- [x] Update flake8 config to exclude very large files (lemmatization tables etc.)
- [x] Update code to be compatible with flake8 rules
- [x] Fix various small bugs, inconsistencies and messy stuff in the language data
- [x] Update docs to explain new code style (`black`, `flake8`, when to use `# fmt: off` and `# fmt: on` and what `# noqa` means)
Once #2932 is merged, which auto-formats and tidies up the CLI, we'll be able to run `flake8 spacy` actually get meaningful results.
At the moment, the code style and linting isn't applied automatically, but I'm hoping that the new [GitHub Actions](https://github.com/features/actions) will let us auto-format pull requests and post comments with relevant linting information.
### Types of change
enhancement, code style
## Checklist
<!--- Before you submit the PR, go over this checklist and make sure you can
tick off all the boxes. [] -> [x] -->
- [x] I have submitted the spaCy Contributor Agreement.
- [x] I ran the tests, and all new and existing tests passed.
- [x] My changes don't require a change to the documentation, or if they do, I've added all required information.
2018-11-30 19:03:03 +03:00
|
|
|
|
self._meta.setdefault("name", "model")
|
|
|
|
|
self._meta.setdefault("version", "0.0.0")
|
2020-05-30 16:01:58 +03:00
|
|
|
|
self._meta.setdefault("spacy_version", spacy_version)
|
💫 Tidy up and auto-format .py files (#2983)
<!--- Provide a general summary of your changes in the title. -->
## Description
- [x] Use [`black`](https://github.com/ambv/black) to auto-format all `.py` files.
- [x] Update flake8 config to exclude very large files (lemmatization tables etc.)
- [x] Update code to be compatible with flake8 rules
- [x] Fix various small bugs, inconsistencies and messy stuff in the language data
- [x] Update docs to explain new code style (`black`, `flake8`, when to use `# fmt: off` and `# fmt: on` and what `# noqa` means)
Once #2932 is merged, which auto-formats and tidies up the CLI, we'll be able to run `flake8 spacy` actually get meaningful results.
At the moment, the code style and linting isn't applied automatically, but I'm hoping that the new [GitHub Actions](https://github.com/features/actions) will let us auto-format pull requests and post comments with relevant linting information.
### Types of change
enhancement, code style
## Checklist
<!--- Before you submit the PR, go over this checklist and make sure you can
tick off all the boxes. [] -> [x] -->
- [x] I have submitted the spaCy Contributor Agreement.
- [x] I ran the tests, and all new and existing tests passed.
- [x] My changes don't require a change to the documentation, or if they do, I've added all required information.
2018-11-30 19:03:03 +03:00
|
|
|
|
self._meta.setdefault("description", "")
|
|
|
|
|
self._meta.setdefault("author", "")
|
|
|
|
|
self._meta.setdefault("email", "")
|
|
|
|
|
self._meta.setdefault("url", "")
|
|
|
|
|
self._meta.setdefault("license", "")
|
2020-07-02 18:10:27 +03:00
|
|
|
|
self._meta.setdefault("spacy_git_version", GIT_VERSION)
|
💫 Tidy up and auto-format .py files (#2983)
<!--- Provide a general summary of your changes in the title. -->
## Description
- [x] Use [`black`](https://github.com/ambv/black) to auto-format all `.py` files.
- [x] Update flake8 config to exclude very large files (lemmatization tables etc.)
- [x] Update code to be compatible with flake8 rules
- [x] Fix various small bugs, inconsistencies and messy stuff in the language data
- [x] Update docs to explain new code style (`black`, `flake8`, when to use `# fmt: off` and `# fmt: on` and what `# noqa` means)
Once #2932 is merged, which auto-formats and tidies up the CLI, we'll be able to run `flake8 spacy` actually get meaningful results.
At the moment, the code style and linting isn't applied automatically, but I'm hoping that the new [GitHub Actions](https://github.com/features/actions) will let us auto-format pull requests and post comments with relevant linting information.
### Types of change
enhancement, code style
## Checklist
<!--- Before you submit the PR, go over this checklist and make sure you can
tick off all the boxes. [] -> [x] -->
- [x] I have submitted the spaCy Contributor Agreement.
- [x] I ran the tests, and all new and existing tests passed.
- [x] My changes don't require a change to the documentation, or if they do, I've added all required information.
2018-11-30 19:03:03 +03:00
|
|
|
|
self._meta["vectors"] = {
|
|
|
|
|
"width": self.vocab.vectors_length,
|
|
|
|
|
"vectors": len(self.vocab.vectors),
|
|
|
|
|
"keys": self.vocab.vectors.n_keys,
|
|
|
|
|
"name": self.vocab.vectors.name,
|
|
|
|
|
}
|
2019-09-12 12:34:25 +03:00
|
|
|
|
self._meta["labels"] = self.pipe_labels
|
2020-07-28 17:14:23 +03:00
|
|
|
|
# TODO: Adding this back to prevent breaking people's code etc., but
|
|
|
|
|
# we should consider removing it
|
|
|
|
|
self._meta["pipeline"] = self.pipe_names
|
2017-07-23 01:50:18 +03:00
|
|
|
|
return self._meta
|
|
|
|
|
|
|
|
|
|
@meta.setter
|
2020-07-22 14:42:59 +03:00
|
|
|
|
def meta(self, value: Dict[str, Any]) -> None:
|
2017-07-23 01:50:18 +03:00
|
|
|
|
self._meta = value
|
|
|
|
|
|
2017-06-04 23:52:09 +03:00
|
|
|
|
@property
|
2020-07-22 14:42:59 +03:00
|
|
|
|
def config(self) -> Config:
|
2020-07-29 00:12:42 +03:00
|
|
|
|
"""Trainable config for the current language instance. Includes the
|
|
|
|
|
current pipeline components, as well as default training config.
|
|
|
|
|
|
|
|
|
|
RETURNS (thinc.api.Config): The config.
|
|
|
|
|
|
|
|
|
|
DOCS: https://spacy.io/api/language#config
|
|
|
|
|
"""
|
2020-07-22 14:42:59 +03:00
|
|
|
|
self._config.setdefault("nlp", {})
|
2020-07-26 14:18:43 +03:00
|
|
|
|
self._config.setdefault("training", {})
|
2020-07-22 14:42:59 +03:00
|
|
|
|
self._config["nlp"]["lang"] = self.lang
|
|
|
|
|
# We're storing the filled config for each pipeline component and so
|
|
|
|
|
# we can populate the config again later
|
|
|
|
|
pipeline = {}
|
2020-07-26 14:18:43 +03:00
|
|
|
|
score_weights = []
|
2020-07-22 14:42:59 +03:00
|
|
|
|
for pipe_name in self.pipe_names:
|
|
|
|
|
pipe_meta = self.get_pipe_meta(pipe_name)
|
|
|
|
|
pipe_config = self.get_pipe_config(pipe_name)
|
2020-07-22 18:29:31 +03:00
|
|
|
|
pipeline[pipe_name] = {"factory": pipe_meta.factory, **pipe_config}
|
2020-07-27 13:27:40 +03:00
|
|
|
|
if pipe_meta.default_score_weights:
|
|
|
|
|
score_weights.append(pipe_meta.default_score_weights)
|
2020-07-22 14:42:59 +03:00
|
|
|
|
self._config["nlp"]["pipeline"] = self.pipe_names
|
|
|
|
|
self._config["components"] = pipeline
|
2020-07-26 14:18:43 +03:00
|
|
|
|
self._config["training"]["score_weights"] = combine_score_weights(score_weights)
|
2020-07-22 14:42:59 +03:00
|
|
|
|
if not srsly.is_json_serializable(self._config):
|
|
|
|
|
raise ValueError(Errors.E961.format(config=self._config))
|
2020-02-27 20:42:27 +03:00
|
|
|
|
return self._config
|
2017-10-07 01:25:54 +03:00
|
|
|
|
|
2020-07-22 14:42:59 +03:00
|
|
|
|
@config.setter
|
|
|
|
|
def config(self, value: Config) -> None:
|
|
|
|
|
self._config = value
|
|
|
|
|
|
2017-10-07 01:25:54 +03:00
|
|
|
|
@property
|
2020-07-22 14:42:59 +03:00
|
|
|
|
def factory_names(self) -> List[str]:
|
|
|
|
|
"""Get names of all available factories.
|
|
|
|
|
|
|
|
|
|
RETURNS (List[str]): The factory names.
|
|
|
|
|
"""
|
|
|
|
|
return list(self.factories.keys())
|
|
|
|
|
|
|
|
|
|
@property
|
|
|
|
|
def pipe_names(self) -> List[str]:
|
2017-10-07 01:25:54 +03:00
|
|
|
|
"""Get names of available pipeline components.
|
|
|
|
|
|
2020-07-22 14:42:59 +03:00
|
|
|
|
RETURNS (List[str]): List of component name strings, in order.
|
2017-10-07 01:25:54 +03:00
|
|
|
|
"""
|
|
|
|
|
return [pipe_name for pipe_name, _ in self.pipeline]
|
|
|
|
|
|
2019-10-27 15:35:49 +03:00
|
|
|
|
@property
|
2020-07-22 14:42:59 +03:00
|
|
|
|
def pipe_factories(self) -> Dict[str, str]:
|
2019-10-27 15:35:49 +03:00
|
|
|
|
"""Get the component factories for the available pipeline components.
|
|
|
|
|
|
2020-07-22 14:42:59 +03:00
|
|
|
|
RETURNS (Dict[str, str]): Factory names, keyed by component names.
|
2019-10-27 15:35:49 +03:00
|
|
|
|
"""
|
|
|
|
|
factories = {}
|
|
|
|
|
for pipe_name, pipe in self.pipeline:
|
2020-07-22 14:42:59 +03:00
|
|
|
|
factories[pipe_name] = self.get_pipe_meta(pipe_name).factory
|
2019-10-27 15:35:49 +03:00
|
|
|
|
return factories
|
|
|
|
|
|
2019-09-12 11:56:28 +03:00
|
|
|
|
@property
|
2020-07-22 14:42:59 +03:00
|
|
|
|
def pipe_labels(self) -> Dict[str, List[str]]:
|
2019-09-12 14:03:38 +03:00
|
|
|
|
"""Get the labels set by the pipeline components, if available (if
|
|
|
|
|
the component exposes a labels property).
|
2019-09-12 11:56:28 +03:00
|
|
|
|
|
2020-07-22 14:42:59 +03:00
|
|
|
|
RETURNS (Dict[str, List[str]]): Labels keyed by component name.
|
2019-09-12 11:56:28 +03:00
|
|
|
|
"""
|
2019-12-22 03:53:56 +03:00
|
|
|
|
labels = {}
|
2019-09-12 11:56:28 +03:00
|
|
|
|
for name, pipe in self.pipeline:
|
|
|
|
|
if hasattr(pipe, "labels"):
|
|
|
|
|
labels[name] = list(pipe.labels)
|
|
|
|
|
return labels
|
|
|
|
|
|
2020-07-22 14:42:59 +03:00
|
|
|
|
@classmethod
|
|
|
|
|
def has_factory(cls, name: str) -> bool:
|
|
|
|
|
"""RETURNS (bool): Whether a factory of that name is registered."""
|
|
|
|
|
internal_name = cls.get_factory_name(name)
|
|
|
|
|
return name in registry.factories or internal_name in registry.factories
|
|
|
|
|
|
|
|
|
|
@classmethod
|
|
|
|
|
def get_factory_name(cls, name: str) -> str:
|
|
|
|
|
"""Get the internal factory name based on the language subclass.
|
|
|
|
|
|
|
|
|
|
name (str): The factory name.
|
|
|
|
|
RETURNS (str): The internal factory name.
|
|
|
|
|
"""
|
|
|
|
|
if cls.lang is None:
|
|
|
|
|
return name
|
|
|
|
|
return f"{cls.lang}.{name}"
|
|
|
|
|
|
|
|
|
|
@classmethod
|
|
|
|
|
def get_factory_meta(cls, name: str) -> "FactoryMeta":
|
|
|
|
|
"""Get the meta information for a given factory name.
|
|
|
|
|
|
|
|
|
|
name (str): The component factory name.
|
|
|
|
|
RETURNS (FactoryMeta): The meta for the given factory name.
|
|
|
|
|
"""
|
|
|
|
|
internal_name = cls.get_factory_name(name)
|
|
|
|
|
if internal_name in cls._factory_meta:
|
|
|
|
|
return cls._factory_meta[internal_name]
|
|
|
|
|
if name in cls._factory_meta:
|
|
|
|
|
return cls._factory_meta[name]
|
|
|
|
|
raise ValueError(Errors.E967.format(meta="factory", name=name))
|
|
|
|
|
|
|
|
|
|
@classmethod
|
|
|
|
|
def set_factory_meta(cls, name: str, value: "FactoryMeta") -> None:
|
|
|
|
|
"""Set the meta information for a given factory name.
|
|
|
|
|
|
|
|
|
|
name (str): The component factory name.
|
|
|
|
|
value (FactoryMeta): The meta to set.
|
|
|
|
|
"""
|
|
|
|
|
cls._factory_meta[cls.get_factory_name(name)] = value
|
|
|
|
|
|
|
|
|
|
def get_pipe_meta(self, name: str) -> "FactoryMeta":
|
|
|
|
|
"""Get the meta information for a given component name.
|
|
|
|
|
|
|
|
|
|
name (str): The component name.
|
|
|
|
|
RETURNS (FactoryMeta): The meta for the given component name.
|
|
|
|
|
"""
|
|
|
|
|
if name not in self._pipe_meta:
|
|
|
|
|
raise ValueError(Errors.E967.format(meta="component", name=name))
|
|
|
|
|
return self._pipe_meta[name]
|
|
|
|
|
|
|
|
|
|
def get_pipe_config(self, name: str) -> Config:
|
|
|
|
|
"""Get the config used to create a pipeline component.
|
|
|
|
|
|
|
|
|
|
name (str): The component name.
|
|
|
|
|
RETURNS (Config): The config used to create the pipeline component.
|
|
|
|
|
"""
|
|
|
|
|
if name not in self._pipe_configs:
|
|
|
|
|
raise ValueError(Errors.E960.format(name=name))
|
|
|
|
|
pipe_config = self._pipe_configs[name]
|
|
|
|
|
pipe_config.pop("nlp", None)
|
|
|
|
|
pipe_config.pop("name", None)
|
|
|
|
|
return pipe_config
|
|
|
|
|
|
|
|
|
|
@classmethod
|
|
|
|
|
def factory(
|
|
|
|
|
cls,
|
|
|
|
|
name: str,
|
|
|
|
|
*,
|
|
|
|
|
default_config: Dict[str, Any] = SimpleFrozenDict(),
|
|
|
|
|
assigns: Iterable[str] = tuple(),
|
|
|
|
|
requires: Iterable[str] = tuple(),
|
|
|
|
|
retokenizes: bool = False,
|
2020-07-26 14:18:43 +03:00
|
|
|
|
scores: Iterable[str] = tuple(),
|
2020-07-27 13:27:40 +03:00
|
|
|
|
default_score_weights: Dict[str, float] = SimpleFrozenDict(),
|
2020-07-22 14:42:59 +03:00
|
|
|
|
func: Optional[Callable] = None,
|
|
|
|
|
) -> Callable:
|
|
|
|
|
"""Register a new pipeline component factory. Can be used as a decorator
|
|
|
|
|
on a function or classmethod, or called as a function with the factory
|
|
|
|
|
provided as the func keyword argument. To create a component and add
|
|
|
|
|
it to the pipeline, you can use nlp.add_pipe(name).
|
|
|
|
|
|
|
|
|
|
name (str): The name of the component factory.
|
|
|
|
|
default_config (Dict[str, Any]): Default configuration, describing the
|
|
|
|
|
default values of the factory arguments.
|
|
|
|
|
assigns (Iterable[str]): Doc/Token attributes assigned by this component,
|
|
|
|
|
e.g. "token.ent_id". Used for pipeline analyis.
|
|
|
|
|
requires (Iterable[str]): Doc/Token attributes required by this component,
|
|
|
|
|
e.g. "token.ent_id". Used for pipeline analyis.
|
|
|
|
|
retokenizes (bool): Whether the component changes the tokenization.
|
|
|
|
|
Used for pipeline analysis.
|
2020-07-28 12:22:24 +03:00
|
|
|
|
scores (Iterable[str]): All scores set by the component if it's trainable,
|
|
|
|
|
e.g. ["ents_f", "ents_r", "ents_p"].
|
|
|
|
|
default_score_weights (Dict[str, float]): The scores to report during
|
|
|
|
|
training, and their default weight towards the final score used to
|
|
|
|
|
select the best model. Weights should sum to 1.0 per component and
|
|
|
|
|
will be combined and normalized for the whole pipeline.
|
2020-07-22 14:42:59 +03:00
|
|
|
|
func (Optional[Callable]): Factory function if not used as a decorator.
|
2020-07-29 00:12:42 +03:00
|
|
|
|
|
|
|
|
|
DOCS: https://spacy.io/api/language#factory
|
2020-07-22 14:42:59 +03:00
|
|
|
|
"""
|
|
|
|
|
if not isinstance(name, str):
|
|
|
|
|
raise ValueError(Errors.E963.format(decorator="factory"))
|
|
|
|
|
if not isinstance(default_config, dict):
|
|
|
|
|
err = Errors.E962.format(
|
|
|
|
|
style="default config", name=name, cfg_type=type(default_config)
|
|
|
|
|
)
|
|
|
|
|
raise ValueError(err)
|
|
|
|
|
internal_name = cls.get_factory_name(name)
|
|
|
|
|
if internal_name in registry.factories:
|
|
|
|
|
# We only check for the internal name here – it's okay if it's a
|
|
|
|
|
# subclass and the base class has a factory of the same name
|
|
|
|
|
raise ValueError(Errors.E004.format(name=name))
|
|
|
|
|
|
|
|
|
|
def add_factory(factory_func: Callable) -> Callable:
|
|
|
|
|
arg_names = util.get_arg_names(factory_func)
|
|
|
|
|
if "nlp" not in arg_names or "name" not in arg_names:
|
|
|
|
|
raise ValueError(Errors.E964.format(name=name))
|
|
|
|
|
# Officially register the factory so we can later call
|
|
|
|
|
# registry.make_from_config and refer to it in the config as
|
|
|
|
|
# @factories = "spacy.Language.xyz". We use the class name here so
|
|
|
|
|
# different classes can have different factories.
|
|
|
|
|
registry.factories.register(internal_name, func=factory_func)
|
|
|
|
|
factory_meta = FactoryMeta(
|
|
|
|
|
factory=name,
|
|
|
|
|
default_config=default_config,
|
|
|
|
|
assigns=validate_attrs(assigns),
|
|
|
|
|
requires=validate_attrs(requires),
|
2020-07-26 14:18:43 +03:00
|
|
|
|
scores=scores,
|
2020-07-27 13:27:40 +03:00
|
|
|
|
default_score_weights=default_score_weights,
|
2020-07-22 14:42:59 +03:00
|
|
|
|
retokenizes=retokenizes,
|
|
|
|
|
)
|
|
|
|
|
cls.set_factory_meta(name, factory_meta)
|
|
|
|
|
# We're overwriting the class attr with a frozen dict to handle
|
|
|
|
|
# backwards-compat (writing to Language.factories directly). This
|
|
|
|
|
# wouldn't work with an instance property and just produce a
|
|
|
|
|
# confusing error – here we can show a custom error
|
|
|
|
|
cls.factories = SimpleFrozenDict(
|
|
|
|
|
registry.factories.get_all(), error=Errors.E957
|
|
|
|
|
)
|
|
|
|
|
return factory_func
|
|
|
|
|
|
|
|
|
|
if func is not None: # Support non-decorator use cases
|
|
|
|
|
return add_factory(func)
|
|
|
|
|
return add_factory
|
|
|
|
|
|
|
|
|
|
@classmethod
|
|
|
|
|
def component(
|
|
|
|
|
cls,
|
|
|
|
|
name: Optional[str] = None,
|
|
|
|
|
*,
|
|
|
|
|
assigns: Iterable[str] = tuple(),
|
|
|
|
|
requires: Iterable[str] = tuple(),
|
|
|
|
|
retokenizes: bool = False,
|
2020-07-26 14:18:43 +03:00
|
|
|
|
scores: Iterable[str] = tuple(),
|
2020-07-27 13:27:40 +03:00
|
|
|
|
default_score_weights: Dict[str, float] = SimpleFrozenDict(),
|
2020-07-22 14:42:59 +03:00
|
|
|
|
func: Optional[Callable[[Doc], Doc]] = None,
|
|
|
|
|
) -> Callable:
|
|
|
|
|
"""Register a new pipeline component. Can be used for stateless function
|
|
|
|
|
components that don't require a separate factory. Can be used as a
|
|
|
|
|
decorator on a function or classmethod, or called as a function with the
|
|
|
|
|
factory provided as the func keyword argument. To create a component and
|
|
|
|
|
add it to the pipeline, you can use nlp.add_pipe(name).
|
|
|
|
|
|
|
|
|
|
name (str): The name of the component factory.
|
|
|
|
|
assigns (Iterable[str]): Doc/Token attributes assigned by this component,
|
|
|
|
|
e.g. "token.ent_id". Used for pipeline analyis.
|
|
|
|
|
requires (Iterable[str]): Doc/Token attributes required by this component,
|
|
|
|
|
e.g. "token.ent_id". Used for pipeline analyis.
|
|
|
|
|
retokenizes (bool): Whether the component changes the tokenization.
|
|
|
|
|
Used for pipeline analysis.
|
2020-07-28 12:22:24 +03:00
|
|
|
|
scores (Iterable[str]): All scores set by the component if it's trainable,
|
|
|
|
|
e.g. ["ents_f", "ents_r", "ents_p"].
|
|
|
|
|
default_score_weights (Dict[str, float]): The scores to report during
|
|
|
|
|
training, and their default weight towards the final score used to
|
|
|
|
|
select the best model. Weights should sum to 1.0 per component and
|
|
|
|
|
will be combined and normalized for the whole pipeline.
|
2020-07-22 14:42:59 +03:00
|
|
|
|
func (Optional[Callable]): Factory function if not used as a decorator.
|
2020-07-29 00:12:42 +03:00
|
|
|
|
|
|
|
|
|
DOCS: https://spacy.io/api/language#component
|
2020-07-22 14:42:59 +03:00
|
|
|
|
"""
|
|
|
|
|
if name is not None and not isinstance(name, str):
|
|
|
|
|
raise ValueError(Errors.E963.format(decorator="component"))
|
|
|
|
|
component_name = name if name is not None else util.get_object_name(func)
|
|
|
|
|
|
|
|
|
|
def add_component(component_func: Callable[[Doc], Doc]) -> Callable:
|
|
|
|
|
if isinstance(func, type): # function is a class
|
|
|
|
|
raise ValueError(Errors.E965.format(name=component_name))
|
|
|
|
|
|
|
|
|
|
def factory_func(nlp: cls, name: str) -> Callable[[Doc], Doc]:
|
|
|
|
|
return component_func
|
|
|
|
|
|
|
|
|
|
cls.factory(
|
|
|
|
|
component_name,
|
|
|
|
|
assigns=assigns,
|
|
|
|
|
requires=requires,
|
|
|
|
|
retokenizes=retokenizes,
|
2020-07-26 14:18:43 +03:00
|
|
|
|
scores=scores,
|
2020-07-27 13:27:40 +03:00
|
|
|
|
default_score_weights=default_score_weights,
|
2020-07-22 14:42:59 +03:00
|
|
|
|
func=factory_func,
|
|
|
|
|
)
|
|
|
|
|
return component_func
|
|
|
|
|
|
|
|
|
|
if func is not None: # Support non-decorator use cases
|
|
|
|
|
return add_component(func)
|
|
|
|
|
return add_component
|
|
|
|
|
|
|
|
|
|
def get_pipe(self, name: str) -> Callable[[Doc], Doc]:
|
2017-10-07 01:25:54 +03:00
|
|
|
|
"""Get a pipeline component for a given component name.
|
|
|
|
|
|
2020-05-24 18:20:58 +03:00
|
|
|
|
name (str): Name of pipeline component to get.
|
2017-10-07 01:25:54 +03:00
|
|
|
|
RETURNS (callable): The pipeline component.
|
2019-03-15 18:23:17 +03:00
|
|
|
|
|
|
|
|
|
DOCS: https://spacy.io/api/language#get_pipe
|
2017-10-07 01:25:54 +03:00
|
|
|
|
"""
|
|
|
|
|
for pipe_name, component in self.pipeline:
|
|
|
|
|
if pipe_name == name:
|
|
|
|
|
return component
|
2018-04-03 16:50:31 +03:00
|
|
|
|
raise KeyError(Errors.E001.format(name=name, opts=self.pipe_names))
|
2017-10-07 01:25:54 +03:00
|
|
|
|
|
2020-07-22 14:42:59 +03:00
|
|
|
|
def create_pipe(
|
|
|
|
|
self,
|
|
|
|
|
factory_name: str,
|
|
|
|
|
name: Optional[str] = None,
|
2020-07-29 00:12:42 +03:00
|
|
|
|
*,
|
2020-07-22 14:42:59 +03:00
|
|
|
|
config: Optional[Dict[str, Any]] = SimpleFrozenDict(),
|
|
|
|
|
overrides: Optional[Dict[str, Any]] = SimpleFrozenDict(),
|
|
|
|
|
validate: bool = True,
|
|
|
|
|
) -> Callable[[Doc], Doc]:
|
|
|
|
|
"""Create a pipeline component. Mostly used internally. To create and
|
|
|
|
|
add a component to the pipeline, you can use nlp.add_pipe.
|
|
|
|
|
|
|
|
|
|
factory_name (str): Name of component factory.
|
|
|
|
|
name (Optional[str]): Optional name to assign to component instance.
|
|
|
|
|
Defaults to factory name if not set.
|
|
|
|
|
config (Optional[Dict[str, Any]]): Config parameters to use for this
|
|
|
|
|
component. Will be merged with default config, if available.
|
|
|
|
|
overrides (Optional[Dict[str, Any]]): Config overrides, typically
|
|
|
|
|
passed in via the CLI.
|
|
|
|
|
validate (bool): Whether to validate the component config against the
|
|
|
|
|
arguments and types expected by the factory.
|
|
|
|
|
RETURNS (Callable[[Doc], Doc]): The pipeline component.
|
2020-07-29 00:12:42 +03:00
|
|
|
|
|
|
|
|
|
DOCS: https://spacy.io/api/language#create_pipe
|
2017-10-07 01:25:54 +03:00
|
|
|
|
"""
|
2020-07-22 14:42:59 +03:00
|
|
|
|
name = name if name is not None else factory_name
|
|
|
|
|
if not isinstance(config, dict):
|
|
|
|
|
err = Errors.E962.format(style="config", name=name, cfg_type=type(config))
|
|
|
|
|
raise ValueError(err)
|
|
|
|
|
if not srsly.is_json_serializable(config):
|
|
|
|
|
raise ValueError(Errors.E961.format(config=config))
|
|
|
|
|
if not self.has_factory(factory_name):
|
|
|
|
|
err = Errors.E002.format(
|
|
|
|
|
name=factory_name,
|
|
|
|
|
opts=", ".join(self.factory_names),
|
|
|
|
|
method="create_pipe",
|
|
|
|
|
lang=util.get_object_name(self),
|
|
|
|
|
lang_code=self.lang,
|
2020-05-21 19:39:06 +03:00
|
|
|
|
)
|
2020-07-22 14:42:59 +03:00
|
|
|
|
raise ValueError(err)
|
|
|
|
|
pipe_meta = self.get_factory_meta(factory_name)
|
|
|
|
|
config = config or {}
|
|
|
|
|
# This is unideal, but the alternative would mean you always need to
|
|
|
|
|
# specify the full config settings, which is not really viable.
|
|
|
|
|
if pipe_meta.default_config:
|
|
|
|
|
config = util.deep_merge_configs(config, pipe_meta.default_config)
|
|
|
|
|
# We need to create a top-level key because Thinc doesn't allow resolving
|
|
|
|
|
# top-level references to registered functions. Also gives nicer errors.
|
|
|
|
|
# The name allows components to know their pipe name and use it in the
|
|
|
|
|
# losses etc. (even if multiple instances of the same factory are used)
|
|
|
|
|
internal_name = self.get_factory_name(factory_name)
|
|
|
|
|
# If the language-specific factory doesn't exist, try again with the
|
|
|
|
|
# not-specific name
|
|
|
|
|
if internal_name not in registry.factories:
|
|
|
|
|
internal_name = factory_name
|
|
|
|
|
config = {"nlp": self, "name": name, **config, "@factories": internal_name}
|
|
|
|
|
cfg = {factory_name: config}
|
|
|
|
|
# We're calling the internal _fill here to avoid constructing the
|
|
|
|
|
# registered functions twice
|
|
|
|
|
# TODO: customize validation to make it more readable / relate it to
|
|
|
|
|
# pipeline component and why it failed, explain default config
|
|
|
|
|
resolved, filled = registry.resolve(cfg, validate=validate, overrides=overrides)
|
|
|
|
|
filled = filled[factory_name]
|
2020-07-22 18:29:31 +03:00
|
|
|
|
filled["factory"] = factory_name
|
2020-07-26 16:11:24 +03:00
|
|
|
|
filled.pop("@factories", None)
|
2020-07-22 14:42:59 +03:00
|
|
|
|
self._pipe_configs[name] = filled
|
|
|
|
|
return resolved[factory_name]
|
2017-10-07 01:25:54 +03:00
|
|
|
|
|
💫 Tidy up and auto-format .py files (#2983)
<!--- Provide a general summary of your changes in the title. -->
## Description
- [x] Use [`black`](https://github.com/ambv/black) to auto-format all `.py` files.
- [x] Update flake8 config to exclude very large files (lemmatization tables etc.)
- [x] Update code to be compatible with flake8 rules
- [x] Fix various small bugs, inconsistencies and messy stuff in the language data
- [x] Update docs to explain new code style (`black`, `flake8`, when to use `# fmt: off` and `# fmt: on` and what `# noqa` means)
Once #2932 is merged, which auto-formats and tidies up the CLI, we'll be able to run `flake8 spacy` actually get meaningful results.
At the moment, the code style and linting isn't applied automatically, but I'm hoping that the new [GitHub Actions](https://github.com/features/actions) will let us auto-format pull requests and post comments with relevant linting information.
### Types of change
enhancement, code style
## Checklist
<!--- Before you submit the PR, go over this checklist and make sure you can
tick off all the boxes. [] -> [x] -->
- [x] I have submitted the spaCy Contributor Agreement.
- [x] I ran the tests, and all new and existing tests passed.
- [x] My changes don't require a change to the documentation, or if they do, I've added all required information.
2018-11-30 19:03:03 +03:00
|
|
|
|
def add_pipe(
|
2020-07-22 14:42:59 +03:00
|
|
|
|
self,
|
|
|
|
|
factory_name: str,
|
|
|
|
|
name: Optional[str] = None,
|
|
|
|
|
*,
|
|
|
|
|
before: Optional[Union[str, int]] = None,
|
|
|
|
|
after: Optional[Union[str, int]] = None,
|
|
|
|
|
first: Optional[bool] = None,
|
|
|
|
|
last: Optional[bool] = None,
|
|
|
|
|
config: Optional[Dict[str, Any]] = SimpleFrozenDict(),
|
|
|
|
|
overrides: Optional[Dict[str, Any]] = SimpleFrozenDict(),
|
|
|
|
|
validate: bool = True,
|
|
|
|
|
) -> Callable[[Doc], Doc]:
|
2017-10-07 01:25:54 +03:00
|
|
|
|
"""Add a component to the processing pipeline. Valid components are
|
2017-10-27 15:40:14 +03:00
|
|
|
|
callables that take a `Doc` object, modify it and return it. Only one
|
|
|
|
|
of before/after/first/last can be set. Default behaviour is "last".
|
2017-10-07 01:25:54 +03:00
|
|
|
|
|
2020-07-22 14:42:59 +03:00
|
|
|
|
factory_name (str): Name of the component factory.
|
2020-05-24 18:20:58 +03:00
|
|
|
|
name (str): Name of pipeline component. Overwrites existing
|
2017-10-07 01:25:54 +03:00
|
|
|
|
component.name attribute if available. If no name is set and
|
|
|
|
|
the component exposes no name attribute, component.__name__ is
|
2017-10-27 15:40:14 +03:00
|
|
|
|
used. An error is raised if a name already exists in the pipeline.
|
2020-07-22 14:42:59 +03:00
|
|
|
|
before (Union[str, int]): Name or index of the component to insert new
|
|
|
|
|
component directly before.
|
|
|
|
|
after (Union[str, int]): Name or index of the component to insert new
|
|
|
|
|
component directly after.
|
|
|
|
|
first (bool): If True, insert component first in the pipeline.
|
|
|
|
|
last (bool): If True, insert component last in the pipeline.
|
|
|
|
|
config (Optional[Dict[str, Any]]): Config parameters to use for this
|
|
|
|
|
component. Will be merged with default config, if available.
|
|
|
|
|
overrides (Optional[Dict[str, Any]]): Config overrides, typically
|
|
|
|
|
passed in via the CLI.
|
|
|
|
|
validate (bool): Whether to validate the component config against the
|
|
|
|
|
arguments and types expected by the factory.
|
|
|
|
|
RETURNS (Callable[[Doc], Doc]): The pipeline component.
|
2017-10-07 01:25:54 +03:00
|
|
|
|
|
2019-03-15 18:23:17 +03:00
|
|
|
|
DOCS: https://spacy.io/api/language#add_pipe
|
2017-10-07 01:25:54 +03:00
|
|
|
|
"""
|
2020-07-22 14:42:59 +03:00
|
|
|
|
if not isinstance(factory_name, str):
|
|
|
|
|
bad_val = repr(factory_name)
|
|
|
|
|
err = Errors.E966.format(component=bad_val, name=name)
|
|
|
|
|
raise ValueError(err)
|
|
|
|
|
if not self.has_factory(factory_name):
|
|
|
|
|
err = Errors.E002.format(
|
|
|
|
|
name=factory_name,
|
|
|
|
|
opts=", ".join(self.factory_names),
|
|
|
|
|
method="add_pipe",
|
|
|
|
|
lang=util.get_object_name(self),
|
|
|
|
|
lang_code=self.lang,
|
|
|
|
|
)
|
|
|
|
|
name = name if name is not None else factory_name
|
2017-10-07 01:25:54 +03:00
|
|
|
|
if name in self.pipe_names:
|
2018-04-03 16:50:31 +03:00
|
|
|
|
raise ValueError(Errors.E007.format(name=name, opts=self.pipe_names))
|
2020-07-22 14:42:59 +03:00
|
|
|
|
pipe_component = self.create_pipe(
|
|
|
|
|
factory_name,
|
|
|
|
|
name=name,
|
|
|
|
|
config=config,
|
|
|
|
|
overrides=overrides,
|
|
|
|
|
validate=validate,
|
|
|
|
|
)
|
|
|
|
|
pipe_index = self._get_pipe_index(before, after, first, last)
|
|
|
|
|
self._pipe_meta[name] = self.get_factory_meta(factory_name)
|
|
|
|
|
self.pipeline.insert(pipe_index, (name, pipe_component))
|
2019-10-27 15:35:49 +03:00
|
|
|
|
if ENABLE_PIPELINE_ANALYSIS:
|
2020-07-22 14:42:59 +03:00
|
|
|
|
analyze_pipes(self, name, pipe_index)
|
|
|
|
|
return pipe_component
|
2017-06-04 23:52:09 +03:00
|
|
|
|
|
2020-07-22 14:42:59 +03:00
|
|
|
|
def _get_pipe_index(
|
|
|
|
|
self,
|
|
|
|
|
before: Optional[Union[str, int]] = None,
|
|
|
|
|
after: Optional[Union[str, int]] = None,
|
|
|
|
|
first: Optional[bool] = None,
|
|
|
|
|
last: Optional[bool] = None,
|
|
|
|
|
) -> int:
|
|
|
|
|
"""Determine where to insert a pipeline component based on the before/
|
|
|
|
|
after/first/last values.
|
|
|
|
|
|
|
|
|
|
before (str): Name or index of the component to insert directly before.
|
|
|
|
|
after (str): Name or index of component to insert directly after.
|
|
|
|
|
first (bool): If True, insert component first in the pipeline.
|
|
|
|
|
last (bool): If True, insert component last in the pipeline.
|
|
|
|
|
RETURNS (int): The index of the new pipeline component.
|
|
|
|
|
"""
|
|
|
|
|
all_args = {"before": before, "after": after, "first": first, "last": last}
|
|
|
|
|
if sum(arg is not None for arg in [before, after, first, last]) >= 2:
|
|
|
|
|
raise ValueError(Errors.E006.format(args=all_args, opts=self.pipe_names))
|
|
|
|
|
if last or not any(value is not None for value in [first, before, after]):
|
|
|
|
|
return len(self.pipeline)
|
|
|
|
|
elif first:
|
|
|
|
|
return 0
|
|
|
|
|
elif isinstance(before, str):
|
|
|
|
|
if before not in self.pipe_names:
|
|
|
|
|
raise ValueError(Errors.E001.format(name=before, opts=self.pipe_names))
|
|
|
|
|
return self.pipe_names.index(before)
|
|
|
|
|
elif isinstance(after, str):
|
|
|
|
|
if after not in self.pipe_names:
|
|
|
|
|
raise ValueError(Errors.E001.format(name=after, opts=self.pipe_names))
|
|
|
|
|
return self.pipe_names.index(after) + 1
|
|
|
|
|
# We're only accepting indices referring to components that exist
|
|
|
|
|
# (can't just do isinstance here because bools are instance of int, too)
|
|
|
|
|
elif type(before) == int:
|
|
|
|
|
if before >= len(self.pipeline) or before < 0:
|
|
|
|
|
err = Errors.E959.format(dir="before", idx=before, opts=self.pipe_names)
|
|
|
|
|
raise ValueError(err)
|
|
|
|
|
return before
|
|
|
|
|
elif type(after) == int:
|
|
|
|
|
if after >= len(self.pipeline) or after < 0:
|
|
|
|
|
err = Errors.E959.format(dir="after", idx=after, opts=self.pipe_names)
|
|
|
|
|
raise ValueError(err)
|
|
|
|
|
return after + 1
|
|
|
|
|
raise ValueError(Errors.E006.format(args=all_args, opts=self.pipe_names))
|
|
|
|
|
|
|
|
|
|
def has_pipe(self, name: str) -> bool:
|
2017-10-17 12:20:07 +03:00
|
|
|
|
"""Check if a component name is present in the pipeline. Equivalent to
|
|
|
|
|
`name in nlp.pipe_names`.
|
|
|
|
|
|
2020-05-24 18:20:58 +03:00
|
|
|
|
name (str): Name of the component.
|
2017-10-27 15:40:14 +03:00
|
|
|
|
RETURNS (bool): Whether a component of the name exists in the pipeline.
|
2019-03-15 18:23:17 +03:00
|
|
|
|
|
|
|
|
|
DOCS: https://spacy.io/api/language#has_pipe
|
2017-10-17 12:20:07 +03:00
|
|
|
|
"""
|
|
|
|
|
return name in self.pipe_names
|
|
|
|
|
|
2020-07-22 14:42:59 +03:00
|
|
|
|
def replace_pipe(
|
|
|
|
|
self,
|
|
|
|
|
name: str,
|
|
|
|
|
factory_name: str,
|
2020-07-29 00:12:42 +03:00
|
|
|
|
*,
|
2020-07-22 14:42:59 +03:00
|
|
|
|
config: Dict[str, Any] = SimpleFrozenDict(),
|
|
|
|
|
validate: bool = True,
|
|
|
|
|
) -> None:
|
2017-10-07 01:25:54 +03:00
|
|
|
|
"""Replace a component in the pipeline.
|
|
|
|
|
|
2020-05-24 18:20:58 +03:00
|
|
|
|
name (str): Name of the component to replace.
|
2020-07-22 14:42:59 +03:00
|
|
|
|
factory_name (str): Factory name of replacement component.
|
|
|
|
|
config (Optional[Dict[str, Any]]): Config parameters to use for this
|
|
|
|
|
component. Will be merged with default config, if available.
|
|
|
|
|
validate (bool): Whether to validate the component config against the
|
|
|
|
|
arguments and types expected by the factory.
|
2019-03-15 18:23:17 +03:00
|
|
|
|
|
|
|
|
|
DOCS: https://spacy.io/api/language#replace_pipe
|
2017-10-07 01:25:54 +03:00
|
|
|
|
"""
|
|
|
|
|
if name not in self.pipe_names:
|
2018-04-03 16:50:31 +03:00
|
|
|
|
raise ValueError(Errors.E001.format(name=name, opts=self.pipe_names))
|
2020-07-22 14:42:59 +03:00
|
|
|
|
if hasattr(factory_name, "__call__"):
|
|
|
|
|
err = Errors.E968.format(component=repr(factory_name), name=name)
|
|
|
|
|
raise ValueError(err)
|
|
|
|
|
# We need to delegate to Language.add_pipe here instead of just writing
|
|
|
|
|
# to Language.pipeline to make sure the configs are handled correctly
|
|
|
|
|
pipe_index = self.pipe_names.index(name)
|
|
|
|
|
self.remove_pipe(name)
|
|
|
|
|
if not len(self.pipeline): # we have no components to insert before/after
|
|
|
|
|
self.add_pipe(factory_name, name=name)
|
|
|
|
|
else:
|
|
|
|
|
self.add_pipe(factory_name, name=name, before=pipe_index)
|
2019-10-27 15:35:49 +03:00
|
|
|
|
if ENABLE_PIPELINE_ANALYSIS:
|
2020-07-22 14:42:59 +03:00
|
|
|
|
analyze_all_pipes(self)
|
2017-10-07 01:25:54 +03:00
|
|
|
|
|
2020-07-22 14:42:59 +03:00
|
|
|
|
def rename_pipe(self, old_name: str, new_name: str) -> None:
|
2017-10-07 01:25:54 +03:00
|
|
|
|
"""Rename a pipeline component.
|
|
|
|
|
|
2020-05-24 18:20:58 +03:00
|
|
|
|
old_name (str): Name of the component to rename.
|
|
|
|
|
new_name (str): New name of the component.
|
2019-03-15 18:23:17 +03:00
|
|
|
|
|
|
|
|
|
DOCS: https://spacy.io/api/language#rename_pipe
|
2017-10-07 01:25:54 +03:00
|
|
|
|
"""
|
|
|
|
|
if old_name not in self.pipe_names:
|
2018-04-03 16:50:31 +03:00
|
|
|
|
raise ValueError(Errors.E001.format(name=old_name, opts=self.pipe_names))
|
2017-10-07 01:25:54 +03:00
|
|
|
|
if new_name in self.pipe_names:
|
2018-04-03 16:50:31 +03:00
|
|
|
|
raise ValueError(Errors.E007.format(name=new_name, opts=self.pipe_names))
|
2017-10-07 01:25:54 +03:00
|
|
|
|
i = self.pipe_names.index(old_name)
|
|
|
|
|
self.pipeline[i] = (new_name, self.pipeline[i][1])
|
2020-07-22 14:42:59 +03:00
|
|
|
|
self._pipe_meta[new_name] = self._pipe_meta.pop(old_name)
|
|
|
|
|
self._pipe_configs[new_name] = self._pipe_configs.pop(old_name)
|
2017-10-07 01:25:54 +03:00
|
|
|
|
|
2020-07-22 14:42:59 +03:00
|
|
|
|
def remove_pipe(self, name: str) -> Tuple[str, Callable[[Doc], Doc]]:
|
2017-10-07 01:25:54 +03:00
|
|
|
|
"""Remove a component from the pipeline.
|
|
|
|
|
|
2020-05-24 18:20:58 +03:00
|
|
|
|
name (str): Name of the component to remove.
|
2017-10-07 02:04:50 +03:00
|
|
|
|
RETURNS (tuple): A `(name, component)` tuple of the removed component.
|
2019-03-15 18:23:17 +03:00
|
|
|
|
|
|
|
|
|
DOCS: https://spacy.io/api/language#remove_pipe
|
2017-10-07 01:25:54 +03:00
|
|
|
|
"""
|
|
|
|
|
if name not in self.pipe_names:
|
2018-04-03 16:50:31 +03:00
|
|
|
|
raise ValueError(Errors.E001.format(name=name, opts=self.pipe_names))
|
2019-10-30 21:04:17 +03:00
|
|
|
|
removed = self.pipeline.pop(self.pipe_names.index(name))
|
2020-07-22 14:42:59 +03:00
|
|
|
|
# We're only removing the component itself from the metas/configs here
|
|
|
|
|
# because factory may be used for something else
|
|
|
|
|
self._pipe_meta.pop(name)
|
|
|
|
|
self._pipe_configs.pop(name)
|
2019-10-27 15:35:49 +03:00
|
|
|
|
if ENABLE_PIPELINE_ANALYSIS:
|
2020-07-22 14:42:59 +03:00
|
|
|
|
analyze_all_pipes(self)
|
2019-10-30 21:04:17 +03:00
|
|
|
|
return removed
|
2017-06-04 23:52:09 +03:00
|
|
|
|
|
2020-07-22 14:42:59 +03:00
|
|
|
|
def __call__(
|
|
|
|
|
self,
|
|
|
|
|
text: str,
|
2020-07-29 00:12:42 +03:00
|
|
|
|
*,
|
2020-07-22 14:42:59 +03:00
|
|
|
|
disable: Iterable[str] = tuple(),
|
|
|
|
|
component_cfg: Optional[Dict[str, Dict[str, Any]]] = None,
|
|
|
|
|
) -> Doc:
|
2017-10-07 01:26:05 +03:00
|
|
|
|
"""Apply the pipeline to some text. The text can span multiple sentences,
|
2020-05-21 00:06:39 +03:00
|
|
|
|
and can contain arbitrary whitespace. Alignment into the original string
|
2015-08-25 16:37:17 +03:00
|
|
|
|
is preserved.
|
2016-12-18 18:54:52 +03:00
|
|
|
|
|
2020-05-24 18:20:58 +03:00
|
|
|
|
text (str): The text to be processed.
|
2017-05-26 13:33:54 +03:00
|
|
|
|
disable (list): Names of the pipeline components to disable.
|
2020-07-29 00:12:42 +03:00
|
|
|
|
component_cfg (Dict[str, dict]): An optional dictionary with extra
|
|
|
|
|
keyword arguments for specific components.
|
2017-05-19 00:57:38 +03:00
|
|
|
|
RETURNS (Doc): A container for accessing the annotations.
|
2016-11-01 14:25:36 +03:00
|
|
|
|
|
2019-03-15 18:23:17 +03:00
|
|
|
|
DOCS: https://spacy.io/api/language#call
|
2015-08-25 16:37:17 +03:00
|
|
|
|
"""
|
💫 Port master changes over to develop (#2979)
* Create aryaprabhudesai.md (#2681)
* Update _install.jade (#2688)
Typo fix: "models" -> "model"
* Add FAC to spacy.explain (resolves #2706)
* Remove docstrings for deprecated arguments (see #2703)
* When calling getoption() in conftest.py, pass a default option (#2709)
* When calling getoption() in conftest.py, pass a default option
This is necessary to allow testing an installed spacy by running:
pytest --pyargs spacy
* Add contributor agreement
* update bengali token rules for hyphen and digits (#2731)
* Less norm computations in token similarity (#2730)
* Less norm computations in token similarity
* Contributor agreement
* Remove ')' for clarity (#2737)
Sorry, don't mean to be nitpicky, I just noticed this when going through the CLI and thought it was a quick fix. That said, if this was intention than please let me know.
* added contributor agreement for mbkupfer (#2738)
* Basic support for Telugu language (#2751)
* Lex _attrs for polish language (#2750)
* Signed spaCy contributor agreement
* Added polish version of english lex_attrs
* Introduces a bulk merge function, in order to solve issue #653 (#2696)
* Fix comment
* Introduce bulk merge to increase performance on many span merges
* Sign contributor agreement
* Implement pull request suggestions
* Describe converters more explicitly (see #2643)
* Add multi-threading note to Language.pipe (resolves #2582) [ci skip]
* Fix formatting
* Fix dependency scheme docs (closes #2705) [ci skip]
* Don't set stop word in example (closes #2657) [ci skip]
* Add words to portuguese language _num_words (#2759)
* Add words to portuguese language _num_words
* Add words to portuguese language _num_words
* Update Indonesian model (#2752)
* adding e-KTP in tokenizer exceptions list
* add exception token
* removing lines with containing space as it won't matter since we use .split() method in the end, added new tokens in exception
* add tokenizer exceptions list
* combining base_norms with norm_exceptions
* adding norm_exception
* fix double key in lemmatizer
* remove unused import on punctuation.py
* reformat stop_words to reduce number of lines, improve readibility
* updating tokenizer exception
* implement is_currency for lang/id
* adding orth_first_upper in tokenizer_exceptions
* update the norm_exception list
* remove bunch of abbreviations
* adding contributors file
* Fixed spaCy+Keras example (#2763)
* bug fixes in keras example
* created contributor agreement
* Adding French hyphenated first name (#2786)
* Fix typo (closes #2784)
* Fix typo (#2795) [ci skip]
Fixed typo on line 6 "regcognizer --> recognizer"
* Adding basic support for Sinhala language. (#2788)
* adding Sinhala language package, stop words, examples and lex_attrs.
* Adding contributor agreement
* Updating contributor agreement
* Also include lowercase norm exceptions
* Fix error (#2802)
* Fix error
ValueError: cannot resize an array that references or is referenced
by another array in this way. Use the resize function
* added spaCy Contributor Agreement
* Add charlax's contributor agreement (#2805)
* agreement of contributor, may I introduce a tiny pl languge contribution (#2799)
* Contributors agreement
* Contributors agreement
* Contributors agreement
* Add jupyter=True to displacy.render in documentation (#2806)
* Revert "Also include lowercase norm exceptions"
This reverts commit 70f4e8adf37cfcfab60be2b97d6deae949b30e9e.
* Remove deprecated encoding argument to msgpack
* Set up dependency tree pattern matching skeleton (#2732)
* Fix bug when too many entity types. Fixes #2800
* Fix Python 2 test failure
* Require older msgpack-numpy
* Restore encoding arg on msgpack-numpy
* Try to fix version pin for msgpack-numpy
* Update Portuguese Language (#2790)
* Add words to portuguese language _num_words
* Add words to portuguese language _num_words
* Portuguese - Add/remove stopwords, fix tokenizer, add currency symbols
* Extended punctuation and norm_exceptions in the Portuguese language
* Correct error in spacy universe docs concerning spacy-lookup (#2814)
* Update Keras Example for (Parikh et al, 2016) implementation (#2803)
* bug fixes in keras example
* created contributor agreement
* baseline for Parikh model
* initial version of parikh 2016 implemented
* tested asymmetric models
* fixed grevious error in normalization
* use standard SNLI test file
* begin to rework parikh example
* initial version of running example
* start to document the new version
* start to document the new version
* Update Decompositional Attention.ipynb
* fixed calls to similarity
* updated the README
* import sys package duh
* simplified indexing on mapping word to IDs
* stupid python indent error
* added code from https://github.com/tensorflow/tensorflow/issues/3388 for tf bug workaround
* Fix typo (closes #2815) [ci skip]
* Update regex version dependency
* Set version to 2.0.13.dev3
* Skip seemingly problematic test
* Remove problematic test
* Try previous version of regex
* Revert "Remove problematic test"
This reverts commit bdebbef45552d698d390aa430b527ee27830f11b.
* Unskip test
* Try older version of regex
* 💫 Update training examples and use minibatching (#2830)
<!--- Provide a general summary of your changes in the title. -->
## Description
Update the training examples in `/examples/training` to show usage of spaCy's `minibatch` and `compounding` helpers ([see here](https://spacy.io/usage/training#tips-batch-size) for details). The lack of batching in the examples has caused some confusion in the past, especially for beginners who would copy-paste the examples, update them with large training sets and experienced slow and unsatisfying results.
### Types of change
enhancements
## Checklist
<!--- Before you submit the PR, go over this checklist and make sure you can
tick off all the boxes. [] -> [x] -->
- [x] I have submitted the spaCy Contributor Agreement.
- [x] I ran the tests, and all new and existing tests passed.
- [x] My changes don't require a change to the documentation, or if they do, I've added all required information.
* Visual C++ link updated (#2842) (closes #2841) [ci skip]
* New landing page
* Add contribution agreement
* Correcting lang/ru/examples.py (#2845)
* Correct some grammatical inaccuracies in lang\ru\examples.py; filled Contributor Agreement
* Correct some grammatical inaccuracies in lang\ru\examples.py
* Move contributor agreement to separate file
* Set version to 2.0.13.dev4
* Add Persian(Farsi) language support (#2797)
* Also include lowercase norm exceptions
* Remove in favour of https://github.com/explosion/spaCy/graphs/contributors
* Rule-based French Lemmatizer (#2818)
<!--- Provide a general summary of your changes in the title. -->
## Description
<!--- Use this section to describe your changes. If your changes required
testing, include information about the testing environment and the tests you
ran. If your test fixes a bug reported in an issue, don't forget to include the
issue number. If your PR is still a work in progress, that's totally fine – just
include a note to let us know. -->
Add a rule-based French Lemmatizer following the english one and the excellent PR for [greek language optimizations](https://github.com/explosion/spaCy/pull/2558) to adapt the Lemmatizer class.
### Types of change
<!-- What type of change does your PR cover? Is it a bug fix, an enhancement
or new feature, or a change to the documentation? -->
- Lemma dictionary used can be found [here](http://infolingu.univ-mlv.fr/DonneesLinguistiques/Dictionnaires/telechargement.html), I used the XML version.
- Add several files containing exhaustive list of words for each part of speech
- Add some lemma rules
- Add POS that are not checked in the standard Lemmatizer, i.e PRON, DET, ADV and AUX
- Modify the Lemmatizer class to check in lookup table as a last resort if POS not mentionned
- Modify the lemmatize function to check in lookup table as a last resort
- Init files are updated so the model can support all the functionalities mentioned above
- Add words to tokenizer_exceptions_list.py in respect to regex used in tokenizer_exceptions.py
## Checklist
<!--- Before you submit the PR, go over this checklist and make sure you can
tick off all the boxes. [] -> [x] -->
- [X] I have submitted the spaCy Contributor Agreement.
- [X] I ran the tests, and all new and existing tests passed.
- [X] My changes don't require a change to the documentation, or if they do, I've added all required information.
* Set version to 2.0.13
* Fix formatting and consistency
* Update docs for new version [ci skip]
* Increment version [ci skip]
* Add info on wheels [ci skip]
* Adding "This is a sentence" example to Sinhala (#2846)
* Add wheels badge
* Update badge [ci skip]
* Update README.rst [ci skip]
* Update murmurhash pin
* Increment version to 2.0.14.dev0
* Update GPU docs for v2.0.14
* Add wheel to setup_requires
* Import prefer_gpu and require_gpu functions from Thinc
* Add tests for prefer_gpu() and require_gpu()
* Update requirements and setup.py
* Workaround bug in thinc require_gpu
* Set version to v2.0.14
* Update push-tag script
* Unhack prefer_gpu
* Require thinc 6.10.6
* Update prefer_gpu and require_gpu docs [ci skip]
* Fix specifiers for GPU
* Set version to 2.0.14.dev1
* Set version to 2.0.14
* Update Thinc version pin
* Increment version
* Fix msgpack-numpy version pin
* Increment version
* Update version to 2.0.16
* Update version [ci skip]
* Redundant ')' in the Stop words' example (#2856)
<!--- Provide a general summary of your changes in the title. -->
## Description
<!--- Use this section to describe your changes. If your changes required
testing, include information about the testing environment and the tests you
ran. If your test fixes a bug reported in an issue, don't forget to include the
issue number. If your PR is still a work in progress, that's totally fine – just
include a note to let us know. -->
### Types of change
<!-- What type of change does your PR cover? Is it a bug fix, an enhancement
or new feature, or a change to the documentation? -->
## Checklist
<!--- Before you submit the PR, go over this checklist and make sure you can
tick off all the boxes. [] -> [x] -->
- [ ] I have submitted the spaCy Contributor Agreement.
- [ ] I ran the tests, and all new and existing tests passed.
- [ ] My changes don't require a change to the documentation, or if they do, I've added all required information.
* Documentation improvement regarding joblib and SO (#2867)
Some documentation improvements
## Description
1. Fixed the dead URL to joblib
2. Fixed Stack Overflow brand name (with space)
### Types of change
Documentation
## Checklist
<!--- Before you submit the PR, go over this checklist and make sure you can
tick off all the boxes. [] -> [x] -->
- [x] I have submitted the spaCy Contributor Agreement.
- [x] I ran the tests, and all new and existing tests passed.
- [x] My changes don't require a change to the documentation, or if they do, I've added all required information.
* raise error when setting overlapping entities as doc.ents (#2880)
* Fix out-of-bounds access in NER training
The helper method state.B(1) gets the index of the first token of the
buffer, or -1 if no such token exists. Normally this is safe because we
pass this to functions like state.safe_get(), which returns an empty
token. Here we used it directly as an array index, which is not okay!
This error may have been the cause of out-of-bounds access errors during
training. Similar errors may still be around, so much be hunted down.
Hunting this one down took a long time...I printed out values across
training runs and diffed, looking for points of divergence between
runs, when no randomness should be allowed.
* Change PyThaiNLP Url (#2876)
* Fix missing comma
* Add example showing a fix-up rule for space entities
* Set version to 2.0.17.dev0
* Update regex version
* Revert "Update regex version"
This reverts commit 62358dd867d15bc6a475942dff34effba69dd70a.
* Try setting older regex version, to align with conda
* Set version to 2.0.17
* Add spacy-js to universe [ci-skip]
* Add spacy-raspberry to universe (closes #2889)
* Add script to validate universe json [ci skip]
* Removed space in docs + added contributor indo (#2909)
* - removed unneeded space in documentation
* - added contributor info
* Allow input text of length up to max_length, inclusive (#2922)
* Include universe spec for spacy-wordnet component (#2919)
* feat: include universe spec for spacy-wordnet component
* chore: include spaCy contributor agreement
* Minor formatting changes [ci skip]
* Fix image [ci skip]
Twitter URL doesn't work on live site
* Check if the word is in one of the regular lists specific to each POS (#2886)
* 💫 Create random IDs for SVGs to prevent ID clashes (#2927)
Resolves #2924.
## Description
Fixes problem where multiple visualizations in Jupyter notebooks would have clashing arc IDs, resulting in weirdly positioned arc labels. Generating a random ID prefix so even identical parses won't receive the same IDs for consistency (even if effect of ID clash isn't noticable here.)
### Types of change
bug fix
## Checklist
<!--- Before you submit the PR, go over this checklist and make sure you can
tick off all the boxes. [] -> [x] -->
- [x] I have submitted the spaCy Contributor Agreement.
- [x] I ran the tests, and all new and existing tests passed.
- [x] My changes don't require a change to the documentation, or if they do, I've added all required information.
* Fix typo [ci skip]
* fixes symbolic link on py3 and windows (#2949)
* fixes symbolic link on py3 and windows
during setup of spacy using command
python -m spacy link en_core_web_sm en
closes #2948
* Update spacy/compat.py
Co-Authored-By: cicorias <cicorias@users.noreply.github.com>
* Fix formatting
* Update universe [ci skip]
* Catalan Language Support (#2940)
* Catalan language Support
* Ddding Catalan to documentation
* Sort languages alphabetically [ci skip]
* Update tests for pytest 4.x (#2965)
<!--- Provide a general summary of your changes in the title. -->
## Description
- [x] Replace marks in params for pytest 4.0 compat ([see here](https://docs.pytest.org/en/latest/deprecations.html#marks-in-pytest-mark-parametrize))
- [x] Un-xfail passing tests (some fixes in a recent update resolved a bunch of issues, but tests were apparently never updated here)
### Types of change
<!-- What type of change does your PR cover? Is it a bug fix, an enhancement
or new feature, or a change to the documentation? -->
## Checklist
<!--- Before you submit the PR, go over this checklist and make sure you can
tick off all the boxes. [] -> [x] -->
- [x] I have submitted the spaCy Contributor Agreement.
- [x] I ran the tests, and all new and existing tests passed.
- [x] My changes don't require a change to the documentation, or if they do, I've added all required information.
* Fix regex pin to harmonize with conda (#2964)
* Update README.rst
* Fix bug where Vocab.prune_vector did not use 'batch_size' (#2977)
Fixes #2976
* Fix typo
* Fix typo
* Remove duplicate file
* Require thinc 7.0.0.dev2
Fixes bug in gpu_ops that would use cupy instead of numpy on CPU
* Add missing import
* Fix error IDs
* Fix tests
2018-11-29 18:30:29 +03:00
|
|
|
|
if len(text) > self.max_length:
|
💫 Tidy up and auto-format .py files (#2983)
<!--- Provide a general summary of your changes in the title. -->
## Description
- [x] Use [`black`](https://github.com/ambv/black) to auto-format all `.py` files.
- [x] Update flake8 config to exclude very large files (lemmatization tables etc.)
- [x] Update code to be compatible with flake8 rules
- [x] Fix various small bugs, inconsistencies and messy stuff in the language data
- [x] Update docs to explain new code style (`black`, `flake8`, when to use `# fmt: off` and `# fmt: on` and what `# noqa` means)
Once #2932 is merged, which auto-formats and tidies up the CLI, we'll be able to run `flake8 spacy` actually get meaningful results.
At the moment, the code style and linting isn't applied automatically, but I'm hoping that the new [GitHub Actions](https://github.com/features/actions) will let us auto-format pull requests and post comments with relevant linting information.
### Types of change
enhancement, code style
## Checklist
<!--- Before you submit the PR, go over this checklist and make sure you can
tick off all the boxes. [] -> [x] -->
- [x] I have submitted the spaCy Contributor Agreement.
- [x] I ran the tests, and all new and existing tests passed.
- [x] My changes don't require a change to the documentation, or if they do, I've added all required information.
2018-11-30 19:03:03 +03:00
|
|
|
|
raise ValueError(
|
|
|
|
|
Errors.E088.format(length=len(text), max_length=self.max_length)
|
|
|
|
|
)
|
2016-10-14 18:38:29 +03:00
|
|
|
|
doc = self.make_doc(text)
|
2019-03-11 01:36:47 +03:00
|
|
|
|
if component_cfg is None:
|
|
|
|
|
component_cfg = {}
|
2017-10-07 01:25:54 +03:00
|
|
|
|
for name, proc in self.pipeline:
|
2017-05-26 13:33:54 +03:00
|
|
|
|
if name in disable:
|
2017-05-16 12:21:59 +03:00
|
|
|
|
continue
|
💫 Tidy up and auto-format .py files (#2983)
<!--- Provide a general summary of your changes in the title. -->
## Description
- [x] Use [`black`](https://github.com/ambv/black) to auto-format all `.py` files.
- [x] Update flake8 config to exclude very large files (lemmatization tables etc.)
- [x] Update code to be compatible with flake8 rules
- [x] Fix various small bugs, inconsistencies and messy stuff in the language data
- [x] Update docs to explain new code style (`black`, `flake8`, when to use `# fmt: off` and `# fmt: on` and what `# noqa` means)
Once #2932 is merged, which auto-formats and tidies up the CLI, we'll be able to run `flake8 spacy` actually get meaningful results.
At the moment, the code style and linting isn't applied automatically, but I'm hoping that the new [GitHub Actions](https://github.com/features/actions) will let us auto-format pull requests and post comments with relevant linting information.
### Types of change
enhancement, code style
## Checklist
<!--- Before you submit the PR, go over this checklist and make sure you can
tick off all the boxes. [] -> [x] -->
- [x] I have submitted the spaCy Contributor Agreement.
- [x] I ran the tests, and all new and existing tests passed.
- [x] My changes don't require a change to the documentation, or if they do, I've added all required information.
2018-11-30 19:03:03 +03:00
|
|
|
|
if not hasattr(proc, "__call__"):
|
2018-04-03 16:50:31 +03:00
|
|
|
|
raise ValueError(Errors.E003.format(component=type(proc), name=name))
|
2020-02-27 20:42:27 +03:00
|
|
|
|
try:
|
|
|
|
|
doc = proc(doc, **component_cfg.get(name, {}))
|
|
|
|
|
except KeyError:
|
|
|
|
|
raise ValueError(Errors.E109.format(name=name))
|
2018-04-03 16:50:31 +03:00
|
|
|
|
if doc is None:
|
|
|
|
|
raise ValueError(Errors.E005.format(name=name))
|
2016-05-17 17:55:42 +03:00
|
|
|
|
return doc
|
2015-08-25 16:37:17 +03:00
|
|
|
|
|
2020-07-22 14:42:59 +03:00
|
|
|
|
def disable_pipes(self, *names) -> "DisabledPipes":
|
2017-10-27 15:40:14 +03:00
|
|
|
|
"""Disable one or more pipeline components. If used as a context
|
|
|
|
|
manager, the pipeline will be restored to the initial state at the end
|
|
|
|
|
of the block. Otherwise, a DisabledPipes object is returned, that has
|
|
|
|
|
a `.restore()` method you can use to undo your changes.
|
2017-10-25 14:46:41 +03:00
|
|
|
|
|
2020-05-18 23:27:10 +03:00
|
|
|
|
This method has been deprecated since 3.0
|
2017-10-27 15:40:14 +03:00
|
|
|
|
"""
|
2020-05-18 23:27:10 +03:00
|
|
|
|
warnings.warn(Warnings.W096, DeprecationWarning)
|
2019-10-25 17:19:08 +03:00
|
|
|
|
if len(names) == 1 and isinstance(names[0], (list, tuple)):
|
|
|
|
|
names = names[0] # support list of names instead of spread
|
2020-05-18 23:27:10 +03:00
|
|
|
|
return DisabledPipes(self, names)
|
|
|
|
|
|
2020-07-22 14:42:59 +03:00
|
|
|
|
def select_pipes(
|
|
|
|
|
self,
|
2020-07-29 00:12:42 +03:00
|
|
|
|
*,
|
2020-07-22 14:42:59 +03:00
|
|
|
|
disable: Optional[Union[str, Iterable[str]]] = None,
|
|
|
|
|
enable: Optional[Union[str, Iterable[str]]] = None,
|
|
|
|
|
) -> "DisabledPipes":
|
2020-05-18 23:27:10 +03:00
|
|
|
|
"""Disable one or more pipeline components. If used as a context
|
|
|
|
|
manager, the pipeline will be restored to the initial state at the end
|
|
|
|
|
of the block. Otherwise, a DisabledPipes object is returned, that has
|
|
|
|
|
a `.restore()` method you can use to undo your changes.
|
|
|
|
|
|
|
|
|
|
disable (str or iterable): The name(s) of the pipes to disable
|
|
|
|
|
enable (str or iterable): The name(s) of the pipes to enable - all others will be disabled
|
|
|
|
|
|
|
|
|
|
DOCS: https://spacy.io/api/language#select_pipes
|
|
|
|
|
"""
|
|
|
|
|
if enable is None and disable is None:
|
|
|
|
|
raise ValueError(Errors.E991)
|
|
|
|
|
if disable is not None and isinstance(disable, str):
|
|
|
|
|
disable = [disable]
|
|
|
|
|
if enable is not None:
|
|
|
|
|
if isinstance(enable, str):
|
|
|
|
|
enable = [enable]
|
|
|
|
|
to_disable = [pipe for pipe in self.pipe_names if pipe not in enable]
|
|
|
|
|
# raise an error if the enable and disable keywords are not consistent
|
|
|
|
|
if disable is not None and disable != to_disable:
|
2020-05-19 17:20:03 +03:00
|
|
|
|
raise ValueError(
|
|
|
|
|
Errors.E992.format(
|
|
|
|
|
enable=enable, disable=disable, names=self.pipe_names
|
|
|
|
|
)
|
|
|
|
|
)
|
2020-05-18 23:27:10 +03:00
|
|
|
|
disable = to_disable
|
|
|
|
|
return DisabledPipes(self, disable)
|
2017-10-25 14:46:41 +03:00
|
|
|
|
|
2020-07-22 14:42:59 +03:00
|
|
|
|
def make_doc(self, text: str) -> Doc:
|
|
|
|
|
"""Turn a text into a Doc object.
|
|
|
|
|
|
|
|
|
|
text (str): The text to process.
|
|
|
|
|
RETURNS (Doc): The processed doc.
|
|
|
|
|
"""
|
2017-05-29 16:40:45 +03:00
|
|
|
|
return self.tokenizer(text)
|
|
|
|
|
|
2020-05-21 19:39:06 +03:00
|
|
|
|
def update(
|
|
|
|
|
self,
|
2020-07-22 14:42:59 +03:00
|
|
|
|
examples: Iterable[Example],
|
2020-07-29 00:12:42 +03:00
|
|
|
|
_: Optional[Any] = None,
|
2020-05-21 19:39:06 +03:00
|
|
|
|
*,
|
2020-07-22 14:42:59 +03:00
|
|
|
|
drop: float = 0.0,
|
|
|
|
|
sgd: Optional[Optimizer] = None,
|
|
|
|
|
losses: Optional[Dict[str, float]] = None,
|
|
|
|
|
component_cfg: Optional[Dict[str, Dict[str, Any]]] = None,
|
2020-05-21 19:39:06 +03:00
|
|
|
|
):
|
2017-05-19 00:57:38 +03:00
|
|
|
|
"""Update the models in the pipeline.
|
|
|
|
|
|
2020-07-09 20:43:39 +03:00
|
|
|
|
examples (Iterable[Example]): A batch of examples
|
2020-07-29 00:12:42 +03:00
|
|
|
|
_: Should not be set - serves to catch backwards-incompatible scripts.
|
2019-10-14 13:28:53 +03:00
|
|
|
|
drop (float): The dropout rate.
|
2020-07-09 20:43:39 +03:00
|
|
|
|
sgd (Optimizer): An optimizer.
|
|
|
|
|
losses (Dict[str, float]): Dictionary to update with the loss, keyed by component.
|
|
|
|
|
component_cfg (Dict[str, Dict]): Config parameters for specific pipeline
|
2019-05-24 15:06:26 +03:00
|
|
|
|
components, keyed by component name.
|
2020-07-09 20:43:39 +03:00
|
|
|
|
RETURNS (Dict[str, float]): The updated losses dictionary
|
2017-05-19 00:57:38 +03:00
|
|
|
|
|
2019-03-15 18:23:17 +03:00
|
|
|
|
DOCS: https://spacy.io/api/language#update
|
2017-05-19 00:57:38 +03:00
|
|
|
|
"""
|
2020-07-29 00:12:42 +03:00
|
|
|
|
if _ is not None:
|
2020-05-20 12:41:12 +03:00
|
|
|
|
raise ValueError(Errors.E989)
|
2020-07-09 20:43:39 +03:00
|
|
|
|
if losses is None:
|
|
|
|
|
losses = {}
|
2019-11-11 19:35:27 +03:00
|
|
|
|
if len(examples) == 0:
|
2020-07-09 20:43:39 +03:00
|
|
|
|
return losses
|
2020-07-22 14:42:59 +03:00
|
|
|
|
if not isinstance(examples, IterableInstance):
|
2020-07-12 15:03:23 +03:00
|
|
|
|
raise TypeError(
|
|
|
|
|
Errors.E978.format(
|
|
|
|
|
name="language", method="update", types=type(examples)
|
|
|
|
|
)
|
|
|
|
|
)
|
2020-07-06 14:02:36 +03:00
|
|
|
|
wrong_types = set([type(eg) for eg in examples if not isinstance(eg, Example)])
|
|
|
|
|
if wrong_types:
|
2020-07-12 15:03:23 +03:00
|
|
|
|
raise TypeError(
|
|
|
|
|
Errors.E978.format(name="language", method="update", types=wrong_types)
|
|
|
|
|
)
|
2017-08-20 15:42:07 +03:00
|
|
|
|
if sgd is None:
|
|
|
|
|
if self._optimizer is None:
|
2020-01-29 19:06:46 +03:00
|
|
|
|
self._optimizer = create_default_optimizer()
|
2017-08-20 15:42:07 +03:00
|
|
|
|
sgd = self._optimizer
|
2019-03-11 01:36:47 +03:00
|
|
|
|
if component_cfg is None:
|
|
|
|
|
component_cfg = {}
|
2020-05-22 16:55:45 +03:00
|
|
|
|
for i, (name, proc) in enumerate(self.pipeline):
|
2020-01-29 19:06:46 +03:00
|
|
|
|
component_cfg.setdefault(name, {})
|
|
|
|
|
component_cfg[name].setdefault("drop", drop)
|
2020-07-08 22:36:51 +03:00
|
|
|
|
component_cfg[name].setdefault("set_annotations", False)
|
2020-01-29 19:06:46 +03:00
|
|
|
|
for name, proc in self.pipeline:
|
💫 Tidy up and auto-format .py files (#2983)
<!--- Provide a general summary of your changes in the title. -->
## Description
- [x] Use [`black`](https://github.com/ambv/black) to auto-format all `.py` files.
- [x] Update flake8 config to exclude very large files (lemmatization tables etc.)
- [x] Update code to be compatible with flake8 rules
- [x] Fix various small bugs, inconsistencies and messy stuff in the language data
- [x] Update docs to explain new code style (`black`, `flake8`, when to use `# fmt: off` and `# fmt: on` and what `# noqa` means)
Once #2932 is merged, which auto-formats and tidies up the CLI, we'll be able to run `flake8 spacy` actually get meaningful results.
At the moment, the code style and linting isn't applied automatically, but I'm hoping that the new [GitHub Actions](https://github.com/features/actions) will let us auto-format pull requests and post comments with relevant linting information.
### Types of change
enhancement, code style
## Checklist
<!--- Before you submit the PR, go over this checklist and make sure you can
tick off all the boxes. [] -> [x] -->
- [x] I have submitted the spaCy Contributor Agreement.
- [x] I ran the tests, and all new and existing tests passed.
- [x] My changes don't require a change to the documentation, or if they do, I've added all required information.
2018-11-30 19:03:03 +03:00
|
|
|
|
if not hasattr(proc, "update"):
|
2017-05-22 02:43:31 +03:00
|
|
|
|
continue
|
2020-01-29 19:06:46 +03:00
|
|
|
|
proc.update(examples, sgd=None, losses=losses, **component_cfg[name])
|
2020-07-08 22:36:51 +03:00
|
|
|
|
if sgd not in (None, False):
|
2020-01-29 19:06:46 +03:00
|
|
|
|
for name, proc in self.pipeline:
|
|
|
|
|
if hasattr(proc, "model"):
|
|
|
|
|
proc.model.finish_update(sgd)
|
2020-07-09 20:43:39 +03:00
|
|
|
|
return losses
|
2017-05-16 17:17:30 +03:00
|
|
|
|
|
2020-07-22 14:42:59 +03:00
|
|
|
|
def rehearse(
|
|
|
|
|
self,
|
|
|
|
|
examples: Iterable[Example],
|
2020-07-29 00:12:42 +03:00
|
|
|
|
*,
|
2020-07-22 14:42:59 +03:00
|
|
|
|
sgd: Optional[Optimizer] = None,
|
|
|
|
|
losses: Optional[Dict[str, float]] = None,
|
|
|
|
|
component_cfg: Optional[Dict[str, Dict[str, Any]]] = None,
|
|
|
|
|
) -> Dict[str, float]:
|
💫 Better support for semi-supervised learning (#3035)
The new spacy pretrain command implemented BERT/ULMFit/etc-like transfer learning, using our Language Modelling with Approximate Outputs version of BERT's cloze task. Pretraining is convenient, but in some ways it's a bit of a strange solution. All we're doing is initialising the weights. At the same time, we're putting a lot of work into our optimisation so that it's less sensitive to initial conditions, and more likely to find good optima. I discuss this a bit in the pseudo-rehearsal blog post: https://explosion.ai/blog/pseudo-rehearsal-catastrophic-forgetting
Support semi-supervised learning in spacy train
One obvious way to improve these pretraining methods is to do multi-task learning, instead of just transfer learning. This has been shown to work very well: https://arxiv.org/pdf/1809.08370.pdf . This patch makes it easy to do this sort of thing.
Add a new argument to spacy train, --raw-text. This takes a jsonl file with unlabelled data that can be used in arbitrary ways to do semi-supervised learning.
Add a new method to the Language class and to pipeline components, .rehearse(). This is like .update(), but doesn't expect GoldParse objects. It takes a batch of Doc objects, and performs an update on some semi-supervised objective.
Move the BERT-LMAO objective out from spacy/cli/pretrain.py into spacy/_ml.py, so we can create a new pipeline component, ClozeMultitask. This can be specified as a parser or NER multitask in the spacy train command. Example usage:
python -m spacy train en ./tmp ~/data/en-core-web/train/nw.json ~/data/en-core-web/dev/nw.json --pipeline parser --raw-textt ~/data/unlabelled/reddit-100k.jsonl --vectors en_vectors_web_lg --parser-multitasks cloze
Implement rehearsal methods for pipeline components
The new --raw-text argument and nlp.rehearse() method also gives us a good place to implement the the idea in the pseudo-rehearsal blog post in the parser. This works as follows:
Add a new nlp.resume_training() method. This allocates copies of pre-trained models in the pipeline, setting things up for the rehearsal updates. It also returns an optimizer object. This also greatly reduces confusion around the nlp.begin_training() method, which randomises the weights, making it not suitable for adding new labels or otherwise fine-tuning a pre-trained model.
Implement rehearsal updates on the Parser class, making it available for the dependency parser and NER. During rehearsal, the initial model is used to supervise the model being trained. The current model is asked to match the predictions of the initial model on some data. This minimises catastrophic forgetting, by keeping the model's predictions close to the original. See the blog post for details.
Implement rehearsal updates for tagger
Implement rehearsal updates for text categoriz
2018-12-10 18:25:33 +03:00
|
|
|
|
"""Make a "rehearsal" update to the models in the pipeline, to prevent
|
|
|
|
|
forgetting. Rehearsal updates run an initial copy of the model over some
|
|
|
|
|
data, and update the model so its current predictions are more like the
|
2019-10-02 11:37:39 +03:00
|
|
|
|
initial ones. This is useful for keeping a pretrained model on-track,
|
💫 Better support for semi-supervised learning (#3035)
The new spacy pretrain command implemented BERT/ULMFit/etc-like transfer learning, using our Language Modelling with Approximate Outputs version of BERT's cloze task. Pretraining is convenient, but in some ways it's a bit of a strange solution. All we're doing is initialising the weights. At the same time, we're putting a lot of work into our optimisation so that it's less sensitive to initial conditions, and more likely to find good optima. I discuss this a bit in the pseudo-rehearsal blog post: https://explosion.ai/blog/pseudo-rehearsal-catastrophic-forgetting
Support semi-supervised learning in spacy train
One obvious way to improve these pretraining methods is to do multi-task learning, instead of just transfer learning. This has been shown to work very well: https://arxiv.org/pdf/1809.08370.pdf . This patch makes it easy to do this sort of thing.
Add a new argument to spacy train, --raw-text. This takes a jsonl file with unlabelled data that can be used in arbitrary ways to do semi-supervised learning.
Add a new method to the Language class and to pipeline components, .rehearse(). This is like .update(), but doesn't expect GoldParse objects. It takes a batch of Doc objects, and performs an update on some semi-supervised objective.
Move the BERT-LMAO objective out from spacy/cli/pretrain.py into spacy/_ml.py, so we can create a new pipeline component, ClozeMultitask. This can be specified as a parser or NER multitask in the spacy train command. Example usage:
python -m spacy train en ./tmp ~/data/en-core-web/train/nw.json ~/data/en-core-web/dev/nw.json --pipeline parser --raw-textt ~/data/unlabelled/reddit-100k.jsonl --vectors en_vectors_web_lg --parser-multitasks cloze
Implement rehearsal methods for pipeline components
The new --raw-text argument and nlp.rehearse() method also gives us a good place to implement the the idea in the pseudo-rehearsal blog post in the parser. This works as follows:
Add a new nlp.resume_training() method. This allocates copies of pre-trained models in the pipeline, setting things up for the rehearsal updates. It also returns an optimizer object. This also greatly reduces confusion around the nlp.begin_training() method, which randomises the weights, making it not suitable for adding new labels or otherwise fine-tuning a pre-trained model.
Implement rehearsal updates on the Parser class, making it available for the dependency parser and NER. During rehearsal, the initial model is used to supervise the model being trained. The current model is asked to match the predictions of the initial model on some data. This minimises catastrophic forgetting, by keeping the model's predictions close to the original. See the blog post for details.
Implement rehearsal updates for tagger
Implement rehearsal updates for text categoriz
2018-12-10 18:25:33 +03:00
|
|
|
|
even if you're updating it with a smaller set of examples.
|
|
|
|
|
|
2020-07-22 14:42:59 +03:00
|
|
|
|
examples (Iterable[Example]): A batch of `Example` objects.
|
|
|
|
|
sgd (Optional[Optimizer]): An optimizer.
|
|
|
|
|
component_cfg (Dict[str, Dict]): Config parameters for specific pipeline
|
|
|
|
|
components, keyed by component name.
|
💫 Better support for semi-supervised learning (#3035)
The new spacy pretrain command implemented BERT/ULMFit/etc-like transfer learning, using our Language Modelling with Approximate Outputs version of BERT's cloze task. Pretraining is convenient, but in some ways it's a bit of a strange solution. All we're doing is initialising the weights. At the same time, we're putting a lot of work into our optimisation so that it's less sensitive to initial conditions, and more likely to find good optima. I discuss this a bit in the pseudo-rehearsal blog post: https://explosion.ai/blog/pseudo-rehearsal-catastrophic-forgetting
Support semi-supervised learning in spacy train
One obvious way to improve these pretraining methods is to do multi-task learning, instead of just transfer learning. This has been shown to work very well: https://arxiv.org/pdf/1809.08370.pdf . This patch makes it easy to do this sort of thing.
Add a new argument to spacy train, --raw-text. This takes a jsonl file with unlabelled data that can be used in arbitrary ways to do semi-supervised learning.
Add a new method to the Language class and to pipeline components, .rehearse(). This is like .update(), but doesn't expect GoldParse objects. It takes a batch of Doc objects, and performs an update on some semi-supervised objective.
Move the BERT-LMAO objective out from spacy/cli/pretrain.py into spacy/_ml.py, so we can create a new pipeline component, ClozeMultitask. This can be specified as a parser or NER multitask in the spacy train command. Example usage:
python -m spacy train en ./tmp ~/data/en-core-web/train/nw.json ~/data/en-core-web/dev/nw.json --pipeline parser --raw-textt ~/data/unlabelled/reddit-100k.jsonl --vectors en_vectors_web_lg --parser-multitasks cloze
Implement rehearsal methods for pipeline components
The new --raw-text argument and nlp.rehearse() method also gives us a good place to implement the the idea in the pseudo-rehearsal blog post in the parser. This works as follows:
Add a new nlp.resume_training() method. This allocates copies of pre-trained models in the pipeline, setting things up for the rehearsal updates. It also returns an optimizer object. This also greatly reduces confusion around the nlp.begin_training() method, which randomises the weights, making it not suitable for adding new labels or otherwise fine-tuning a pre-trained model.
Implement rehearsal updates on the Parser class, making it available for the dependency parser and NER. During rehearsal, the initial model is used to supervise the model being trained. The current model is asked to match the predictions of the initial model on some data. This minimises catastrophic forgetting, by keeping the model's predictions close to the original. See the blog post for details.
Implement rehearsal updates for tagger
Implement rehearsal updates for text categoriz
2018-12-10 18:25:33 +03:00
|
|
|
|
RETURNS (dict): Results from the update.
|
|
|
|
|
|
|
|
|
|
EXAMPLE:
|
|
|
|
|
>>> raw_text_batches = minibatch(raw_texts)
|
2020-07-06 14:02:36 +03:00
|
|
|
|
>>> for labelled_batch in minibatch(examples):
|
2019-11-11 19:35:27 +03:00
|
|
|
|
>>> nlp.update(labelled_batch)
|
2020-07-06 14:02:36 +03:00
|
|
|
|
>>> raw_batch = [Example.from_dict(nlp.make_doc(text), {}) for text in next(raw_text_batches)]
|
💫 Better support for semi-supervised learning (#3035)
The new spacy pretrain command implemented BERT/ULMFit/etc-like transfer learning, using our Language Modelling with Approximate Outputs version of BERT's cloze task. Pretraining is convenient, but in some ways it's a bit of a strange solution. All we're doing is initialising the weights. At the same time, we're putting a lot of work into our optimisation so that it's less sensitive to initial conditions, and more likely to find good optima. I discuss this a bit in the pseudo-rehearsal blog post: https://explosion.ai/blog/pseudo-rehearsal-catastrophic-forgetting
Support semi-supervised learning in spacy train
One obvious way to improve these pretraining methods is to do multi-task learning, instead of just transfer learning. This has been shown to work very well: https://arxiv.org/pdf/1809.08370.pdf . This patch makes it easy to do this sort of thing.
Add a new argument to spacy train, --raw-text. This takes a jsonl file with unlabelled data that can be used in arbitrary ways to do semi-supervised learning.
Add a new method to the Language class and to pipeline components, .rehearse(). This is like .update(), but doesn't expect GoldParse objects. It takes a batch of Doc objects, and performs an update on some semi-supervised objective.
Move the BERT-LMAO objective out from spacy/cli/pretrain.py into spacy/_ml.py, so we can create a new pipeline component, ClozeMultitask. This can be specified as a parser or NER multitask in the spacy train command. Example usage:
python -m spacy train en ./tmp ~/data/en-core-web/train/nw.json ~/data/en-core-web/dev/nw.json --pipeline parser --raw-textt ~/data/unlabelled/reddit-100k.jsonl --vectors en_vectors_web_lg --parser-multitasks cloze
Implement rehearsal methods for pipeline components
The new --raw-text argument and nlp.rehearse() method also gives us a good place to implement the the idea in the pseudo-rehearsal blog post in the parser. This works as follows:
Add a new nlp.resume_training() method. This allocates copies of pre-trained models in the pipeline, setting things up for the rehearsal updates. It also returns an optimizer object. This also greatly reduces confusion around the nlp.begin_training() method, which randomises the weights, making it not suitable for adding new labels or otherwise fine-tuning a pre-trained model.
Implement rehearsal updates on the Parser class, making it available for the dependency parser and NER. During rehearsal, the initial model is used to supervise the model being trained. The current model is asked to match the predictions of the initial model on some data. This minimises catastrophic forgetting, by keeping the model's predictions close to the original. See the blog post for details.
Implement rehearsal updates for tagger
Implement rehearsal updates for text categoriz
2018-12-10 18:25:33 +03:00
|
|
|
|
>>> nlp.rehearse(raw_batch)
|
2020-07-29 00:12:42 +03:00
|
|
|
|
|
|
|
|
|
DOCS: https://spacy.io/api/language#rehearse
|
💫 Better support for semi-supervised learning (#3035)
The new spacy pretrain command implemented BERT/ULMFit/etc-like transfer learning, using our Language Modelling with Approximate Outputs version of BERT's cloze task. Pretraining is convenient, but in some ways it's a bit of a strange solution. All we're doing is initialising the weights. At the same time, we're putting a lot of work into our optimisation so that it's less sensitive to initial conditions, and more likely to find good optima. I discuss this a bit in the pseudo-rehearsal blog post: https://explosion.ai/blog/pseudo-rehearsal-catastrophic-forgetting
Support semi-supervised learning in spacy train
One obvious way to improve these pretraining methods is to do multi-task learning, instead of just transfer learning. This has been shown to work very well: https://arxiv.org/pdf/1809.08370.pdf . This patch makes it easy to do this sort of thing.
Add a new argument to spacy train, --raw-text. This takes a jsonl file with unlabelled data that can be used in arbitrary ways to do semi-supervised learning.
Add a new method to the Language class and to pipeline components, .rehearse(). This is like .update(), but doesn't expect GoldParse objects. It takes a batch of Doc objects, and performs an update on some semi-supervised objective.
Move the BERT-LMAO objective out from spacy/cli/pretrain.py into spacy/_ml.py, so we can create a new pipeline component, ClozeMultitask. This can be specified as a parser or NER multitask in the spacy train command. Example usage:
python -m spacy train en ./tmp ~/data/en-core-web/train/nw.json ~/data/en-core-web/dev/nw.json --pipeline parser --raw-textt ~/data/unlabelled/reddit-100k.jsonl --vectors en_vectors_web_lg --parser-multitasks cloze
Implement rehearsal methods for pipeline components
The new --raw-text argument and nlp.rehearse() method also gives us a good place to implement the the idea in the pseudo-rehearsal blog post in the parser. This works as follows:
Add a new nlp.resume_training() method. This allocates copies of pre-trained models in the pipeline, setting things up for the rehearsal updates. It also returns an optimizer object. This also greatly reduces confusion around the nlp.begin_training() method, which randomises the weights, making it not suitable for adding new labels or otherwise fine-tuning a pre-trained model.
Implement rehearsal updates on the Parser class, making it available for the dependency parser and NER. During rehearsal, the initial model is used to supervise the model being trained. The current model is asked to match the predictions of the initial model on some data. This minimises catastrophic forgetting, by keeping the model's predictions close to the original. See the blog post for details.
Implement rehearsal updates for tagger
Implement rehearsal updates for text categoriz
2018-12-10 18:25:33 +03:00
|
|
|
|
"""
|
2019-11-11 19:35:27 +03:00
|
|
|
|
if len(examples) == 0:
|
💫 Better support for semi-supervised learning (#3035)
The new spacy pretrain command implemented BERT/ULMFit/etc-like transfer learning, using our Language Modelling with Approximate Outputs version of BERT's cloze task. Pretraining is convenient, but in some ways it's a bit of a strange solution. All we're doing is initialising the weights. At the same time, we're putting a lot of work into our optimisation so that it's less sensitive to initial conditions, and more likely to find good optima. I discuss this a bit in the pseudo-rehearsal blog post: https://explosion.ai/blog/pseudo-rehearsal-catastrophic-forgetting
Support semi-supervised learning in spacy train
One obvious way to improve these pretraining methods is to do multi-task learning, instead of just transfer learning. This has been shown to work very well: https://arxiv.org/pdf/1809.08370.pdf . This patch makes it easy to do this sort of thing.
Add a new argument to spacy train, --raw-text. This takes a jsonl file with unlabelled data that can be used in arbitrary ways to do semi-supervised learning.
Add a new method to the Language class and to pipeline components, .rehearse(). This is like .update(), but doesn't expect GoldParse objects. It takes a batch of Doc objects, and performs an update on some semi-supervised objective.
Move the BERT-LMAO objective out from spacy/cli/pretrain.py into spacy/_ml.py, so we can create a new pipeline component, ClozeMultitask. This can be specified as a parser or NER multitask in the spacy train command. Example usage:
python -m spacy train en ./tmp ~/data/en-core-web/train/nw.json ~/data/en-core-web/dev/nw.json --pipeline parser --raw-textt ~/data/unlabelled/reddit-100k.jsonl --vectors en_vectors_web_lg --parser-multitasks cloze
Implement rehearsal methods for pipeline components
The new --raw-text argument and nlp.rehearse() method also gives us a good place to implement the the idea in the pseudo-rehearsal blog post in the parser. This works as follows:
Add a new nlp.resume_training() method. This allocates copies of pre-trained models in the pipeline, setting things up for the rehearsal updates. It also returns an optimizer object. This also greatly reduces confusion around the nlp.begin_training() method, which randomises the weights, making it not suitable for adding new labels or otherwise fine-tuning a pre-trained model.
Implement rehearsal updates on the Parser class, making it available for the dependency parser and NER. During rehearsal, the initial model is used to supervise the model being trained. The current model is asked to match the predictions of the initial model on some data. This minimises catastrophic forgetting, by keeping the model's predictions close to the original. See the blog post for details.
Implement rehearsal updates for tagger
Implement rehearsal updates for text categoriz
2018-12-10 18:25:33 +03:00
|
|
|
|
return
|
2020-07-22 14:42:59 +03:00
|
|
|
|
if not isinstance(examples, IterableInstance):
|
2020-07-12 15:03:23 +03:00
|
|
|
|
raise TypeError(
|
|
|
|
|
Errors.E978.format(
|
|
|
|
|
name="language", method="rehearse", types=type(examples)
|
|
|
|
|
)
|
|
|
|
|
)
|
2020-07-06 14:02:36 +03:00
|
|
|
|
wrong_types = set([type(eg) for eg in examples if not isinstance(eg, Example)])
|
|
|
|
|
if wrong_types:
|
2020-07-12 15:03:23 +03:00
|
|
|
|
raise TypeError(
|
|
|
|
|
Errors.E978.format(
|
|
|
|
|
name="language", method="rehearse", types=wrong_types
|
|
|
|
|
)
|
|
|
|
|
)
|
💫 Better support for semi-supervised learning (#3035)
The new spacy pretrain command implemented BERT/ULMFit/etc-like transfer learning, using our Language Modelling with Approximate Outputs version of BERT's cloze task. Pretraining is convenient, but in some ways it's a bit of a strange solution. All we're doing is initialising the weights. At the same time, we're putting a lot of work into our optimisation so that it's less sensitive to initial conditions, and more likely to find good optima. I discuss this a bit in the pseudo-rehearsal blog post: https://explosion.ai/blog/pseudo-rehearsal-catastrophic-forgetting
Support semi-supervised learning in spacy train
One obvious way to improve these pretraining methods is to do multi-task learning, instead of just transfer learning. This has been shown to work very well: https://arxiv.org/pdf/1809.08370.pdf . This patch makes it easy to do this sort of thing.
Add a new argument to spacy train, --raw-text. This takes a jsonl file with unlabelled data that can be used in arbitrary ways to do semi-supervised learning.
Add a new method to the Language class and to pipeline components, .rehearse(). This is like .update(), but doesn't expect GoldParse objects. It takes a batch of Doc objects, and performs an update on some semi-supervised objective.
Move the BERT-LMAO objective out from spacy/cli/pretrain.py into spacy/_ml.py, so we can create a new pipeline component, ClozeMultitask. This can be specified as a parser or NER multitask in the spacy train command. Example usage:
python -m spacy train en ./tmp ~/data/en-core-web/train/nw.json ~/data/en-core-web/dev/nw.json --pipeline parser --raw-textt ~/data/unlabelled/reddit-100k.jsonl --vectors en_vectors_web_lg --parser-multitasks cloze
Implement rehearsal methods for pipeline components
The new --raw-text argument and nlp.rehearse() method also gives us a good place to implement the the idea in the pseudo-rehearsal blog post in the parser. This works as follows:
Add a new nlp.resume_training() method. This allocates copies of pre-trained models in the pipeline, setting things up for the rehearsal updates. It also returns an optimizer object. This also greatly reduces confusion around the nlp.begin_training() method, which randomises the weights, making it not suitable for adding new labels or otherwise fine-tuning a pre-trained model.
Implement rehearsal updates on the Parser class, making it available for the dependency parser and NER. During rehearsal, the initial model is used to supervise the model being trained. The current model is asked to match the predictions of the initial model on some data. This minimises catastrophic forgetting, by keeping the model's predictions close to the original. See the blog post for details.
Implement rehearsal updates for tagger
Implement rehearsal updates for text categoriz
2018-12-10 18:25:33 +03:00
|
|
|
|
if sgd is None:
|
|
|
|
|
if self._optimizer is None:
|
2020-01-29 19:06:46 +03:00
|
|
|
|
self._optimizer = create_default_optimizer()
|
💫 Better support for semi-supervised learning (#3035)
The new spacy pretrain command implemented BERT/ULMFit/etc-like transfer learning, using our Language Modelling with Approximate Outputs version of BERT's cloze task. Pretraining is convenient, but in some ways it's a bit of a strange solution. All we're doing is initialising the weights. At the same time, we're putting a lot of work into our optimisation so that it's less sensitive to initial conditions, and more likely to find good optima. I discuss this a bit in the pseudo-rehearsal blog post: https://explosion.ai/blog/pseudo-rehearsal-catastrophic-forgetting
Support semi-supervised learning in spacy train
One obvious way to improve these pretraining methods is to do multi-task learning, instead of just transfer learning. This has been shown to work very well: https://arxiv.org/pdf/1809.08370.pdf . This patch makes it easy to do this sort of thing.
Add a new argument to spacy train, --raw-text. This takes a jsonl file with unlabelled data that can be used in arbitrary ways to do semi-supervised learning.
Add a new method to the Language class and to pipeline components, .rehearse(). This is like .update(), but doesn't expect GoldParse objects. It takes a batch of Doc objects, and performs an update on some semi-supervised objective.
Move the BERT-LMAO objective out from spacy/cli/pretrain.py into spacy/_ml.py, so we can create a new pipeline component, ClozeMultitask. This can be specified as a parser or NER multitask in the spacy train command. Example usage:
python -m spacy train en ./tmp ~/data/en-core-web/train/nw.json ~/data/en-core-web/dev/nw.json --pipeline parser --raw-textt ~/data/unlabelled/reddit-100k.jsonl --vectors en_vectors_web_lg --parser-multitasks cloze
Implement rehearsal methods for pipeline components
The new --raw-text argument and nlp.rehearse() method also gives us a good place to implement the the idea in the pseudo-rehearsal blog post in the parser. This works as follows:
Add a new nlp.resume_training() method. This allocates copies of pre-trained models in the pipeline, setting things up for the rehearsal updates. It also returns an optimizer object. This also greatly reduces confusion around the nlp.begin_training() method, which randomises the weights, making it not suitable for adding new labels or otherwise fine-tuning a pre-trained model.
Implement rehearsal updates on the Parser class, making it available for the dependency parser and NER. During rehearsal, the initial model is used to supervise the model being trained. The current model is asked to match the predictions of the initial model on some data. This minimises catastrophic forgetting, by keeping the model's predictions close to the original. See the blog post for details.
Implement rehearsal updates for tagger
Implement rehearsal updates for text categoriz
2018-12-10 18:25:33 +03:00
|
|
|
|
sgd = self._optimizer
|
|
|
|
|
pipes = list(self.pipeline)
|
|
|
|
|
random.shuffle(pipes)
|
2020-07-22 14:42:59 +03:00
|
|
|
|
if component_cfg is None:
|
|
|
|
|
component_cfg = {}
|
💫 Better support for semi-supervised learning (#3035)
The new spacy pretrain command implemented BERT/ULMFit/etc-like transfer learning, using our Language Modelling with Approximate Outputs version of BERT's cloze task. Pretraining is convenient, but in some ways it's a bit of a strange solution. All we're doing is initialising the weights. At the same time, we're putting a lot of work into our optimisation so that it's less sensitive to initial conditions, and more likely to find good optima. I discuss this a bit in the pseudo-rehearsal blog post: https://explosion.ai/blog/pseudo-rehearsal-catastrophic-forgetting
Support semi-supervised learning in spacy train
One obvious way to improve these pretraining methods is to do multi-task learning, instead of just transfer learning. This has been shown to work very well: https://arxiv.org/pdf/1809.08370.pdf . This patch makes it easy to do this sort of thing.
Add a new argument to spacy train, --raw-text. This takes a jsonl file with unlabelled data that can be used in arbitrary ways to do semi-supervised learning.
Add a new method to the Language class and to pipeline components, .rehearse(). This is like .update(), but doesn't expect GoldParse objects. It takes a batch of Doc objects, and performs an update on some semi-supervised objective.
Move the BERT-LMAO objective out from spacy/cli/pretrain.py into spacy/_ml.py, so we can create a new pipeline component, ClozeMultitask. This can be specified as a parser or NER multitask in the spacy train command. Example usage:
python -m spacy train en ./tmp ~/data/en-core-web/train/nw.json ~/data/en-core-web/dev/nw.json --pipeline parser --raw-textt ~/data/unlabelled/reddit-100k.jsonl --vectors en_vectors_web_lg --parser-multitasks cloze
Implement rehearsal methods for pipeline components
The new --raw-text argument and nlp.rehearse() method also gives us a good place to implement the the idea in the pseudo-rehearsal blog post in the parser. This works as follows:
Add a new nlp.resume_training() method. This allocates copies of pre-trained models in the pipeline, setting things up for the rehearsal updates. It also returns an optimizer object. This also greatly reduces confusion around the nlp.begin_training() method, which randomises the weights, making it not suitable for adding new labels or otherwise fine-tuning a pre-trained model.
Implement rehearsal updates on the Parser class, making it available for the dependency parser and NER. During rehearsal, the initial model is used to supervise the model being trained. The current model is asked to match the predictions of the initial model on some data. This minimises catastrophic forgetting, by keeping the model's predictions close to the original. See the blog post for details.
Implement rehearsal updates for tagger
Implement rehearsal updates for text categoriz
2018-12-10 18:25:33 +03:00
|
|
|
|
grads = {}
|
|
|
|
|
|
|
|
|
|
def get_grads(W, dW, key=None):
|
|
|
|
|
grads[key] = (W, dW)
|
|
|
|
|
|
2020-01-29 19:06:46 +03:00
|
|
|
|
get_grads.learn_rate = sgd.learn_rate
|
💫 Better support for semi-supervised learning (#3035)
The new spacy pretrain command implemented BERT/ULMFit/etc-like transfer learning, using our Language Modelling with Approximate Outputs version of BERT's cloze task. Pretraining is convenient, but in some ways it's a bit of a strange solution. All we're doing is initialising the weights. At the same time, we're putting a lot of work into our optimisation so that it's less sensitive to initial conditions, and more likely to find good optima. I discuss this a bit in the pseudo-rehearsal blog post: https://explosion.ai/blog/pseudo-rehearsal-catastrophic-forgetting
Support semi-supervised learning in spacy train
One obvious way to improve these pretraining methods is to do multi-task learning, instead of just transfer learning. This has been shown to work very well: https://arxiv.org/pdf/1809.08370.pdf . This patch makes it easy to do this sort of thing.
Add a new argument to spacy train, --raw-text. This takes a jsonl file with unlabelled data that can be used in arbitrary ways to do semi-supervised learning.
Add a new method to the Language class and to pipeline components, .rehearse(). This is like .update(), but doesn't expect GoldParse objects. It takes a batch of Doc objects, and performs an update on some semi-supervised objective.
Move the BERT-LMAO objective out from spacy/cli/pretrain.py into spacy/_ml.py, so we can create a new pipeline component, ClozeMultitask. This can be specified as a parser or NER multitask in the spacy train command. Example usage:
python -m spacy train en ./tmp ~/data/en-core-web/train/nw.json ~/data/en-core-web/dev/nw.json --pipeline parser --raw-textt ~/data/unlabelled/reddit-100k.jsonl --vectors en_vectors_web_lg --parser-multitasks cloze
Implement rehearsal methods for pipeline components
The new --raw-text argument and nlp.rehearse() method also gives us a good place to implement the the idea in the pseudo-rehearsal blog post in the parser. This works as follows:
Add a new nlp.resume_training() method. This allocates copies of pre-trained models in the pipeline, setting things up for the rehearsal updates. It also returns an optimizer object. This also greatly reduces confusion around the nlp.begin_training() method, which randomises the weights, making it not suitable for adding new labels or otherwise fine-tuning a pre-trained model.
Implement rehearsal updates on the Parser class, making it available for the dependency parser and NER. During rehearsal, the initial model is used to supervise the model being trained. The current model is asked to match the predictions of the initial model on some data. This minimises catastrophic forgetting, by keeping the model's predictions close to the original. See the blog post for details.
Implement rehearsal updates for tagger
Implement rehearsal updates for text categoriz
2018-12-10 18:25:33 +03:00
|
|
|
|
get_grads.b1 = sgd.b1
|
|
|
|
|
get_grads.b2 = sgd.b2
|
|
|
|
|
for name, proc in pipes:
|
|
|
|
|
if not hasattr(proc, "rehearse"):
|
|
|
|
|
continue
|
|
|
|
|
grads = {}
|
2020-02-03 15:02:12 +03:00
|
|
|
|
proc.rehearse(
|
2020-07-22 14:42:59 +03:00
|
|
|
|
examples, sgd=get_grads, losses=losses, **component_cfg.get(name, {})
|
2020-02-03 15:02:12 +03:00
|
|
|
|
)
|
2020-01-29 19:06:46 +03:00
|
|
|
|
for key, (W, dW) in grads.items():
|
|
|
|
|
sgd(W, dW, key=key)
|
💫 Better support for semi-supervised learning (#3035)
The new spacy pretrain command implemented BERT/ULMFit/etc-like transfer learning, using our Language Modelling with Approximate Outputs version of BERT's cloze task. Pretraining is convenient, but in some ways it's a bit of a strange solution. All we're doing is initialising the weights. At the same time, we're putting a lot of work into our optimisation so that it's less sensitive to initial conditions, and more likely to find good optima. I discuss this a bit in the pseudo-rehearsal blog post: https://explosion.ai/blog/pseudo-rehearsal-catastrophic-forgetting
Support semi-supervised learning in spacy train
One obvious way to improve these pretraining methods is to do multi-task learning, instead of just transfer learning. This has been shown to work very well: https://arxiv.org/pdf/1809.08370.pdf . This patch makes it easy to do this sort of thing.
Add a new argument to spacy train, --raw-text. This takes a jsonl file with unlabelled data that can be used in arbitrary ways to do semi-supervised learning.
Add a new method to the Language class and to pipeline components, .rehearse(). This is like .update(), but doesn't expect GoldParse objects. It takes a batch of Doc objects, and performs an update on some semi-supervised objective.
Move the BERT-LMAO objective out from spacy/cli/pretrain.py into spacy/_ml.py, so we can create a new pipeline component, ClozeMultitask. This can be specified as a parser or NER multitask in the spacy train command. Example usage:
python -m spacy train en ./tmp ~/data/en-core-web/train/nw.json ~/data/en-core-web/dev/nw.json --pipeline parser --raw-textt ~/data/unlabelled/reddit-100k.jsonl --vectors en_vectors_web_lg --parser-multitasks cloze
Implement rehearsal methods for pipeline components
The new --raw-text argument and nlp.rehearse() method also gives us a good place to implement the the idea in the pseudo-rehearsal blog post in the parser. This works as follows:
Add a new nlp.resume_training() method. This allocates copies of pre-trained models in the pipeline, setting things up for the rehearsal updates. It also returns an optimizer object. This also greatly reduces confusion around the nlp.begin_training() method, which randomises the weights, making it not suitable for adding new labels or otherwise fine-tuning a pre-trained model.
Implement rehearsal updates on the Parser class, making it available for the dependency parser and NER. During rehearsal, the initial model is used to supervise the model being trained. The current model is asked to match the predictions of the initial model on some data. This minimises catastrophic forgetting, by keeping the model's predictions close to the original. See the blog post for details.
Implement rehearsal updates for tagger
Implement rehearsal updates for text categoriz
2018-12-10 18:25:33 +03:00
|
|
|
|
return losses
|
|
|
|
|
|
2020-07-22 14:42:59 +03:00
|
|
|
|
def begin_training(
|
|
|
|
|
self,
|
2020-07-29 00:12:42 +03:00
|
|
|
|
get_examples: Optional[Callable[[], Iterable[Example]]] = None,
|
|
|
|
|
*,
|
2020-07-22 14:42:59 +03:00
|
|
|
|
sgd: Optional[Optimizer] = None,
|
|
|
|
|
device: int = -1,
|
|
|
|
|
) -> Optimizer:
|
2020-07-29 00:12:42 +03:00
|
|
|
|
"""Initialize the pipe for training, using data examples if available.
|
2017-05-19 00:57:38 +03:00
|
|
|
|
|
2020-07-29 00:12:42 +03:00
|
|
|
|
get_examples (Callable[[], Iterable[Example]]): Optional function that
|
|
|
|
|
returns gold-standard Example objects.
|
|
|
|
|
sgd (thinc.api.Optimizer): Optional optimizer. Will be created with
|
|
|
|
|
create_optimizer if it doesn't exist.
|
|
|
|
|
RETURNS (thinc.api.Optimizer): The optimizer.
|
2019-03-15 18:23:17 +03:00
|
|
|
|
|
|
|
|
|
DOCS: https://spacy.io/api/language#begin_training
|
2017-05-19 00:57:38 +03:00
|
|
|
|
"""
|
2019-11-11 19:35:27 +03:00
|
|
|
|
# TODO: throw warning when get_gold_tuples is provided instead of get_examples
|
|
|
|
|
if get_examples is None:
|
|
|
|
|
get_examples = lambda: []
|
2020-07-22 14:42:59 +03:00
|
|
|
|
else: # Populate vocab
|
2019-11-11 19:35:27 +03:00
|
|
|
|
for example in get_examples():
|
2020-06-26 20:34:12 +03:00
|
|
|
|
for word in [t.text for t in example.reference]:
|
2019-11-25 18:03:28 +03:00
|
|
|
|
_ = self.vocab[word] # noqa: F841
|
2020-07-22 14:42:59 +03:00
|
|
|
|
if device >= 0: # TODO: do we need this here?
|
|
|
|
|
require_gpu(device)
|
2017-09-19 02:04:16 +03:00
|
|
|
|
if self.vocab.vectors.data.shape[1] >= 1:
|
2020-01-29 19:06:46 +03:00
|
|
|
|
ops = get_current_ops()
|
|
|
|
|
self.vocab.vectors.data = ops.asarray(self.vocab.vectors.data)
|
2017-09-23 04:11:52 +03:00
|
|
|
|
link_vectors_to_models(self.vocab)
|
2017-11-06 16:26:00 +03:00
|
|
|
|
if sgd is None:
|
2020-01-29 19:06:46 +03:00
|
|
|
|
sgd = create_default_optimizer()
|
2017-11-06 16:26:00 +03:00
|
|
|
|
self._optimizer = sgd
|
2017-10-07 01:25:54 +03:00
|
|
|
|
for name, proc in self.pipeline:
|
💫 Tidy up and auto-format .py files (#2983)
<!--- Provide a general summary of your changes in the title. -->
## Description
- [x] Use [`black`](https://github.com/ambv/black) to auto-format all `.py` files.
- [x] Update flake8 config to exclude very large files (lemmatization tables etc.)
- [x] Update code to be compatible with flake8 rules
- [x] Fix various small bugs, inconsistencies and messy stuff in the language data
- [x] Update docs to explain new code style (`black`, `flake8`, when to use `# fmt: off` and `# fmt: on` and what `# noqa` means)
Once #2932 is merged, which auto-formats and tidies up the CLI, we'll be able to run `flake8 spacy` actually get meaningful results.
At the moment, the code style and linting isn't applied automatically, but I'm hoping that the new [GitHub Actions](https://github.com/features/actions) will let us auto-format pull requests and post comments with relevant linting information.
### Types of change
enhancement, code style
## Checklist
<!--- Before you submit the PR, go over this checklist and make sure you can
tick off all the boxes. [] -> [x] -->
- [x] I have submitted the spaCy Contributor Agreement.
- [x] I ran the tests, and all new and existing tests passed.
- [x] My changes don't require a change to the documentation, or if they do, I've added all required information.
2018-11-30 19:03:03 +03:00
|
|
|
|
if hasattr(proc, "begin_training"):
|
|
|
|
|
proc.begin_training(
|
2020-07-22 14:42:59 +03:00
|
|
|
|
get_examples, pipeline=self.pipeline, sgd=self._optimizer
|
💫 Tidy up and auto-format .py files (#2983)
<!--- Provide a general summary of your changes in the title. -->
## Description
- [x] Use [`black`](https://github.com/ambv/black) to auto-format all `.py` files.
- [x] Update flake8 config to exclude very large files (lemmatization tables etc.)
- [x] Update code to be compatible with flake8 rules
- [x] Fix various small bugs, inconsistencies and messy stuff in the language data
- [x] Update docs to explain new code style (`black`, `flake8`, when to use `# fmt: off` and `# fmt: on` and what `# noqa` means)
Once #2932 is merged, which auto-formats and tidies up the CLI, we'll be able to run `flake8 spacy` actually get meaningful results.
At the moment, the code style and linting isn't applied automatically, but I'm hoping that the new [GitHub Actions](https://github.com/features/actions) will let us auto-format pull requests and post comments with relevant linting information.
### Types of change
enhancement, code style
## Checklist
<!--- Before you submit the PR, go over this checklist and make sure you can
tick off all the boxes. [] -> [x] -->
- [x] I have submitted the spaCy Contributor Agreement.
- [x] I ran the tests, and all new and existing tests passed.
- [x] My changes don't require a change to the documentation, or if they do, I've added all required information.
2018-11-30 19:03:03 +03:00
|
|
|
|
)
|
2020-01-29 19:06:46 +03:00
|
|
|
|
self._link_components()
|
2017-08-20 15:42:07 +03:00
|
|
|
|
return self._optimizer
|
2017-05-21 17:07:06 +03:00
|
|
|
|
|
2020-07-22 14:42:59 +03:00
|
|
|
|
def resume_training(
|
2020-07-29 00:12:42 +03:00
|
|
|
|
self, *, sgd: Optional[Optimizer] = None, device: int = -1
|
2020-07-22 14:42:59 +03:00
|
|
|
|
) -> Optimizer:
|
2019-10-02 11:37:39 +03:00
|
|
|
|
"""Continue training a pretrained model.
|
2018-12-18 15:48:10 +03:00
|
|
|
|
|
💫 Better support for semi-supervised learning (#3035)
The new spacy pretrain command implemented BERT/ULMFit/etc-like transfer learning, using our Language Modelling with Approximate Outputs version of BERT's cloze task. Pretraining is convenient, but in some ways it's a bit of a strange solution. All we're doing is initialising the weights. At the same time, we're putting a lot of work into our optimisation so that it's less sensitive to initial conditions, and more likely to find good optima. I discuss this a bit in the pseudo-rehearsal blog post: https://explosion.ai/blog/pseudo-rehearsal-catastrophic-forgetting
Support semi-supervised learning in spacy train
One obvious way to improve these pretraining methods is to do multi-task learning, instead of just transfer learning. This has been shown to work very well: https://arxiv.org/pdf/1809.08370.pdf . This patch makes it easy to do this sort of thing.
Add a new argument to spacy train, --raw-text. This takes a jsonl file with unlabelled data that can be used in arbitrary ways to do semi-supervised learning.
Add a new method to the Language class and to pipeline components, .rehearse(). This is like .update(), but doesn't expect GoldParse objects. It takes a batch of Doc objects, and performs an update on some semi-supervised objective.
Move the BERT-LMAO objective out from spacy/cli/pretrain.py into spacy/_ml.py, so we can create a new pipeline component, ClozeMultitask. This can be specified as a parser or NER multitask in the spacy train command. Example usage:
python -m spacy train en ./tmp ~/data/en-core-web/train/nw.json ~/data/en-core-web/dev/nw.json --pipeline parser --raw-textt ~/data/unlabelled/reddit-100k.jsonl --vectors en_vectors_web_lg --parser-multitasks cloze
Implement rehearsal methods for pipeline components
The new --raw-text argument and nlp.rehearse() method also gives us a good place to implement the the idea in the pseudo-rehearsal blog post in the parser. This works as follows:
Add a new nlp.resume_training() method. This allocates copies of pre-trained models in the pipeline, setting things up for the rehearsal updates. It also returns an optimizer object. This also greatly reduces confusion around the nlp.begin_training() method, which randomises the weights, making it not suitable for adding new labels or otherwise fine-tuning a pre-trained model.
Implement rehearsal updates on the Parser class, making it available for the dependency parser and NER. During rehearsal, the initial model is used to supervise the model being trained. The current model is asked to match the predictions of the initial model on some data. This minimises catastrophic forgetting, by keeping the model's predictions close to the original. See the blog post for details.
Implement rehearsal updates for tagger
Implement rehearsal updates for text categoriz
2018-12-10 18:25:33 +03:00
|
|
|
|
Create and return an optimizer, and initialize "rehearsal" for any pipeline
|
|
|
|
|
component that has a .rehearse() method. Rehearsal is used to prevent
|
2020-07-29 00:12:42 +03:00
|
|
|
|
models from "forgetting" their initialized "knowledge". To perform
|
💫 Better support for semi-supervised learning (#3035)
The new spacy pretrain command implemented BERT/ULMFit/etc-like transfer learning, using our Language Modelling with Approximate Outputs version of BERT's cloze task. Pretraining is convenient, but in some ways it's a bit of a strange solution. All we're doing is initialising the weights. At the same time, we're putting a lot of work into our optimisation so that it's less sensitive to initial conditions, and more likely to find good optima. I discuss this a bit in the pseudo-rehearsal blog post: https://explosion.ai/blog/pseudo-rehearsal-catastrophic-forgetting
Support semi-supervised learning in spacy train
One obvious way to improve these pretraining methods is to do multi-task learning, instead of just transfer learning. This has been shown to work very well: https://arxiv.org/pdf/1809.08370.pdf . This patch makes it easy to do this sort of thing.
Add a new argument to spacy train, --raw-text. This takes a jsonl file with unlabelled data that can be used in arbitrary ways to do semi-supervised learning.
Add a new method to the Language class and to pipeline components, .rehearse(). This is like .update(), but doesn't expect GoldParse objects. It takes a batch of Doc objects, and performs an update on some semi-supervised objective.
Move the BERT-LMAO objective out from spacy/cli/pretrain.py into spacy/_ml.py, so we can create a new pipeline component, ClozeMultitask. This can be specified as a parser or NER multitask in the spacy train command. Example usage:
python -m spacy train en ./tmp ~/data/en-core-web/train/nw.json ~/data/en-core-web/dev/nw.json --pipeline parser --raw-textt ~/data/unlabelled/reddit-100k.jsonl --vectors en_vectors_web_lg --parser-multitasks cloze
Implement rehearsal methods for pipeline components
The new --raw-text argument and nlp.rehearse() method also gives us a good place to implement the the idea in the pseudo-rehearsal blog post in the parser. This works as follows:
Add a new nlp.resume_training() method. This allocates copies of pre-trained models in the pipeline, setting things up for the rehearsal updates. It also returns an optimizer object. This also greatly reduces confusion around the nlp.begin_training() method, which randomises the weights, making it not suitable for adding new labels or otherwise fine-tuning a pre-trained model.
Implement rehearsal updates on the Parser class, making it available for the dependency parser and NER. During rehearsal, the initial model is used to supervise the model being trained. The current model is asked to match the predictions of the initial model on some data. This minimises catastrophic forgetting, by keeping the model's predictions close to the original. See the blog post for details.
Implement rehearsal updates for tagger
Implement rehearsal updates for text categoriz
2018-12-10 18:25:33 +03:00
|
|
|
|
rehearsal, collect samples of text you want the models to retain performance
|
2020-07-06 14:02:36 +03:00
|
|
|
|
on, and call nlp.rehearse() with a batch of Example objects.
|
2020-07-22 14:42:59 +03:00
|
|
|
|
|
|
|
|
|
sgd (Optional[Optimizer]): An optimizer.
|
|
|
|
|
RETURNS (Optimizer): The optimizer.
|
2020-07-29 00:12:42 +03:00
|
|
|
|
|
|
|
|
|
DOCS: https://spacy.io/api/language#resume_training
|
💫 Better support for semi-supervised learning (#3035)
The new spacy pretrain command implemented BERT/ULMFit/etc-like transfer learning, using our Language Modelling with Approximate Outputs version of BERT's cloze task. Pretraining is convenient, but in some ways it's a bit of a strange solution. All we're doing is initialising the weights. At the same time, we're putting a lot of work into our optimisation so that it's less sensitive to initial conditions, and more likely to find good optima. I discuss this a bit in the pseudo-rehearsal blog post: https://explosion.ai/blog/pseudo-rehearsal-catastrophic-forgetting
Support semi-supervised learning in spacy train
One obvious way to improve these pretraining methods is to do multi-task learning, instead of just transfer learning. This has been shown to work very well: https://arxiv.org/pdf/1809.08370.pdf . This patch makes it easy to do this sort of thing.
Add a new argument to spacy train, --raw-text. This takes a jsonl file with unlabelled data that can be used in arbitrary ways to do semi-supervised learning.
Add a new method to the Language class and to pipeline components, .rehearse(). This is like .update(), but doesn't expect GoldParse objects. It takes a batch of Doc objects, and performs an update on some semi-supervised objective.
Move the BERT-LMAO objective out from spacy/cli/pretrain.py into spacy/_ml.py, so we can create a new pipeline component, ClozeMultitask. This can be specified as a parser or NER multitask in the spacy train command. Example usage:
python -m spacy train en ./tmp ~/data/en-core-web/train/nw.json ~/data/en-core-web/dev/nw.json --pipeline parser --raw-textt ~/data/unlabelled/reddit-100k.jsonl --vectors en_vectors_web_lg --parser-multitasks cloze
Implement rehearsal methods for pipeline components
The new --raw-text argument and nlp.rehearse() method also gives us a good place to implement the the idea in the pseudo-rehearsal blog post in the parser. This works as follows:
Add a new nlp.resume_training() method. This allocates copies of pre-trained models in the pipeline, setting things up for the rehearsal updates. It also returns an optimizer object. This also greatly reduces confusion around the nlp.begin_training() method, which randomises the weights, making it not suitable for adding new labels or otherwise fine-tuning a pre-trained model.
Implement rehearsal updates on the Parser class, making it available for the dependency parser and NER. During rehearsal, the initial model is used to supervise the model being trained. The current model is asked to match the predictions of the initial model on some data. This minimises catastrophic forgetting, by keeping the model's predictions close to the original. See the blog post for details.
Implement rehearsal updates for tagger
Implement rehearsal updates for text categoriz
2018-12-10 18:25:33 +03:00
|
|
|
|
"""
|
2020-07-22 14:42:59 +03:00
|
|
|
|
if device >= 0: # TODO: do we need this here?
|
|
|
|
|
require_gpu(device)
|
2020-01-29 19:06:46 +03:00
|
|
|
|
ops = get_current_ops()
|
💫 Better support for semi-supervised learning (#3035)
The new spacy pretrain command implemented BERT/ULMFit/etc-like transfer learning, using our Language Modelling with Approximate Outputs version of BERT's cloze task. Pretraining is convenient, but in some ways it's a bit of a strange solution. All we're doing is initialising the weights. At the same time, we're putting a lot of work into our optimisation so that it's less sensitive to initial conditions, and more likely to find good optima. I discuss this a bit in the pseudo-rehearsal blog post: https://explosion.ai/blog/pseudo-rehearsal-catastrophic-forgetting
Support semi-supervised learning in spacy train
One obvious way to improve these pretraining methods is to do multi-task learning, instead of just transfer learning. This has been shown to work very well: https://arxiv.org/pdf/1809.08370.pdf . This patch makes it easy to do this sort of thing.
Add a new argument to spacy train, --raw-text. This takes a jsonl file with unlabelled data that can be used in arbitrary ways to do semi-supervised learning.
Add a new method to the Language class and to pipeline components, .rehearse(). This is like .update(), but doesn't expect GoldParse objects. It takes a batch of Doc objects, and performs an update on some semi-supervised objective.
Move the BERT-LMAO objective out from spacy/cli/pretrain.py into spacy/_ml.py, so we can create a new pipeline component, ClozeMultitask. This can be specified as a parser or NER multitask in the spacy train command. Example usage:
python -m spacy train en ./tmp ~/data/en-core-web/train/nw.json ~/data/en-core-web/dev/nw.json --pipeline parser --raw-textt ~/data/unlabelled/reddit-100k.jsonl --vectors en_vectors_web_lg --parser-multitasks cloze
Implement rehearsal methods for pipeline components
The new --raw-text argument and nlp.rehearse() method also gives us a good place to implement the the idea in the pseudo-rehearsal blog post in the parser. This works as follows:
Add a new nlp.resume_training() method. This allocates copies of pre-trained models in the pipeline, setting things up for the rehearsal updates. It also returns an optimizer object. This also greatly reduces confusion around the nlp.begin_training() method, which randomises the weights, making it not suitable for adding new labels or otherwise fine-tuning a pre-trained model.
Implement rehearsal updates on the Parser class, making it available for the dependency parser and NER. During rehearsal, the initial model is used to supervise the model being trained. The current model is asked to match the predictions of the initial model on some data. This minimises catastrophic forgetting, by keeping the model's predictions close to the original. See the blog post for details.
Implement rehearsal updates for tagger
Implement rehearsal updates for text categoriz
2018-12-10 18:25:33 +03:00
|
|
|
|
if self.vocab.vectors.data.shape[1] >= 1:
|
2020-01-29 19:06:46 +03:00
|
|
|
|
self.vocab.vectors.data = ops.asarray(self.vocab.vectors.data)
|
💫 Better support for semi-supervised learning (#3035)
The new spacy pretrain command implemented BERT/ULMFit/etc-like transfer learning, using our Language Modelling with Approximate Outputs version of BERT's cloze task. Pretraining is convenient, but in some ways it's a bit of a strange solution. All we're doing is initialising the weights. At the same time, we're putting a lot of work into our optimisation so that it's less sensitive to initial conditions, and more likely to find good optima. I discuss this a bit in the pseudo-rehearsal blog post: https://explosion.ai/blog/pseudo-rehearsal-catastrophic-forgetting
Support semi-supervised learning in spacy train
One obvious way to improve these pretraining methods is to do multi-task learning, instead of just transfer learning. This has been shown to work very well: https://arxiv.org/pdf/1809.08370.pdf . This patch makes it easy to do this sort of thing.
Add a new argument to spacy train, --raw-text. This takes a jsonl file with unlabelled data that can be used in arbitrary ways to do semi-supervised learning.
Add a new method to the Language class and to pipeline components, .rehearse(). This is like .update(), but doesn't expect GoldParse objects. It takes a batch of Doc objects, and performs an update on some semi-supervised objective.
Move the BERT-LMAO objective out from spacy/cli/pretrain.py into spacy/_ml.py, so we can create a new pipeline component, ClozeMultitask. This can be specified as a parser or NER multitask in the spacy train command. Example usage:
python -m spacy train en ./tmp ~/data/en-core-web/train/nw.json ~/data/en-core-web/dev/nw.json --pipeline parser --raw-textt ~/data/unlabelled/reddit-100k.jsonl --vectors en_vectors_web_lg --parser-multitasks cloze
Implement rehearsal methods for pipeline components
The new --raw-text argument and nlp.rehearse() method also gives us a good place to implement the the idea in the pseudo-rehearsal blog post in the parser. This works as follows:
Add a new nlp.resume_training() method. This allocates copies of pre-trained models in the pipeline, setting things up for the rehearsal updates. It also returns an optimizer object. This also greatly reduces confusion around the nlp.begin_training() method, which randomises the weights, making it not suitable for adding new labels or otherwise fine-tuning a pre-trained model.
Implement rehearsal updates on the Parser class, making it available for the dependency parser and NER. During rehearsal, the initial model is used to supervise the model being trained. The current model is asked to match the predictions of the initial model on some data. This minimises catastrophic forgetting, by keeping the model's predictions close to the original. See the blog post for details.
Implement rehearsal updates for tagger
Implement rehearsal updates for text categoriz
2018-12-10 18:25:33 +03:00
|
|
|
|
link_vectors_to_models(self.vocab)
|
|
|
|
|
if sgd is None:
|
2020-01-29 19:06:46 +03:00
|
|
|
|
sgd = create_default_optimizer()
|
💫 Better support for semi-supervised learning (#3035)
The new spacy pretrain command implemented BERT/ULMFit/etc-like transfer learning, using our Language Modelling with Approximate Outputs version of BERT's cloze task. Pretraining is convenient, but in some ways it's a bit of a strange solution. All we're doing is initialising the weights. At the same time, we're putting a lot of work into our optimisation so that it's less sensitive to initial conditions, and more likely to find good optima. I discuss this a bit in the pseudo-rehearsal blog post: https://explosion.ai/blog/pseudo-rehearsal-catastrophic-forgetting
Support semi-supervised learning in spacy train
One obvious way to improve these pretraining methods is to do multi-task learning, instead of just transfer learning. This has been shown to work very well: https://arxiv.org/pdf/1809.08370.pdf . This patch makes it easy to do this sort of thing.
Add a new argument to spacy train, --raw-text. This takes a jsonl file with unlabelled data that can be used in arbitrary ways to do semi-supervised learning.
Add a new method to the Language class and to pipeline components, .rehearse(). This is like .update(), but doesn't expect GoldParse objects. It takes a batch of Doc objects, and performs an update on some semi-supervised objective.
Move the BERT-LMAO objective out from spacy/cli/pretrain.py into spacy/_ml.py, so we can create a new pipeline component, ClozeMultitask. This can be specified as a parser or NER multitask in the spacy train command. Example usage:
python -m spacy train en ./tmp ~/data/en-core-web/train/nw.json ~/data/en-core-web/dev/nw.json --pipeline parser --raw-textt ~/data/unlabelled/reddit-100k.jsonl --vectors en_vectors_web_lg --parser-multitasks cloze
Implement rehearsal methods for pipeline components
The new --raw-text argument and nlp.rehearse() method also gives us a good place to implement the the idea in the pseudo-rehearsal blog post in the parser. This works as follows:
Add a new nlp.resume_training() method. This allocates copies of pre-trained models in the pipeline, setting things up for the rehearsal updates. It also returns an optimizer object. This also greatly reduces confusion around the nlp.begin_training() method, which randomises the weights, making it not suitable for adding new labels or otherwise fine-tuning a pre-trained model.
Implement rehearsal updates on the Parser class, making it available for the dependency parser and NER. During rehearsal, the initial model is used to supervise the model being trained. The current model is asked to match the predictions of the initial model on some data. This minimises catastrophic forgetting, by keeping the model's predictions close to the original. See the blog post for details.
Implement rehearsal updates for tagger
Implement rehearsal updates for text categoriz
2018-12-10 18:25:33 +03:00
|
|
|
|
self._optimizer = sgd
|
|
|
|
|
for name, proc in self.pipeline:
|
|
|
|
|
if hasattr(proc, "_rehearsal_model"):
|
|
|
|
|
proc._rehearsal_model = deepcopy(proc.model)
|
|
|
|
|
return self._optimizer
|
|
|
|
|
|
2019-03-11 01:36:47 +03:00
|
|
|
|
def evaluate(
|
2020-07-22 14:42:59 +03:00
|
|
|
|
self,
|
|
|
|
|
examples: Iterable[Example],
|
2020-07-29 00:12:42 +03:00
|
|
|
|
*,
|
2020-07-22 14:42:59 +03:00
|
|
|
|
verbose: bool = False,
|
|
|
|
|
batch_size: int = 256,
|
|
|
|
|
scorer: Optional[Scorer] = None,
|
|
|
|
|
component_cfg: Optional[Dict[str, Dict[str, Any]]] = None,
|
2020-07-29 00:12:42 +03:00
|
|
|
|
) -> Dict[str, Union[float, dict]]:
|
2019-05-24 15:06:36 +03:00
|
|
|
|
"""Evaluate a model's pipeline components.
|
|
|
|
|
|
2020-07-22 14:42:59 +03:00
|
|
|
|
examples (Iterable[Example]): `Example` objects.
|
2019-05-24 15:06:36 +03:00
|
|
|
|
verbose (bool): Print debugging information.
|
|
|
|
|
batch_size (int): Batch size to use.
|
2020-07-22 14:42:59 +03:00
|
|
|
|
scorer (Optional[Scorer]): Scorer to use. If not passed in, a new one
|
2019-05-24 15:06:36 +03:00
|
|
|
|
will be created.
|
|
|
|
|
component_cfg (dict): An optional dictionary with extra keyword
|
|
|
|
|
arguments for specific components.
|
|
|
|
|
RETURNS (Scorer): The scorer containing the evaluation results.
|
|
|
|
|
|
|
|
|
|
DOCS: https://spacy.io/api/language#evaluate
|
|
|
|
|
"""
|
2020-07-22 14:42:59 +03:00
|
|
|
|
if not isinstance(examples, IterableInstance):
|
|
|
|
|
err = Errors.E978.format(
|
|
|
|
|
name="language", method="evaluate", types=type(examples)
|
2020-07-12 15:03:23 +03:00
|
|
|
|
)
|
2020-07-22 14:42:59 +03:00
|
|
|
|
raise TypeError(err)
|
2020-07-06 14:02:36 +03:00
|
|
|
|
wrong_types = set([type(eg) for eg in examples if not isinstance(eg, Example)])
|
|
|
|
|
if wrong_types:
|
2020-07-22 14:42:59 +03:00
|
|
|
|
err = Errors.E978.format(
|
|
|
|
|
name="language", method="evaluate", types=wrong_types
|
2020-07-12 15:03:23 +03:00
|
|
|
|
)
|
2020-07-22 14:42:59 +03:00
|
|
|
|
raise TypeError(err)
|
2019-03-15 17:20:09 +03:00
|
|
|
|
if component_cfg is None:
|
|
|
|
|
component_cfg = {}
|
Refactor the Scorer to improve flexibility (#5731)
* Refactor the Scorer to improve flexibility
Refactor the `Scorer` to improve flexibility for arbitrary pipeline
components.
* Individual pipeline components provide their own `evaluate` methods
that score a list of `Example`s and return a dictionary of scores
* `Scorer` is initialized either:
* with a provided pipeline containing components to be scored
* with a default pipeline containing the built-in statistical
components (senter, tagger, morphologizer, parser, ner)
* `Scorer.score` evaluates a list of `Example`s and returns a dictionary
of scores referring to the scores provided by the components in the
pipeline
Significant differences:
* `tags_acc` is renamed to `tag_acc` to be consistent with `token_acc`
and the new `morph_acc`, `pos_acc`, and `lemma_acc`
* Scoring is no longer cumulative: `Scorer.score` scores a list of
examples rather than a single example and does not retain any state
about previously scored examples
* PRF values in the returned scores are no longer multiplied by 100
* Add kwargs to Morphologizer.evaluate
* Create generalized scoring methods in Scorer
* Generalized static scoring methods are added to `Scorer`
* Methods require an attribute (either on Token or Doc) that is
used to key the returned scores
Naming differences:
* `uas`, `las`, and `las_per_type` in the scores dict are renamed to
`dep_uas`, `dep_las`, and `dep_las_per_type`
Scoring differences:
* `Doc.sents` is now scored as spans rather than on sentence-initial
token positions so that `Doc.sents` and `Doc.ents` can be scored with
the same method (this lowers scores since a single incorrect sentence
start results in two incorrect spans)
* Simplify / extend hasattr check for eval method
* Add hasattr check to tokenizer scoring
* Simplify to hasattr check for component scoring
* Reset Example alignment if docs are set
Reset the Example alignment if either doc is set in case the
tokenization has changed.
* Add PRF tokenization scoring for tokens as spans
Add PRF scores for tokens as character spans. The scores are:
* token_acc: # correct tokens / # gold tokens
* token_p/r/f: PRF for (token.idx, token.idx + len(token))
* Add docstring to Scorer.score_tokenization
* Rename component.evaluate() to component.score()
* Update Scorer API docs
* Update scoring for positive_label in textcat
* Fix TextCategorizer.score kwargs
* Update Language.evaluate docs
* Update score names in default config
2020-07-25 13:53:02 +03:00
|
|
|
|
if scorer is None:
|
|
|
|
|
kwargs = component_cfg.get("scorer", {})
|
|
|
|
|
kwargs.setdefault("verbose", verbose)
|
|
|
|
|
kwargs.setdefault("nlp", self)
|
|
|
|
|
scorer = Scorer(**kwargs)
|
2020-06-26 20:34:12 +03:00
|
|
|
|
docs = list(eg.predicted for eg in examples)
|
2017-10-07 01:25:54 +03:00
|
|
|
|
for name, pipe in self.pipeline:
|
2019-03-11 01:36:47 +03:00
|
|
|
|
kwargs = component_cfg.get(name, {})
|
|
|
|
|
kwargs.setdefault("batch_size", batch_size)
|
💫 Tidy up and auto-format .py files (#2983)
<!--- Provide a general summary of your changes in the title. -->
## Description
- [x] Use [`black`](https://github.com/ambv/black) to auto-format all `.py` files.
- [x] Update flake8 config to exclude very large files (lemmatization tables etc.)
- [x] Update code to be compatible with flake8 rules
- [x] Fix various small bugs, inconsistencies and messy stuff in the language data
- [x] Update docs to explain new code style (`black`, `flake8`, when to use `# fmt: off` and `# fmt: on` and what `# noqa` means)
Once #2932 is merged, which auto-formats and tidies up the CLI, we'll be able to run `flake8 spacy` actually get meaningful results.
At the moment, the code style and linting isn't applied automatically, but I'm hoping that the new [GitHub Actions](https://github.com/features/actions) will let us auto-format pull requests and post comments with relevant linting information.
### Types of change
enhancement, code style
## Checklist
<!--- Before you submit the PR, go over this checklist and make sure you can
tick off all the boxes. [] -> [x] -->
- [x] I have submitted the spaCy Contributor Agreement.
- [x] I ran the tests, and all new and existing tests passed.
- [x] My changes don't require a change to the documentation, or if they do, I've added all required information.
2018-11-30 19:03:03 +03:00
|
|
|
|
if not hasattr(pipe, "pipe"):
|
2019-11-16 22:20:37 +03:00
|
|
|
|
docs = _pipe(docs, pipe, kwargs)
|
2017-08-18 23:26:12 +03:00
|
|
|
|
else:
|
2019-03-11 01:36:47 +03:00
|
|
|
|
docs = pipe.pipe(docs, **kwargs)
|
2020-06-26 20:34:12 +03:00
|
|
|
|
for i, (doc, eg) in enumerate(zip(docs, examples)):
|
2017-10-03 17:14:57 +03:00
|
|
|
|
if verbose:
|
|
|
|
|
print(doc)
|
2020-06-26 20:34:12 +03:00
|
|
|
|
eg.predicted = doc
|
Refactor the Scorer to improve flexibility (#5731)
* Refactor the Scorer to improve flexibility
Refactor the `Scorer` to improve flexibility for arbitrary pipeline
components.
* Individual pipeline components provide their own `evaluate` methods
that score a list of `Example`s and return a dictionary of scores
* `Scorer` is initialized either:
* with a provided pipeline containing components to be scored
* with a default pipeline containing the built-in statistical
components (senter, tagger, morphologizer, parser, ner)
* `Scorer.score` evaluates a list of `Example`s and returns a dictionary
of scores referring to the scores provided by the components in the
pipeline
Significant differences:
* `tags_acc` is renamed to `tag_acc` to be consistent with `token_acc`
and the new `morph_acc`, `pos_acc`, and `lemma_acc`
* Scoring is no longer cumulative: `Scorer.score` scores a list of
examples rather than a single example and does not retain any state
about previously scored examples
* PRF values in the returned scores are no longer multiplied by 100
* Add kwargs to Morphologizer.evaluate
* Create generalized scoring methods in Scorer
* Generalized static scoring methods are added to `Scorer`
* Methods require an attribute (either on Token or Doc) that is
used to key the returned scores
Naming differences:
* `uas`, `las`, and `las_per_type` in the scores dict are renamed to
`dep_uas`, `dep_las`, and `dep_las_per_type`
Scoring differences:
* `Doc.sents` is now scored as spans rather than on sentence-initial
token positions so that `Doc.sents` and `Doc.ents` can be scored with
the same method (this lowers scores since a single incorrect sentence
start results in two incorrect spans)
* Simplify / extend hasattr check for eval method
* Add hasattr check to tokenizer scoring
* Simplify to hasattr check for component scoring
* Reset Example alignment if docs are set
Reset the Example alignment if either doc is set in case the
tokenization has changed.
* Add PRF tokenization scoring for tokens as spans
Add PRF scores for tokens as character spans. The scores are:
* token_acc: # correct tokens / # gold tokens
* token_p/r/f: PRF for (token.idx, token.idx + len(token))
* Add docstring to Scorer.score_tokenization
* Rename component.evaluate() to component.score()
* Update Scorer API docs
* Update scoring for positive_label in textcat
* Fix TextCategorizer.score kwargs
* Update Language.evaluate docs
* Update score names in default config
2020-07-25 13:53:02 +03:00
|
|
|
|
return scorer.score(examples)
|
2017-05-16 12:21:59 +03:00
|
|
|
|
|
2017-05-18 12:25:19 +03:00
|
|
|
|
@contextmanager
|
2020-07-28 14:37:31 +03:00
|
|
|
|
def use_params(self, params: dict):
|
2017-05-19 00:57:38 +03:00
|
|
|
|
"""Replace weights of models in the pipeline with those provided in the
|
|
|
|
|
params dictionary. Can be used as a contextmanager, in which case,
|
|
|
|
|
models go back to their original weights after the block.
|
|
|
|
|
|
|
|
|
|
params (dict): A dictionary of parameters keyed by model ID.
|
|
|
|
|
|
|
|
|
|
EXAMPLE:
|
|
|
|
|
>>> with nlp.use_params(optimizer.averages):
|
2020-07-29 00:12:42 +03:00
|
|
|
|
>>> nlp.to_disk("/tmp/checkpoint")
|
|
|
|
|
|
|
|
|
|
DOCS: https://spacy.io/api/language#use_params
|
2017-05-19 00:57:38 +03:00
|
|
|
|
"""
|
💫 Tidy up and auto-format .py files (#2983)
<!--- Provide a general summary of your changes in the title. -->
## Description
- [x] Use [`black`](https://github.com/ambv/black) to auto-format all `.py` files.
- [x] Update flake8 config to exclude very large files (lemmatization tables etc.)
- [x] Update code to be compatible with flake8 rules
- [x] Fix various small bugs, inconsistencies and messy stuff in the language data
- [x] Update docs to explain new code style (`black`, `flake8`, when to use `# fmt: off` and `# fmt: on` and what `# noqa` means)
Once #2932 is merged, which auto-formats and tidies up the CLI, we'll be able to run `flake8 spacy` actually get meaningful results.
At the moment, the code style and linting isn't applied automatically, but I'm hoping that the new [GitHub Actions](https://github.com/features/actions) will let us auto-format pull requests and post comments with relevant linting information.
### Types of change
enhancement, code style
## Checklist
<!--- Before you submit the PR, go over this checklist and make sure you can
tick off all the boxes. [] -> [x] -->
- [x] I have submitted the spaCy Contributor Agreement.
- [x] I ran the tests, and all new and existing tests passed.
- [x] My changes don't require a change to the documentation, or if they do, I've added all required information.
2018-11-30 19:03:03 +03:00
|
|
|
|
contexts = [
|
|
|
|
|
pipe.use_params(params)
|
|
|
|
|
for name, pipe in self.pipeline
|
2020-05-20 12:41:12 +03:00
|
|
|
|
if hasattr(pipe, "use_params") and hasattr(pipe, "model")
|
💫 Tidy up and auto-format .py files (#2983)
<!--- Provide a general summary of your changes in the title. -->
## Description
- [x] Use [`black`](https://github.com/ambv/black) to auto-format all `.py` files.
- [x] Update flake8 config to exclude very large files (lemmatization tables etc.)
- [x] Update code to be compatible with flake8 rules
- [x] Fix various small bugs, inconsistencies and messy stuff in the language data
- [x] Update docs to explain new code style (`black`, `flake8`, when to use `# fmt: off` and `# fmt: on` and what `# noqa` means)
Once #2932 is merged, which auto-formats and tidies up the CLI, we'll be able to run `flake8 spacy` actually get meaningful results.
At the moment, the code style and linting isn't applied automatically, but I'm hoping that the new [GitHub Actions](https://github.com/features/actions) will let us auto-format pull requests and post comments with relevant linting information.
### Types of change
enhancement, code style
## Checklist
<!--- Before you submit the PR, go over this checklist and make sure you can
tick off all the boxes. [] -> [x] -->
- [x] I have submitted the spaCy Contributor Agreement.
- [x] I ran the tests, and all new and existing tests passed.
- [x] My changes don't require a change to the documentation, or if they do, I've added all required information.
2018-11-30 19:03:03 +03:00
|
|
|
|
]
|
2017-05-18 16:30:59 +03:00
|
|
|
|
# TODO: Having trouble with contextlib
|
|
|
|
|
# Workaround: these aren't actually context managers atm.
|
|
|
|
|
for context in contexts:
|
|
|
|
|
try:
|
|
|
|
|
next(context)
|
|
|
|
|
except StopIteration:
|
|
|
|
|
pass
|
2017-05-18 12:25:19 +03:00
|
|
|
|
yield
|
|
|
|
|
for context in contexts:
|
|
|
|
|
try:
|
2017-05-18 16:30:59 +03:00
|
|
|
|
next(context)
|
2017-05-18 12:25:19 +03:00
|
|
|
|
except StopIteration:
|
|
|
|
|
pass
|
|
|
|
|
|
💫 Tidy up and auto-format .py files (#2983)
<!--- Provide a general summary of your changes in the title. -->
## Description
- [x] Use [`black`](https://github.com/ambv/black) to auto-format all `.py` files.
- [x] Update flake8 config to exclude very large files (lemmatization tables etc.)
- [x] Update code to be compatible with flake8 rules
- [x] Fix various small bugs, inconsistencies and messy stuff in the language data
- [x] Update docs to explain new code style (`black`, `flake8`, when to use `# fmt: off` and `# fmt: on` and what `# noqa` means)
Once #2932 is merged, which auto-formats and tidies up the CLI, we'll be able to run `flake8 spacy` actually get meaningful results.
At the moment, the code style and linting isn't applied automatically, but I'm hoping that the new [GitHub Actions](https://github.com/features/actions) will let us auto-format pull requests and post comments with relevant linting information.
### Types of change
enhancement, code style
## Checklist
<!--- Before you submit the PR, go over this checklist and make sure you can
tick off all the boxes. [] -> [x] -->
- [x] I have submitted the spaCy Contributor Agreement.
- [x] I ran the tests, and all new and existing tests passed.
- [x] My changes don't require a change to the documentation, or if they do, I've added all required information.
2018-11-30 19:03:03 +03:00
|
|
|
|
def pipe(
|
|
|
|
|
self,
|
2020-07-22 14:42:59 +03:00
|
|
|
|
texts: Iterable[str],
|
2020-07-29 00:12:42 +03:00
|
|
|
|
*,
|
2020-07-22 14:42:59 +03:00
|
|
|
|
as_tuples: bool = False,
|
|
|
|
|
batch_size: int = 1000,
|
|
|
|
|
disable: Iterable[str] = tuple(),
|
|
|
|
|
cleanup: bool = False,
|
|
|
|
|
component_cfg: Optional[Dict[str, Dict[str, Any]]] = None,
|
|
|
|
|
n_process: int = 1,
|
💫 Tidy up and auto-format .py files (#2983)
<!--- Provide a general summary of your changes in the title. -->
## Description
- [x] Use [`black`](https://github.com/ambv/black) to auto-format all `.py` files.
- [x] Update flake8 config to exclude very large files (lemmatization tables etc.)
- [x] Update code to be compatible with flake8 rules
- [x] Fix various small bugs, inconsistencies and messy stuff in the language data
- [x] Update docs to explain new code style (`black`, `flake8`, when to use `# fmt: off` and `# fmt: on` and what `# noqa` means)
Once #2932 is merged, which auto-formats and tidies up the CLI, we'll be able to run `flake8 spacy` actually get meaningful results.
At the moment, the code style and linting isn't applied automatically, but I'm hoping that the new [GitHub Actions](https://github.com/features/actions) will let us auto-format pull requests and post comments with relevant linting information.
### Types of change
enhancement, code style
## Checklist
<!--- Before you submit the PR, go over this checklist and make sure you can
tick off all the boxes. [] -> [x] -->
- [x] I have submitted the spaCy Contributor Agreement.
- [x] I ran the tests, and all new and existing tests passed.
- [x] My changes don't require a change to the documentation, or if they do, I've added all required information.
2018-11-30 19:03:03 +03:00
|
|
|
|
):
|
2017-10-27 15:40:14 +03:00
|
|
|
|
"""Process texts as a stream, and yield `Doc` objects in order.
|
2017-05-19 00:57:38 +03:00
|
|
|
|
|
2020-07-09 20:43:39 +03:00
|
|
|
|
texts (Iterable[str]): A sequence of texts to process.
|
2019-03-15 18:23:17 +03:00
|
|
|
|
as_tuples (bool): If set to True, inputs should be a sequence of
|
2017-08-19 13:21:33 +03:00
|
|
|
|
(text, context) tuples. Output will then be a sequence of
|
|
|
|
|
(doc, context) tuples. Defaults to False.
|
2017-05-19 00:57:38 +03:00
|
|
|
|
batch_size (int): The number of texts to buffer.
|
2020-07-09 20:43:39 +03:00
|
|
|
|
disable (List[str]): Names of the pipeline components to disable.
|
2019-03-15 18:23:17 +03:00
|
|
|
|
cleanup (bool): If True, unneeded strings are freed to control memory
|
|
|
|
|
use. Experimental.
|
2020-07-09 20:43:39 +03:00
|
|
|
|
component_cfg (Dict[str, Dict]): An optional dictionary with extra keyword
|
2019-03-15 18:23:17 +03:00
|
|
|
|
arguments for specific components.
|
2020-07-09 20:43:39 +03:00
|
|
|
|
n_process (int): Number of processors to process texts. If -1, set `multiprocessing.cpu_count()`.
|
2017-05-19 00:57:38 +03:00
|
|
|
|
YIELDS (Doc): Documents in the order of the original text.
|
|
|
|
|
|
2019-03-15 18:23:17 +03:00
|
|
|
|
DOCS: https://spacy.io/api/language#pipe
|
2017-04-15 12:59:21 +03:00
|
|
|
|
"""
|
2019-10-08 13:20:55 +03:00
|
|
|
|
if n_process == -1:
|
|
|
|
|
n_process = mp.cpu_count()
|
2017-08-19 13:21:33 +03:00
|
|
|
|
if as_tuples:
|
2017-07-25 19:57:59 +03:00
|
|
|
|
text_context1, text_context2 = itertools.tee(texts)
|
|
|
|
|
texts = (tc[0] for tc in text_context1)
|
|
|
|
|
contexts = (tc[1] for tc in text_context2)
|
💫 Tidy up and auto-format .py files (#2983)
<!--- Provide a general summary of your changes in the title. -->
## Description
- [x] Use [`black`](https://github.com/ambv/black) to auto-format all `.py` files.
- [x] Update flake8 config to exclude very large files (lemmatization tables etc.)
- [x] Update code to be compatible with flake8 rules
- [x] Fix various small bugs, inconsistencies and messy stuff in the language data
- [x] Update docs to explain new code style (`black`, `flake8`, when to use `# fmt: off` and `# fmt: on` and what `# noqa` means)
Once #2932 is merged, which auto-formats and tidies up the CLI, we'll be able to run `flake8 spacy` actually get meaningful results.
At the moment, the code style and linting isn't applied automatically, but I'm hoping that the new [GitHub Actions](https://github.com/features/actions) will let us auto-format pull requests and post comments with relevant linting information.
### Types of change
enhancement, code style
## Checklist
<!--- Before you submit the PR, go over this checklist and make sure you can
tick off all the boxes. [] -> [x] -->
- [x] I have submitted the spaCy Contributor Agreement.
- [x] I ran the tests, and all new and existing tests passed.
- [x] My changes don't require a change to the documentation, or if they do, I've added all required information.
2018-11-30 19:03:03 +03:00
|
|
|
|
docs = self.pipe(
|
2019-03-11 01:36:47 +03:00
|
|
|
|
texts,
|
|
|
|
|
batch_size=batch_size,
|
|
|
|
|
disable=disable,
|
2019-11-04 22:29:03 +03:00
|
|
|
|
n_process=n_process,
|
2019-03-11 01:36:47 +03:00
|
|
|
|
component_cfg=component_cfg,
|
💫 Tidy up and auto-format .py files (#2983)
<!--- Provide a general summary of your changes in the title. -->
## Description
- [x] Use [`black`](https://github.com/ambv/black) to auto-format all `.py` files.
- [x] Update flake8 config to exclude very large files (lemmatization tables etc.)
- [x] Update code to be compatible with flake8 rules
- [x] Fix various small bugs, inconsistencies and messy stuff in the language data
- [x] Update docs to explain new code style (`black`, `flake8`, when to use `# fmt: off` and `# fmt: on` and what `# noqa` means)
Once #2932 is merged, which auto-formats and tidies up the CLI, we'll be able to run `flake8 spacy` actually get meaningful results.
At the moment, the code style and linting isn't applied automatically, but I'm hoping that the new [GitHub Actions](https://github.com/features/actions) will let us auto-format pull requests and post comments with relevant linting information.
### Types of change
enhancement, code style
## Checklist
<!--- Before you submit the PR, go over this checklist and make sure you can
tick off all the boxes. [] -> [x] -->
- [x] I have submitted the spaCy Contributor Agreement.
- [x] I ran the tests, and all new and existing tests passed.
- [x] My changes don't require a change to the documentation, or if they do, I've added all required information.
2018-11-30 19:03:03 +03:00
|
|
|
|
)
|
2019-12-22 03:53:56 +03:00
|
|
|
|
for doc, context in zip(docs, contexts):
|
2017-07-25 19:57:59 +03:00
|
|
|
|
yield (doc, context)
|
|
|
|
|
return
|
2019-03-11 01:36:47 +03:00
|
|
|
|
if component_cfg is None:
|
|
|
|
|
component_cfg = {}
|
2019-10-08 13:20:55 +03:00
|
|
|
|
|
|
|
|
|
pipes = (
|
|
|
|
|
[]
|
2020-01-06 16:57:34 +03:00
|
|
|
|
) # contains functools.partial objects to easily create multiprocess worker.
|
2017-10-07 01:25:54 +03:00
|
|
|
|
for name, proc in self.pipeline:
|
2017-05-26 13:33:54 +03:00
|
|
|
|
if name in disable:
|
2017-05-16 12:21:59 +03:00
|
|
|
|
continue
|
2019-03-11 01:36:47 +03:00
|
|
|
|
kwargs = component_cfg.get(name, {})
|
|
|
|
|
# Allow component_cfg to overwrite the top-level kwargs.
|
|
|
|
|
kwargs.setdefault("batch_size", batch_size)
|
💫 Tidy up and auto-format .py files (#2983)
<!--- Provide a general summary of your changes in the title. -->
## Description
- [x] Use [`black`](https://github.com/ambv/black) to auto-format all `.py` files.
- [x] Update flake8 config to exclude very large files (lemmatization tables etc.)
- [x] Update code to be compatible with flake8 rules
- [x] Fix various small bugs, inconsistencies and messy stuff in the language data
- [x] Update docs to explain new code style (`black`, `flake8`, when to use `# fmt: off` and `# fmt: on` and what `# noqa` means)
Once #2932 is merged, which auto-formats and tidies up the CLI, we'll be able to run `flake8 spacy` actually get meaningful results.
At the moment, the code style and linting isn't applied automatically, but I'm hoping that the new [GitHub Actions](https://github.com/features/actions) will let us auto-format pull requests and post comments with relevant linting information.
### Types of change
enhancement, code style
## Checklist
<!--- Before you submit the PR, go over this checklist and make sure you can
tick off all the boxes. [] -> [x] -->
- [x] I have submitted the spaCy Contributor Agreement.
- [x] I ran the tests, and all new and existing tests passed.
- [x] My changes don't require a change to the documentation, or if they do, I've added all required information.
2018-11-30 19:03:03 +03:00
|
|
|
|
if hasattr(proc, "pipe"):
|
2019-10-08 13:20:55 +03:00
|
|
|
|
f = functools.partial(proc.pipe, **kwargs)
|
2017-05-16 12:21:59 +03:00
|
|
|
|
else:
|
2017-05-22 02:43:31 +03:00
|
|
|
|
# Apply the function, but yield the doc
|
2019-10-08 13:20:55 +03:00
|
|
|
|
f = functools.partial(_pipe, proc=proc, kwargs=kwargs)
|
|
|
|
|
pipes.append(f)
|
|
|
|
|
|
|
|
|
|
if n_process != 1:
|
|
|
|
|
docs = self._multiprocessing_pipe(texts, pipes, n_process, batch_size)
|
|
|
|
|
else:
|
|
|
|
|
# if n_process == 1, no processes are forked.
|
|
|
|
|
docs = (self.make_doc(text) for text in texts)
|
|
|
|
|
for pipe in pipes:
|
|
|
|
|
docs = pipe(docs)
|
|
|
|
|
|
2017-10-16 20:22:40 +03:00
|
|
|
|
# Track weakrefs of "recent" documents, so that we can see when they
|
|
|
|
|
# expire from memory. When they do, we know we don't need old strings.
|
|
|
|
|
# This way, we avoid maintaining an unbounded growth in string entries
|
|
|
|
|
# in the string store.
|
|
|
|
|
recent_refs = weakref.WeakSet()
|
|
|
|
|
old_refs = weakref.WeakSet()
|
2017-11-23 15:19:18 +03:00
|
|
|
|
# Keep track of the original string data, so that if we flush old strings,
|
|
|
|
|
# we can recover the original ones. However, we only want to do this if we're
|
|
|
|
|
# really adding strings, to save up-front costs.
|
|
|
|
|
original_strings_data = None
|
2017-10-16 20:22:40 +03:00
|
|
|
|
nr_seen = 0
|
2017-05-19 21:25:42 +03:00
|
|
|
|
for doc in docs:
|
2016-02-03 04:04:55 +03:00
|
|
|
|
yield doc
|
2017-11-23 15:19:18 +03:00
|
|
|
|
if cleanup:
|
|
|
|
|
recent_refs.add(doc)
|
|
|
|
|
if nr_seen < 10000:
|
|
|
|
|
old_refs.add(doc)
|
|
|
|
|
nr_seen += 1
|
|
|
|
|
elif len(old_refs) == 0:
|
|
|
|
|
old_refs, recent_refs = recent_refs, old_refs
|
|
|
|
|
if original_strings_data is None:
|
|
|
|
|
original_strings_data = list(self.vocab.strings)
|
|
|
|
|
else:
|
💫 Tidy up and auto-format .py files (#2983)
<!--- Provide a general summary of your changes in the title. -->
## Description
- [x] Use [`black`](https://github.com/ambv/black) to auto-format all `.py` files.
- [x] Update flake8 config to exclude very large files (lemmatization tables etc.)
- [x] Update code to be compatible with flake8 rules
- [x] Fix various small bugs, inconsistencies and messy stuff in the language data
- [x] Update docs to explain new code style (`black`, `flake8`, when to use `# fmt: off` and `# fmt: on` and what `# noqa` means)
Once #2932 is merged, which auto-formats and tidies up the CLI, we'll be able to run `flake8 spacy` actually get meaningful results.
At the moment, the code style and linting isn't applied automatically, but I'm hoping that the new [GitHub Actions](https://github.com/features/actions) will let us auto-format pull requests and post comments with relevant linting information.
### Types of change
enhancement, code style
## Checklist
<!--- Before you submit the PR, go over this checklist and make sure you can
tick off all the boxes. [] -> [x] -->
- [x] I have submitted the spaCy Contributor Agreement.
- [x] I ran the tests, and all new and existing tests passed.
- [x] My changes don't require a change to the documentation, or if they do, I've added all required information.
2018-11-30 19:03:03 +03:00
|
|
|
|
keys, strings = self.vocab.strings._cleanup_stale_strings(
|
|
|
|
|
original_strings_data
|
|
|
|
|
)
|
2017-11-23 15:19:18 +03:00
|
|
|
|
self.vocab._reset_cache(keys, strings)
|
|
|
|
|
self.tokenizer._reset_cache(keys)
|
|
|
|
|
nr_seen = 0
|
2016-02-01 11:01:13 +03:00
|
|
|
|
|
2020-07-22 14:42:59 +03:00
|
|
|
|
def _multiprocessing_pipe(
|
|
|
|
|
self,
|
|
|
|
|
texts: Iterable[str],
|
|
|
|
|
pipes: Iterable[Callable[[Doc], Doc]],
|
|
|
|
|
n_process: int,
|
|
|
|
|
batch_size: int,
|
|
|
|
|
) -> None:
|
2019-10-08 13:20:55 +03:00
|
|
|
|
# raw_texts is used later to stop iteration.
|
|
|
|
|
texts, raw_texts = itertools.tee(texts)
|
|
|
|
|
# for sending texts to worker
|
|
|
|
|
texts_q = [mp.Queue() for _ in range(n_process)]
|
2020-01-06 16:57:34 +03:00
|
|
|
|
# for receiving byte-encoded docs from worker
|
2019-10-08 13:20:55 +03:00
|
|
|
|
bytedocs_recv_ch, bytedocs_send_ch = zip(
|
|
|
|
|
*[mp.Pipe(False) for _ in range(n_process)]
|
|
|
|
|
)
|
|
|
|
|
|
2020-05-21 21:05:03 +03:00
|
|
|
|
batch_texts = util.minibatch(texts, batch_size)
|
2019-10-08 13:20:55 +03:00
|
|
|
|
# Sender sends texts to the workers.
|
|
|
|
|
# This is necessary to properly handle infinite length of texts.
|
|
|
|
|
# (In this case, all data cannot be sent to the workers at once)
|
|
|
|
|
sender = _Sender(batch_texts, texts_q, chunk_size=n_process)
|
2020-01-06 16:57:34 +03:00
|
|
|
|
# send twice to make process busy
|
2019-10-08 13:20:55 +03:00
|
|
|
|
sender.send()
|
|
|
|
|
sender.send()
|
|
|
|
|
|
|
|
|
|
procs = [
|
2020-02-12 13:50:42 +03:00
|
|
|
|
mp.Process(
|
|
|
|
|
target=_apply_pipes,
|
2020-03-26 15:38:14 +03:00
|
|
|
|
args=(self.make_doc, pipes, rch, sch, Underscore.get_state()),
|
2020-02-12 13:50:42 +03:00
|
|
|
|
)
|
2019-10-08 13:20:55 +03:00
|
|
|
|
for rch, sch in zip(texts_q, bytedocs_send_ch)
|
|
|
|
|
]
|
|
|
|
|
for proc in procs:
|
|
|
|
|
proc.start()
|
|
|
|
|
|
|
|
|
|
# Cycle channels not to break the order of docs.
|
2020-01-06 16:57:34 +03:00
|
|
|
|
# The received object is a batch of byte-encoded docs, so flatten them with chain.from_iterable.
|
2019-10-08 13:20:55 +03:00
|
|
|
|
byte_docs = chain.from_iterable(recv.recv() for recv in cycle(bytedocs_recv_ch))
|
|
|
|
|
docs = (Doc(self.vocab).from_bytes(byte_doc) for byte_doc in byte_docs)
|
|
|
|
|
try:
|
|
|
|
|
for i, (_, doc) in enumerate(zip(raw_texts, docs), 1):
|
|
|
|
|
yield doc
|
|
|
|
|
if i % batch_size == 0:
|
|
|
|
|
# tell `sender` that one batch was consumed.
|
|
|
|
|
sender.step()
|
|
|
|
|
finally:
|
|
|
|
|
for proc in procs:
|
|
|
|
|
proc.terminate()
|
|
|
|
|
|
2020-07-22 14:42:59 +03:00
|
|
|
|
def _link_components(self) -> None:
|
2020-01-29 19:06:46 +03:00
|
|
|
|
"""Register 'listeners' within pipeline components, to allow them to
|
|
|
|
|
effectively share weights.
|
|
|
|
|
"""
|
|
|
|
|
for i, (name1, proc1) in enumerate(self.pipeline):
|
|
|
|
|
if hasattr(proc1, "find_listeners"):
|
|
|
|
|
for name2, proc2 in self.pipeline[i:]:
|
|
|
|
|
if hasattr(proc2, "model"):
|
|
|
|
|
proc1.find_listeners(proc2.model)
|
|
|
|
|
|
2020-07-22 14:42:59 +03:00
|
|
|
|
@classmethod
|
|
|
|
|
def from_config(
|
|
|
|
|
cls,
|
|
|
|
|
config: Union[Dict[str, Any], Config] = {},
|
2020-07-27 01:27:53 +03:00
|
|
|
|
*,
|
2020-07-22 14:42:59 +03:00
|
|
|
|
disable: Iterable[str] = tuple(),
|
|
|
|
|
overrides: Dict[str, Any] = {},
|
|
|
|
|
auto_fill: bool = True,
|
|
|
|
|
validate: bool = True,
|
|
|
|
|
) -> "Language":
|
|
|
|
|
"""Create the nlp object from a loaded config. Will set up the tokenizer
|
|
|
|
|
and language data, add pipeline components etc. If no config is provided,
|
|
|
|
|
the default config of the given language is used.
|
2020-07-29 00:12:42 +03:00
|
|
|
|
|
|
|
|
|
config (Dict[str, Any] / Config): The loaded config.
|
|
|
|
|
disable (Iterable[str]): List of pipeline component names to disable.
|
|
|
|
|
auto_fill (bool): Automatically fill in missing values in config based
|
|
|
|
|
on defaults and function argument annotations.
|
|
|
|
|
validate (bool): Validate the component config and arguments against
|
|
|
|
|
the types expected by the factory.
|
|
|
|
|
RETURNS (Language): The initialized Language class.
|
|
|
|
|
|
|
|
|
|
DOCS: https://spacy.io/api/language#from_config
|
2020-07-22 14:42:59 +03:00
|
|
|
|
"""
|
|
|
|
|
if auto_fill:
|
|
|
|
|
config = util.deep_merge_configs(config, cls.default_config)
|
|
|
|
|
if "nlp" not in config:
|
|
|
|
|
raise ValueError(Errors.E985.format(config=config))
|
2020-07-24 15:50:26 +03:00
|
|
|
|
config_lang = config["nlp"]["lang"]
|
2020-07-22 14:42:59 +03:00
|
|
|
|
if cls.lang is not None and config_lang is not None and config_lang != cls.lang:
|
|
|
|
|
raise ValueError(
|
|
|
|
|
Errors.E958.format(
|
2020-07-24 15:50:26 +03:00
|
|
|
|
bad_lang_code=config["nlp"]["lang"],
|
2020-07-22 14:42:59 +03:00
|
|
|
|
lang_code=cls.lang,
|
|
|
|
|
lang=util.get_object_name(cls),
|
|
|
|
|
)
|
|
|
|
|
)
|
2020-07-24 15:50:26 +03:00
|
|
|
|
config["nlp"]["lang"] = cls.lang
|
2020-07-22 14:42:59 +03:00
|
|
|
|
# This isn't very elegant, but we remove the [components] block here to prevent
|
|
|
|
|
# it from getting resolved (causes problems because we expect to pass in
|
|
|
|
|
# the nlp and name args for each component). If we're auto-filling, we're
|
|
|
|
|
# using the nlp.config with all defaults.
|
|
|
|
|
config = util.copy_config(config)
|
|
|
|
|
orig_pipeline = config.pop("components", {})
|
|
|
|
|
config["components"] = {}
|
|
|
|
|
non_pipe_overrides, pipe_overrides = _get_config_overrides(overrides)
|
|
|
|
|
resolved, filled = registry.resolve(
|
|
|
|
|
config, validate=validate, schema=ConfigSchema, overrides=non_pipe_overrides
|
|
|
|
|
)
|
|
|
|
|
filled["components"] = orig_pipeline
|
|
|
|
|
config["components"] = orig_pipeline
|
|
|
|
|
create_tokenizer = resolved["nlp"]["tokenizer"]
|
2020-07-24 15:50:26 +03:00
|
|
|
|
create_lemmatizer = resolved["nlp"]["lemmatizer"]
|
|
|
|
|
nlp = cls(
|
|
|
|
|
create_tokenizer=create_tokenizer, create_lemmatizer=create_lemmatizer,
|
2020-07-22 14:42:59 +03:00
|
|
|
|
)
|
|
|
|
|
pipeline = config.get("components", {})
|
2020-07-24 15:50:26 +03:00
|
|
|
|
for pipe_name in config["nlp"]["pipeline"]:
|
2020-07-22 14:42:59 +03:00
|
|
|
|
if pipe_name not in pipeline:
|
|
|
|
|
opts = ", ".join(pipeline.keys())
|
|
|
|
|
raise ValueError(Errors.E956.format(name=pipe_name, opts=opts))
|
2020-07-22 18:29:31 +03:00
|
|
|
|
pipe_cfg = util.copy_config(pipeline[pipe_name])
|
2020-07-22 14:42:59 +03:00
|
|
|
|
if pipe_name not in disable:
|
2020-07-22 18:29:31 +03:00
|
|
|
|
if "factory" not in pipe_cfg:
|
2020-07-22 14:42:59 +03:00
|
|
|
|
err = Errors.E984.format(name=pipe_name, config=pipe_cfg)
|
|
|
|
|
raise ValueError(err)
|
2020-07-22 18:29:31 +03:00
|
|
|
|
factory = pipe_cfg.pop("factory")
|
2020-07-22 14:42:59 +03:00
|
|
|
|
# The pipe name (key in the config) here is the unique name of the
|
|
|
|
|
# component, not necessarily the factory
|
|
|
|
|
nlp.add_pipe(
|
|
|
|
|
factory,
|
|
|
|
|
name=pipe_name,
|
|
|
|
|
config=pipe_cfg,
|
|
|
|
|
overrides=pipe_overrides,
|
|
|
|
|
validate=validate,
|
|
|
|
|
)
|
|
|
|
|
nlp.config = filled if auto_fill else config
|
|
|
|
|
nlp.resolved = resolved
|
|
|
|
|
return nlp
|
|
|
|
|
|
|
|
|
|
def to_disk(self, path: Union[str, Path], exclude: Iterable[str] = tuple()) -> None:
|
2017-05-26 13:33:54 +03:00
|
|
|
|
"""Save the current state to a directory. If a model is loaded, this
|
|
|
|
|
will include the model.
|
2017-04-17 02:40:26 +03:00
|
|
|
|
|
2020-05-24 19:51:10 +03:00
|
|
|
|
path (str / Path): Path to a directory, which will be created if
|
2019-03-10 21:16:45 +03:00
|
|
|
|
it doesn't exist.
|
|
|
|
|
exclude (list): Names of components or serialization fields to exclude.
|
2017-05-19 00:57:38 +03:00
|
|
|
|
|
2019-03-10 21:16:45 +03:00
|
|
|
|
DOCS: https://spacy.io/api/language#to_disk
|
2017-05-17 13:04:50 +03:00
|
|
|
|
"""
|
|
|
|
|
path = util.ensure_path(path)
|
2019-12-22 03:53:56 +03:00
|
|
|
|
serializers = {}
|
2019-08-01 18:13:01 +03:00
|
|
|
|
serializers["tokenizer"] = lambda p: self.tokenizer.to_disk(
|
|
|
|
|
p, exclude=["vocab"]
|
|
|
|
|
)
|
2020-04-06 19:54:32 +03:00
|
|
|
|
serializers["meta.json"] = lambda p: srsly.write_json(p, self.meta)
|
2020-02-27 20:42:27 +03:00
|
|
|
|
serializers["config.cfg"] = lambda p: self.config.to_disk(p)
|
2017-10-07 01:25:54 +03:00
|
|
|
|
for name, proc in self.pipeline:
|
💫 Tidy up and auto-format .py files (#2983)
<!--- Provide a general summary of your changes in the title. -->
## Description
- [x] Use [`black`](https://github.com/ambv/black) to auto-format all `.py` files.
- [x] Update flake8 config to exclude very large files (lemmatization tables etc.)
- [x] Update code to be compatible with flake8 rules
- [x] Fix various small bugs, inconsistencies and messy stuff in the language data
- [x] Update docs to explain new code style (`black`, `flake8`, when to use `# fmt: off` and `# fmt: on` and what `# noqa` means)
Once #2932 is merged, which auto-formats and tidies up the CLI, we'll be able to run `flake8 spacy` actually get meaningful results.
At the moment, the code style and linting isn't applied automatically, but I'm hoping that the new [GitHub Actions](https://github.com/features/actions) will let us auto-format pull requests and post comments with relevant linting information.
### Types of change
enhancement, code style
## Checklist
<!--- Before you submit the PR, go over this checklist and make sure you can
tick off all the boxes. [] -> [x] -->
- [x] I have submitted the spaCy Contributor Agreement.
- [x] I ran the tests, and all new and existing tests passed.
- [x] My changes don't require a change to the documentation, or if they do, I've added all required information.
2018-11-30 19:03:03 +03:00
|
|
|
|
if not hasattr(proc, "name"):
|
2017-05-31 14:42:39 +03:00
|
|
|
|
continue
|
2019-03-10 21:16:45 +03:00
|
|
|
|
if name in exclude:
|
2017-05-31 14:42:39 +03:00
|
|
|
|
continue
|
💫 Tidy up and auto-format .py files (#2983)
<!--- Provide a general summary of your changes in the title. -->
## Description
- [x] Use [`black`](https://github.com/ambv/black) to auto-format all `.py` files.
- [x] Update flake8 config to exclude very large files (lemmatization tables etc.)
- [x] Update code to be compatible with flake8 rules
- [x] Fix various small bugs, inconsistencies and messy stuff in the language data
- [x] Update docs to explain new code style (`black`, `flake8`, when to use `# fmt: off` and `# fmt: on` and what `# noqa` means)
Once #2932 is merged, which auto-formats and tidies up the CLI, we'll be able to run `flake8 spacy` actually get meaningful results.
At the moment, the code style and linting isn't applied automatically, but I'm hoping that the new [GitHub Actions](https://github.com/features/actions) will let us auto-format pull requests and post comments with relevant linting information.
### Types of change
enhancement, code style
## Checklist
<!--- Before you submit the PR, go over this checklist and make sure you can
tick off all the boxes. [] -> [x] -->
- [x] I have submitted the spaCy Contributor Agreement.
- [x] I ran the tests, and all new and existing tests passed.
- [x] My changes don't require a change to the documentation, or if they do, I've added all required information.
2018-11-30 19:03:03 +03:00
|
|
|
|
if not hasattr(proc, "to_disk"):
|
2017-05-31 14:42:39 +03:00
|
|
|
|
continue
|
2019-03-10 21:16:45 +03:00
|
|
|
|
serializers[name] = lambda p, proc=proc: proc.to_disk(p, exclude=["vocab"])
|
💫 Tidy up and auto-format .py files (#2983)
<!--- Provide a general summary of your changes in the title. -->
## Description
- [x] Use [`black`](https://github.com/ambv/black) to auto-format all `.py` files.
- [x] Update flake8 config to exclude very large files (lemmatization tables etc.)
- [x] Update code to be compatible with flake8 rules
- [x] Fix various small bugs, inconsistencies and messy stuff in the language data
- [x] Update docs to explain new code style (`black`, `flake8`, when to use `# fmt: off` and `# fmt: on` and what `# noqa` means)
Once #2932 is merged, which auto-formats and tidies up the CLI, we'll be able to run `flake8 spacy` actually get meaningful results.
At the moment, the code style and linting isn't applied automatically, but I'm hoping that the new [GitHub Actions](https://github.com/features/actions) will let us auto-format pull requests and post comments with relevant linting information.
### Types of change
enhancement, code style
## Checklist
<!--- Before you submit the PR, go over this checklist and make sure you can
tick off all the boxes. [] -> [x] -->
- [x] I have submitted the spaCy Contributor Agreement.
- [x] I ran the tests, and all new and existing tests passed.
- [x] My changes don't require a change to the documentation, or if they do, I've added all required information.
2018-11-30 19:03:03 +03:00
|
|
|
|
serializers["vocab"] = lambda p: self.vocab.to_disk(p)
|
2019-03-10 21:16:45 +03:00
|
|
|
|
util.to_disk(path, serializers, exclude)
|
2017-05-31 14:42:39 +03:00
|
|
|
|
|
2020-07-22 14:42:59 +03:00
|
|
|
|
def from_disk(
|
|
|
|
|
self, path: Union[str, Path], exclude: Iterable[str] = tuple()
|
|
|
|
|
) -> "Language":
|
2017-05-19 00:57:38 +03:00
|
|
|
|
"""Loads state from a directory. Modifies the object in place and
|
2017-05-26 13:33:54 +03:00
|
|
|
|
returns it. If the saved `Language` object contains a model, the
|
|
|
|
|
model will be loaded.
|
2017-05-17 13:04:50 +03:00
|
|
|
|
|
2020-05-24 19:51:10 +03:00
|
|
|
|
path (str / Path): A path to a directory.
|
2019-03-10 21:16:45 +03:00
|
|
|
|
exclude (list): Names of components or serialization fields to exclude.
|
2017-05-19 00:57:38 +03:00
|
|
|
|
RETURNS (Language): The modified `Language` object.
|
2017-05-17 13:04:50 +03:00
|
|
|
|
|
2019-03-10 21:16:45 +03:00
|
|
|
|
DOCS: https://spacy.io/api/language#from_disk
|
2017-05-17 13:04:50 +03:00
|
|
|
|
"""
|
2020-06-20 16:52:00 +03:00
|
|
|
|
|
2020-07-22 14:42:59 +03:00
|
|
|
|
def deserialize_meta(path: Path) -> None:
|
2020-05-27 15:48:54 +03:00
|
|
|
|
if path.exists():
|
|
|
|
|
data = srsly.read_json(path)
|
|
|
|
|
self.meta.update(data)
|
|
|
|
|
# self.meta always overrides meta["vectors"] with the metadata
|
|
|
|
|
# from self.vocab.vectors, so set the name directly
|
|
|
|
|
self.vocab.vectors.name = data.get("vectors", {}).get("name")
|
|
|
|
|
|
2020-07-22 14:42:59 +03:00
|
|
|
|
def deserialize_vocab(path: Path) -> None:
|
2020-05-27 15:48:54 +03:00
|
|
|
|
if path.exists():
|
|
|
|
|
self.vocab.from_disk(path)
|
|
|
|
|
_fix_pretrained_vectors_name(self)
|
|
|
|
|
|
2017-05-17 13:04:50 +03:00
|
|
|
|
path = util.ensure_path(path)
|
2019-12-22 03:53:56 +03:00
|
|
|
|
deserializers = {}
|
2020-02-27 20:42:27 +03:00
|
|
|
|
if Path(path / "config.cfg").exists():
|
|
|
|
|
deserializers["config.cfg"] = lambda p: self.config.from_disk(p)
|
2020-05-27 15:48:54 +03:00
|
|
|
|
deserializers["meta.json"] = deserialize_meta
|
|
|
|
|
deserializers["vocab"] = deserialize_vocab
|
2019-08-01 18:13:01 +03:00
|
|
|
|
deserializers["tokenizer"] = lambda p: self.tokenizer.from_disk(
|
|
|
|
|
p, exclude=["vocab"]
|
|
|
|
|
)
|
2017-10-07 01:25:54 +03:00
|
|
|
|
for name, proc in self.pipeline:
|
2019-03-10 21:16:45 +03:00
|
|
|
|
if name in exclude:
|
2017-05-31 14:42:39 +03:00
|
|
|
|
continue
|
💫 Tidy up and auto-format .py files (#2983)
<!--- Provide a general summary of your changes in the title. -->
## Description
- [x] Use [`black`](https://github.com/ambv/black) to auto-format all `.py` files.
- [x] Update flake8 config to exclude very large files (lemmatization tables etc.)
- [x] Update code to be compatible with flake8 rules
- [x] Fix various small bugs, inconsistencies and messy stuff in the language data
- [x] Update docs to explain new code style (`black`, `flake8`, when to use `# fmt: off` and `# fmt: on` and what `# noqa` means)
Once #2932 is merged, which auto-formats and tidies up the CLI, we'll be able to run `flake8 spacy` actually get meaningful results.
At the moment, the code style and linting isn't applied automatically, but I'm hoping that the new [GitHub Actions](https://github.com/features/actions) will let us auto-format pull requests and post comments with relevant linting information.
### Types of change
enhancement, code style
## Checklist
<!--- Before you submit the PR, go over this checklist and make sure you can
tick off all the boxes. [] -> [x] -->
- [x] I have submitted the spaCy Contributor Agreement.
- [x] I ran the tests, and all new and existing tests passed.
- [x] My changes don't require a change to the documentation, or if they do, I've added all required information.
2018-11-30 19:03:03 +03:00
|
|
|
|
if not hasattr(proc, "from_disk"):
|
2017-05-31 14:42:39 +03:00
|
|
|
|
continue
|
2019-08-01 18:13:01 +03:00
|
|
|
|
deserializers[name] = lambda p, proc=proc: proc.from_disk(
|
|
|
|
|
p, exclude=["vocab"]
|
|
|
|
|
)
|
2019-03-10 21:16:45 +03:00
|
|
|
|
if not (path / "vocab").exists() and "vocab" not in exclude:
|
|
|
|
|
# Convert to list here in case exclude is (default) tuple
|
|
|
|
|
exclude = list(exclude) + ["vocab"]
|
2017-06-01 15:38:35 +03:00
|
|
|
|
util.from_disk(path, deserializers, exclude)
|
2017-10-25 12:57:43 +03:00
|
|
|
|
self._path = path
|
2020-01-29 19:06:46 +03:00
|
|
|
|
self._link_components()
|
2017-05-31 14:42:39 +03:00
|
|
|
|
return self
|
2017-05-17 13:04:50 +03:00
|
|
|
|
|
2020-07-22 14:42:59 +03:00
|
|
|
|
def to_bytes(self, exclude: Iterable[str] = tuple()) -> bytes:
|
2017-05-17 13:04:50 +03:00
|
|
|
|
"""Serialize the current state to a binary string.
|
2016-12-18 18:54:52 +03:00
|
|
|
|
|
2019-03-10 21:16:45 +03:00
|
|
|
|
exclude (list): Names of components or serialization fields to exclude.
|
2017-05-19 00:57:38 +03:00
|
|
|
|
RETURNS (bytes): The serialized form of the `Language` object.
|
2019-03-10 21:16:45 +03:00
|
|
|
|
|
|
|
|
|
DOCS: https://spacy.io/api/language#to_bytes
|
2017-05-17 13:04:50 +03:00
|
|
|
|
"""
|
2019-12-22 03:53:56 +03:00
|
|
|
|
serializers = {}
|
2019-03-10 21:16:45 +03:00
|
|
|
|
serializers["vocab"] = lambda: self.vocab.to_bytes()
|
|
|
|
|
serializers["tokenizer"] = lambda: self.tokenizer.to_bytes(exclude=["vocab"])
|
|
|
|
|
serializers["meta.json"] = lambda: srsly.json_dumps(self.meta)
|
2020-02-27 20:42:27 +03:00
|
|
|
|
serializers["config.cfg"] = lambda: self.config.to_bytes()
|
2019-03-10 21:16:45 +03:00
|
|
|
|
for name, proc in self.pipeline:
|
|
|
|
|
if name in exclude:
|
2017-05-29 12:45:45 +03:00
|
|
|
|
continue
|
💫 Tidy up and auto-format .py files (#2983)
<!--- Provide a general summary of your changes in the title. -->
## Description
- [x] Use [`black`](https://github.com/ambv/black) to auto-format all `.py` files.
- [x] Update flake8 config to exclude very large files (lemmatization tables etc.)
- [x] Update code to be compatible with flake8 rules
- [x] Fix various small bugs, inconsistencies and messy stuff in the language data
- [x] Update docs to explain new code style (`black`, `flake8`, when to use `# fmt: off` and `# fmt: on` and what `# noqa` means)
Once #2932 is merged, which auto-formats and tidies up the CLI, we'll be able to run `flake8 spacy` actually get meaningful results.
At the moment, the code style and linting isn't applied automatically, but I'm hoping that the new [GitHub Actions](https://github.com/features/actions) will let us auto-format pull requests and post comments with relevant linting information.
### Types of change
enhancement, code style
## Checklist
<!--- Before you submit the PR, go over this checklist and make sure you can
tick off all the boxes. [] -> [x] -->
- [x] I have submitted the spaCy Contributor Agreement.
- [x] I ran the tests, and all new and existing tests passed.
- [x] My changes don't require a change to the documentation, or if they do, I've added all required information.
2018-11-30 19:03:03 +03:00
|
|
|
|
if not hasattr(proc, "to_bytes"):
|
2017-05-29 12:45:45 +03:00
|
|
|
|
continue
|
2019-03-10 21:16:45 +03:00
|
|
|
|
serializers[name] = lambda proc=proc: proc.to_bytes(exclude=["vocab"])
|
2017-10-17 19:18:10 +03:00
|
|
|
|
return util.to_bytes(serializers, exclude)
|
2017-04-15 13:05:47 +03:00
|
|
|
|
|
2020-07-22 14:42:59 +03:00
|
|
|
|
def from_bytes(
|
|
|
|
|
self, bytes_data: bytes, exclude: Iterable[str] = tuple()
|
|
|
|
|
) -> "Language":
|
2017-05-17 13:04:50 +03:00
|
|
|
|
"""Load state from a binary string.
|
|
|
|
|
|
2017-05-19 00:57:38 +03:00
|
|
|
|
bytes_data (bytes): The data to load from.
|
2019-03-10 21:16:45 +03:00
|
|
|
|
exclude (list): Names of components or serialization fields to exclude.
|
2017-05-19 00:57:38 +03:00
|
|
|
|
RETURNS (Language): The `Language` object.
|
2019-03-10 21:16:45 +03:00
|
|
|
|
|
|
|
|
|
DOCS: https://spacy.io/api/language#from_bytes
|
2017-05-17 13:04:50 +03:00
|
|
|
|
"""
|
2020-06-20 16:52:00 +03:00
|
|
|
|
|
2020-05-27 15:48:54 +03:00
|
|
|
|
def deserialize_meta(b):
|
|
|
|
|
data = srsly.json_loads(b)
|
|
|
|
|
self.meta.update(data)
|
|
|
|
|
# self.meta always overrides meta["vectors"] with the metadata
|
|
|
|
|
# from self.vocab.vectors, so set the name directly
|
|
|
|
|
self.vocab.vectors.name = data.get("vectors", {}).get("name")
|
|
|
|
|
|
|
|
|
|
def deserialize_vocab(b):
|
|
|
|
|
self.vocab.from_bytes(b)
|
|
|
|
|
_fix_pretrained_vectors_name(self)
|
|
|
|
|
|
2019-12-22 03:53:56 +03:00
|
|
|
|
deserializers = {}
|
2020-02-27 20:42:27 +03:00
|
|
|
|
deserializers["config.cfg"] = lambda b: self.config.from_bytes(b)
|
2020-05-27 15:48:54 +03:00
|
|
|
|
deserializers["meta.json"] = deserialize_meta
|
|
|
|
|
deserializers["vocab"] = deserialize_vocab
|
2019-08-01 18:13:01 +03:00
|
|
|
|
deserializers["tokenizer"] = lambda b: self.tokenizer.from_bytes(
|
|
|
|
|
b, exclude=["vocab"]
|
|
|
|
|
)
|
2019-03-10 21:16:45 +03:00
|
|
|
|
for name, proc in self.pipeline:
|
|
|
|
|
if name in exclude:
|
2017-05-29 12:45:45 +03:00
|
|
|
|
continue
|
💫 Tidy up and auto-format .py files (#2983)
<!--- Provide a general summary of your changes in the title. -->
## Description
- [x] Use [`black`](https://github.com/ambv/black) to auto-format all `.py` files.
- [x] Update flake8 config to exclude very large files (lemmatization tables etc.)
- [x] Update code to be compatible with flake8 rules
- [x] Fix various small bugs, inconsistencies and messy stuff in the language data
- [x] Update docs to explain new code style (`black`, `flake8`, when to use `# fmt: off` and `# fmt: on` and what `# noqa` means)
Once #2932 is merged, which auto-formats and tidies up the CLI, we'll be able to run `flake8 spacy` actually get meaningful results.
At the moment, the code style and linting isn't applied automatically, but I'm hoping that the new [GitHub Actions](https://github.com/features/actions) will let us auto-format pull requests and post comments with relevant linting information.
### Types of change
enhancement, code style
## Checklist
<!--- Before you submit the PR, go over this checklist and make sure you can
tick off all the boxes. [] -> [x] -->
- [x] I have submitted the spaCy Contributor Agreement.
- [x] I ran the tests, and all new and existing tests passed.
- [x] My changes don't require a change to the documentation, or if they do, I've added all required information.
2018-11-30 19:03:03 +03:00
|
|
|
|
if not hasattr(proc, "from_bytes"):
|
2017-05-29 12:45:45 +03:00
|
|
|
|
continue
|
2019-08-01 18:13:01 +03:00
|
|
|
|
deserializers[name] = lambda b, proc=proc: proc.from_bytes(
|
|
|
|
|
b, exclude=["vocab"]
|
|
|
|
|
)
|
2019-03-10 21:16:45 +03:00
|
|
|
|
util.from_bytes(bytes_data, deserializers, exclude)
|
2020-01-29 19:06:46 +03:00
|
|
|
|
self._link_components()
|
2017-05-17 13:04:50 +03:00
|
|
|
|
return self
|
2017-05-22 02:43:31 +03:00
|
|
|
|
|
2017-05-29 12:45:45 +03:00
|
|
|
|
|
2020-07-22 14:42:59 +03:00
|
|
|
|
@dataclass
|
|
|
|
|
class FactoryMeta:
|
2020-07-29 00:12:42 +03:00
|
|
|
|
"""Dataclass containing information about a component and its defaults
|
|
|
|
|
provided by the @Language.component or @Language.factory decorator. It's
|
|
|
|
|
created whenever a component is defined and stored on the Language class for
|
|
|
|
|
each component instance and factory instance.
|
|
|
|
|
"""
|
2020-07-22 14:42:59 +03:00
|
|
|
|
factory: str
|
|
|
|
|
default_config: Optional[Dict[str, Any]] = None # noqa: E704
|
|
|
|
|
assigns: Iterable[str] = tuple()
|
|
|
|
|
requires: Iterable[str] = tuple()
|
|
|
|
|
retokenizes: bool = False
|
2020-07-26 14:18:43 +03:00
|
|
|
|
scores: Iterable[str] = tuple()
|
2020-07-28 12:22:24 +03:00
|
|
|
|
default_score_weights: Optional[Dict[str, float]] = None # noqa: E704
|
2019-10-27 15:35:49 +03:00
|
|
|
|
|
|
|
|
|
|
2020-07-22 14:42:59 +03:00
|
|
|
|
def _get_config_overrides(
|
|
|
|
|
items: Dict[str, Any], prefix: str = "components"
|
|
|
|
|
) -> Tuple[Dict[str, Any], Dict[str, Any]]:
|
|
|
|
|
prefix = f"{prefix}."
|
|
|
|
|
non_pipe = {k: v for k, v in items.items() if not k.startswith(prefix)}
|
|
|
|
|
pipe = {k.replace(prefix, ""): v for k, v in items.items() if k.startswith(prefix)}
|
|
|
|
|
return non_pipe, pipe
|
2019-10-27 15:35:49 +03:00
|
|
|
|
|
2020-07-22 14:42:59 +03:00
|
|
|
|
|
|
|
|
|
def _fix_pretrained_vectors_name(nlp: Language) -> None:
|
2018-03-28 17:02:59 +03:00
|
|
|
|
# TODO: Replace this once we handle vectors consistently as static
|
|
|
|
|
# data
|
2020-05-27 15:48:54 +03:00
|
|
|
|
if "vectors" in nlp.meta and "name" in nlp.meta["vectors"]:
|
💫 Tidy up and auto-format .py files (#2983)
<!--- Provide a general summary of your changes in the title. -->
## Description
- [x] Use [`black`](https://github.com/ambv/black) to auto-format all `.py` files.
- [x] Update flake8 config to exclude very large files (lemmatization tables etc.)
- [x] Update code to be compatible with flake8 rules
- [x] Fix various small bugs, inconsistencies and messy stuff in the language data
- [x] Update docs to explain new code style (`black`, `flake8`, when to use `# fmt: off` and `# fmt: on` and what `# noqa` means)
Once #2932 is merged, which auto-formats and tidies up the CLI, we'll be able to run `flake8 spacy` actually get meaningful results.
At the moment, the code style and linting isn't applied automatically, but I'm hoping that the new [GitHub Actions](https://github.com/features/actions) will let us auto-format pull requests and post comments with relevant linting information.
### Types of change
enhancement, code style
## Checklist
<!--- Before you submit the PR, go over this checklist and make sure you can
tick off all the boxes. [] -> [x] -->
- [x] I have submitted the spaCy Contributor Agreement.
- [x] I ran the tests, and all new and existing tests passed.
- [x] My changes don't require a change to the documentation, or if they do, I've added all required information.
2018-11-30 19:03:03 +03:00
|
|
|
|
nlp.vocab.vectors.name = nlp.meta["vectors"]["name"]
|
2018-03-28 22:08:58 +03:00
|
|
|
|
elif not nlp.vocab.vectors.size:
|
|
|
|
|
nlp.vocab.vectors.name = None
|
💫 Tidy up and auto-format .py files (#2983)
<!--- Provide a general summary of your changes in the title. -->
## Description
- [x] Use [`black`](https://github.com/ambv/black) to auto-format all `.py` files.
- [x] Update flake8 config to exclude very large files (lemmatization tables etc.)
- [x] Update code to be compatible with flake8 rules
- [x] Fix various small bugs, inconsistencies and messy stuff in the language data
- [x] Update docs to explain new code style (`black`, `flake8`, when to use `# fmt: off` and `# fmt: on` and what `# noqa` means)
Once #2932 is merged, which auto-formats and tidies up the CLI, we'll be able to run `flake8 spacy` actually get meaningful results.
At the moment, the code style and linting isn't applied automatically, but I'm hoping that the new [GitHub Actions](https://github.com/features/actions) will let us auto-format pull requests and post comments with relevant linting information.
### Types of change
enhancement, code style
## Checklist
<!--- Before you submit the PR, go over this checklist and make sure you can
tick off all the boxes. [] -> [x] -->
- [x] I have submitted the spaCy Contributor Agreement.
- [x] I ran the tests, and all new and existing tests passed.
- [x] My changes don't require a change to the documentation, or if they do, I've added all required information.
2018-11-30 19:03:03 +03:00
|
|
|
|
elif "name" in nlp.meta and "lang" in nlp.meta:
|
2019-12-25 19:59:52 +03:00
|
|
|
|
vectors_name = f"{nlp.meta['lang']}_{nlp.meta['name']}.vectors"
|
2018-03-28 17:02:59 +03:00
|
|
|
|
nlp.vocab.vectors.name = vectors_name
|
|
|
|
|
else:
|
2018-04-03 22:40:29 +03:00
|
|
|
|
raise ValueError(Errors.E092)
|
2018-04-04 02:31:25 +03:00
|
|
|
|
if nlp.vocab.vectors.size != 0:
|
2020-06-20 23:49:37 +03:00
|
|
|
|
link_vectors_to_models(nlp.vocab)
|
2018-03-28 17:02:59 +03:00
|
|
|
|
for name, proc in nlp.pipeline:
|
💫 Tidy up and auto-format .py files (#2983)
<!--- Provide a general summary of your changes in the title. -->
## Description
- [x] Use [`black`](https://github.com/ambv/black) to auto-format all `.py` files.
- [x] Update flake8 config to exclude very large files (lemmatization tables etc.)
- [x] Update code to be compatible with flake8 rules
- [x] Fix various small bugs, inconsistencies and messy stuff in the language data
- [x] Update docs to explain new code style (`black`, `flake8`, when to use `# fmt: off` and `# fmt: on` and what `# noqa` means)
Once #2932 is merged, which auto-formats and tidies up the CLI, we'll be able to run `flake8 spacy` actually get meaningful results.
At the moment, the code style and linting isn't applied automatically, but I'm hoping that the new [GitHub Actions](https://github.com/features/actions) will let us auto-format pull requests and post comments with relevant linting information.
### Types of change
enhancement, code style
## Checklist
<!--- Before you submit the PR, go over this checklist and make sure you can
tick off all the boxes. [] -> [x] -->
- [x] I have submitted the spaCy Contributor Agreement.
- [x] I ran the tests, and all new and existing tests passed.
- [x] My changes don't require a change to the documentation, or if they do, I've added all required information.
2018-11-30 19:03:03 +03:00
|
|
|
|
if not hasattr(proc, "cfg"):
|
2018-03-28 17:02:59 +03:00
|
|
|
|
continue
|
💫 Tidy up and auto-format .py files (#2983)
<!--- Provide a general summary of your changes in the title. -->
## Description
- [x] Use [`black`](https://github.com/ambv/black) to auto-format all `.py` files.
- [x] Update flake8 config to exclude very large files (lemmatization tables etc.)
- [x] Update code to be compatible with flake8 rules
- [x] Fix various small bugs, inconsistencies and messy stuff in the language data
- [x] Update docs to explain new code style (`black`, `flake8`, when to use `# fmt: off` and `# fmt: on` and what `# noqa` means)
Once #2932 is merged, which auto-formats and tidies up the CLI, we'll be able to run `flake8 spacy` actually get meaningful results.
At the moment, the code style and linting isn't applied automatically, but I'm hoping that the new [GitHub Actions](https://github.com/features/actions) will let us auto-format pull requests and post comments with relevant linting information.
### Types of change
enhancement, code style
## Checklist
<!--- Before you submit the PR, go over this checklist and make sure you can
tick off all the boxes. [] -> [x] -->
- [x] I have submitted the spaCy Contributor Agreement.
- [x] I ran the tests, and all new and existing tests passed.
- [x] My changes don't require a change to the documentation, or if they do, I've added all required information.
2018-11-30 19:03:03 +03:00
|
|
|
|
proc.cfg.setdefault("deprecation_fixes", {})
|
|
|
|
|
proc.cfg["deprecation_fixes"]["vectors_name"] = nlp.vocab.vectors.name
|
2018-03-28 17:02:59 +03:00
|
|
|
|
|
2017-05-29 12:45:45 +03:00
|
|
|
|
|
2017-10-25 14:46:41 +03:00
|
|
|
|
class DisabledPipes(list):
|
2017-10-27 15:40:14 +03:00
|
|
|
|
"""Manager for temporary pipeline disabling."""
|
💫 Tidy up and auto-format .py files (#2983)
<!--- Provide a general summary of your changes in the title. -->
## Description
- [x] Use [`black`](https://github.com/ambv/black) to auto-format all `.py` files.
- [x] Update flake8 config to exclude very large files (lemmatization tables etc.)
- [x] Update code to be compatible with flake8 rules
- [x] Fix various small bugs, inconsistencies and messy stuff in the language data
- [x] Update docs to explain new code style (`black`, `flake8`, when to use `# fmt: off` and `# fmt: on` and what `# noqa` means)
Once #2932 is merged, which auto-formats and tidies up the CLI, we'll be able to run `flake8 spacy` actually get meaningful results.
At the moment, the code style and linting isn't applied automatically, but I'm hoping that the new [GitHub Actions](https://github.com/features/actions) will let us auto-format pull requests and post comments with relevant linting information.
### Types of change
enhancement, code style
## Checklist
<!--- Before you submit the PR, go over this checklist and make sure you can
tick off all the boxes. [] -> [x] -->
- [x] I have submitted the spaCy Contributor Agreement.
- [x] I ran the tests, and all new and existing tests passed.
- [x] My changes don't require a change to the documentation, or if they do, I've added all required information.
2018-11-30 19:03:03 +03:00
|
|
|
|
|
2020-07-29 00:12:42 +03:00
|
|
|
|
def __init__(self, nlp: Language, names: List[str]) -> None:
|
2017-10-25 14:46:41 +03:00
|
|
|
|
self.nlp = nlp
|
|
|
|
|
self.names = names
|
|
|
|
|
# Important! Not deep copy -- we just want the container (but we also
|
|
|
|
|
# want to support people providing arbitrarily typed nlp.pipeline
|
|
|
|
|
# objects.)
|
2017-10-27 22:07:59 +03:00
|
|
|
|
self.original_pipeline = copy(nlp.pipeline)
|
2020-07-22 14:42:59 +03:00
|
|
|
|
self.metas = {name: nlp.get_pipe_meta(name) for name in names}
|
|
|
|
|
self.configs = {name: nlp.get_pipe_config(name) for name in names}
|
2017-10-25 14:46:41 +03:00
|
|
|
|
list.__init__(self)
|
|
|
|
|
self.extend(nlp.remove_pipe(name) for name in names)
|
|
|
|
|
|
|
|
|
|
def __enter__(self):
|
2017-10-25 15:56:16 +03:00
|
|
|
|
return self
|
2017-10-25 14:46:41 +03:00
|
|
|
|
|
|
|
|
|
def __exit__(self, *args):
|
|
|
|
|
self.restore()
|
|
|
|
|
|
2020-07-22 14:42:59 +03:00
|
|
|
|
def restore(self) -> None:
|
💫 Tidy up and auto-format .py files (#2983)
<!--- Provide a general summary of your changes in the title. -->
## Description
- [x] Use [`black`](https://github.com/ambv/black) to auto-format all `.py` files.
- [x] Update flake8 config to exclude very large files (lemmatization tables etc.)
- [x] Update code to be compatible with flake8 rules
- [x] Fix various small bugs, inconsistencies and messy stuff in the language data
- [x] Update docs to explain new code style (`black`, `flake8`, when to use `# fmt: off` and `# fmt: on` and what `# noqa` means)
Once #2932 is merged, which auto-formats and tidies up the CLI, we'll be able to run `flake8 spacy` actually get meaningful results.
At the moment, the code style and linting isn't applied automatically, but I'm hoping that the new [GitHub Actions](https://github.com/features/actions) will let us auto-format pull requests and post comments with relevant linting information.
### Types of change
enhancement, code style
## Checklist
<!--- Before you submit the PR, go over this checklist and make sure you can
tick off all the boxes. [] -> [x] -->
- [x] I have submitted the spaCy Contributor Agreement.
- [x] I ran the tests, and all new and existing tests passed.
- [x] My changes don't require a change to the documentation, or if they do, I've added all required information.
2018-11-30 19:03:03 +03:00
|
|
|
|
"""Restore the pipeline to its state when DisabledPipes was created."""
|
2017-10-25 14:46:41 +03:00
|
|
|
|
current, self.nlp.pipeline = self.nlp.pipeline, self.original_pipeline
|
💫 Tidy up and auto-format .py files (#2983)
<!--- Provide a general summary of your changes in the title. -->
## Description
- [x] Use [`black`](https://github.com/ambv/black) to auto-format all `.py` files.
- [x] Update flake8 config to exclude very large files (lemmatization tables etc.)
- [x] Update code to be compatible with flake8 rules
- [x] Fix various small bugs, inconsistencies and messy stuff in the language data
- [x] Update docs to explain new code style (`black`, `flake8`, when to use `# fmt: off` and `# fmt: on` and what `# noqa` means)
Once #2932 is merged, which auto-formats and tidies up the CLI, we'll be able to run `flake8 spacy` actually get meaningful results.
At the moment, the code style and linting isn't applied automatically, but I'm hoping that the new [GitHub Actions](https://github.com/features/actions) will let us auto-format pull requests and post comments with relevant linting information.
### Types of change
enhancement, code style
## Checklist
<!--- Before you submit the PR, go over this checklist and make sure you can
tick off all the boxes. [] -> [x] -->
- [x] I have submitted the spaCy Contributor Agreement.
- [x] I ran the tests, and all new and existing tests passed.
- [x] My changes don't require a change to the documentation, or if they do, I've added all required information.
2018-11-30 19:03:03 +03:00
|
|
|
|
unexpected = [name for name, pipe in current if not self.nlp.has_pipe(name)]
|
2017-10-25 14:46:41 +03:00
|
|
|
|
if unexpected:
|
|
|
|
|
# Don't change the pipeline if we're raising an error.
|
|
|
|
|
self.nlp.pipeline = current
|
2018-04-03 16:50:31 +03:00
|
|
|
|
raise ValueError(Errors.E008.format(names=unexpected))
|
2020-07-22 14:42:59 +03:00
|
|
|
|
self.nlp._pipe_meta.update(self.metas)
|
|
|
|
|
self.nlp._pipe_configs.update(self.configs)
|
2017-10-25 14:46:41 +03:00
|
|
|
|
self[:] = []
|
|
|
|
|
|
|
|
|
|
|
2020-07-22 14:42:59 +03:00
|
|
|
|
def _pipe(
|
|
|
|
|
examples: Iterable[Example], proc: Callable[[Doc], Doc], kwargs: Dict[str, Any]
|
|
|
|
|
) -> Iterator[Example]:
|
2019-03-12 15:32:56 +03:00
|
|
|
|
# We added some args for pipe that __call__ doesn't expect.
|
|
|
|
|
kwargs = dict(kwargs)
|
2020-07-06 14:06:25 +03:00
|
|
|
|
for arg in ["batch_size"]:
|
2019-03-12 15:32:56 +03:00
|
|
|
|
if arg in kwargs:
|
|
|
|
|
kwargs.pop(arg)
|
2020-06-26 20:34:12 +03:00
|
|
|
|
for eg in examples:
|
|
|
|
|
eg = proc(eg, **kwargs)
|
|
|
|
|
yield eg
|
2019-10-08 13:20:55 +03:00
|
|
|
|
|
|
|
|
|
|
2020-07-22 14:42:59 +03:00
|
|
|
|
def _apply_pipes(
|
|
|
|
|
make_doc: Callable[[str], Doc],
|
|
|
|
|
pipes: Iterable[Callable[[Doc], Doc]],
|
|
|
|
|
receiver,
|
|
|
|
|
sender,
|
|
|
|
|
underscore_state: Tuple[dict, dict, dict],
|
|
|
|
|
) -> None:
|
2019-10-08 13:20:55 +03:00
|
|
|
|
"""Worker for Language.pipe
|
|
|
|
|
|
2020-07-22 14:42:59 +03:00
|
|
|
|
make_doc (Callable[[str,] Doc]): Function to create Doc from text.
|
|
|
|
|
pipes (Iterable[Callable[[Doc], Doc]]): The components to apply.
|
2019-10-18 12:33:38 +03:00
|
|
|
|
receiver (multiprocessing.Connection): Pipe to receive text. Usually
|
|
|
|
|
created by `multiprocessing.Pipe()`
|
|
|
|
|
sender (multiprocessing.Connection): Pipe to send doc. Usually created by
|
|
|
|
|
`multiprocessing.Pipe()`
|
2020-07-22 14:42:59 +03:00
|
|
|
|
underscore_state (Tuple[dict, dict, dict]): The data in the Underscore class
|
|
|
|
|
of the parent.
|
2019-10-08 13:20:55 +03:00
|
|
|
|
"""
|
2020-02-12 13:50:42 +03:00
|
|
|
|
Underscore.load_state(underscore_state)
|
2019-10-08 13:20:55 +03:00
|
|
|
|
while True:
|
2020-02-12 13:50:42 +03:00
|
|
|
|
texts = receiver.get()
|
2019-10-08 13:20:55 +03:00
|
|
|
|
docs = (make_doc(text) for text in texts)
|
|
|
|
|
for pipe in pipes:
|
|
|
|
|
docs = pipe(docs)
|
|
|
|
|
# Connection does not accept unpickable objects, so send list.
|
|
|
|
|
sender.send([doc.to_bytes() for doc in docs])
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class _Sender:
|
|
|
|
|
"""Util for sending data to multiprocessing workers in Language.pipe"""
|
|
|
|
|
|
2020-07-22 14:42:59 +03:00
|
|
|
|
def __init__(
|
|
|
|
|
self, data: Iterable[Any], queues: List[mp.Queue], chunk_size: int
|
|
|
|
|
) -> None:
|
2019-10-08 13:20:55 +03:00
|
|
|
|
self.data = iter(data)
|
|
|
|
|
self.queues = iter(cycle(queues))
|
|
|
|
|
self.chunk_size = chunk_size
|
|
|
|
|
self.count = 0
|
|
|
|
|
|
2020-07-22 14:42:59 +03:00
|
|
|
|
def send(self) -> None:
|
2019-10-08 13:20:55 +03:00
|
|
|
|
"""Send chunk_size items from self.data to channels."""
|
|
|
|
|
for item, q in itertools.islice(
|
|
|
|
|
zip(self.data, cycle(self.queues)), self.chunk_size
|
|
|
|
|
):
|
|
|
|
|
# cycle channels so that distribute the texts evenly
|
|
|
|
|
q.put(item)
|
|
|
|
|
|
2020-07-22 14:42:59 +03:00
|
|
|
|
def step(self) -> None:
|
|
|
|
|
"""Tell sender that comsumed one item. Data is sent to the workers after
|
|
|
|
|
every chunk_size calls.
|
|
|
|
|
"""
|
2019-10-08 13:20:55 +03:00
|
|
|
|
self.count += 1
|
|
|
|
|
if self.count >= self.chunk_size:
|
|
|
|
|
self.count = 0
|
|
|
|
|
self.send()
|