From 43b960c01b0c64e56859ad5eb304a5422af46516 Mon Sep 17 00:00:00 2001 From: Ines Montani Date: Wed, 22 Jul 2020 13:42:59 +0200 Subject: [PATCH] Refactor pipeline components, config and language data (#5759) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Update with WIP * Update with WIP * Update with pipeline serialization * Update types and pipe factories * Add deep merge, tidy up and add tests * Fix pipe creation from config * Don't validate default configs on load * Update spacy/language.py Co-authored-by: Ines Montani * Adjust factory/component meta error * Clean up factory args and remove defaults * Add test for failing empty dict defaults * Update pipeline handling and methods * provide KB as registry function instead of as object * small change in test to make functionality more clear * update example script for EL configuration * Fix typo * Simplify test * Simplify test * splitting pipes.pyx into separate files * moving default configs to each component file * fix batch_size type * removing default values from component constructors where possible (TODO: test 4725) * skip instead of xfail * Add test for config -> nlp with multiple instances * pipeline.pipes -> pipeline.pipe * Tidy up, document, remove kwargs * small cleanup/generalization for Tok2VecListener * use DEFAULT_UPSTREAM field * revert to avoid circular imports * Fix tests * Replace deprecated arg * Make model dirs require config * fix pickling of keyword-only arguments in constructor * WIP: clean up and integrate full config * Add helper to handle function args more reliably Now also includes keyword-only args * Fix config composition and serialization * Improve config debugging and add visual diff * Remove unused defaults and fix type * Remove pipeline and factories from meta * Update spacy/default_config.cfg Co-authored-by: Sofie Van Landeghem * Update spacy/default_config.cfg * small UX edits * avoid printing stack trace for debug CLI commands * Add support for language-specific factories * specify the section of the config which holds the model to debug * WIP: add Language.from_config * Update with language data refactor WIP * Auto-format * Add backwards-compat handling for Language.factories * Update morphologizer.pyx * Fix morphologizer * Update and simplify lemmatizers * Fix Japanese tests * Port over tagger changes * Fix Chinese and tests * Update to latest Thinc * WIP: xfail first Russian lemmatizer test * Fix component-specific overrides * fix nO for output layers in debug_model * Fix default value * Fix tests and don't pass objects in config * Fix deep merging * Fix lemma lookup data registry Only load the lookups if an entry is available in the registry (and if spacy-lookups-data is installed) * Add types * Add Vocab.from_config * Fix typo * Fix tests * Make config copying more elegant * Fix pipe analysis * Fix lemmatizers and is_base_form * WIP: move language defaults to config * Fix morphology type * Fix vocab * Remove comment * Update to latest Thinc * Add morph rules to config * Tidy up * Remove set_morphology option from tagger factory * Hack use_gpu * Move [pipeline] to top-level block and make [nlp.pipeline] list Allows separating component blocks from component order – otherwise, ordering the config would mean a changed component order, which is bad. Also allows initial config to define more components and not use all of them * Fix use_gpu and resume in CLI * Auto-format * Remove resume from config * Fix formatting and error * [pipeline] -> [components] * Fix types * Fix tagger test: requires set_morphology? Co-authored-by: Sofie Van Landeghem Co-authored-by: svlandeg Co-authored-by: Matthew Honnibal --- examples/training/train_entity_linker.py | 17 +- pyproject.toml | 2 +- requirements.txt | 4 +- setup.cfg | 6 +- setup.py | 8 +- spacy/__init__.py | 1 - spacy/cli/_util.py | 6 +- spacy/cli/debug_data.py | 82 +- spacy/cli/debug_model.py | 83 +- spacy/cli/evaluate.py | 5 +- spacy/cli/info.py | 1 - spacy/cli/package.py | 1 - spacy/cli/pretrain.py | 36 +- spacy/cli/profile.py | 2 +- spacy/cli/train.py | 175 +- spacy/default_config.cfg | 102 ++ spacy/errors.py | 108 +- spacy/gold/converters/conllu2docs.py | 5 +- spacy/lang/af/__init__.py | 21 +- spacy/lang/ar/__init__.py | 39 +- spacy/lang/bg/__init__.py | 21 +- spacy/lang/bn/__init__.py | 30 +- spacy/lang/ca/__init__.py | 42 +- spacy/lang/cs/__init__.py | 21 +- spacy/lang/da/__init__.py | 37 +- spacy/lang/de/__init__.py | 30 +- spacy/lang/el/__init__.py | 48 +- spacy/lang/el/lemmatizer.py | 10 +- spacy/lang/en/__init__.py | 79 +- spacy/lang/en/lemmatizer.py | 36 + spacy/lang/en/lex_attrs.py | 46 +- spacy/lang/es/__init__.py | 41 +- spacy/lang/et/__init__.py | 21 +- spacy/lang/eu/__init__.py | 30 +- spacy/lang/fa/__init__.py | 46 +- spacy/lang/fi/__init__.py | 34 +- spacy/lang/fr/__init__.py | 55 +- spacy/lang/fr/lemmatizer.py | 103 +- spacy/lang/ga/__init__.py | 24 +- spacy/lang/gu/__init__.py | 19 +- spacy/lang/he/__init__.py | 29 +- spacy/lang/hi/__init__.py | 29 +- spacy/lang/hr/__init__.py | 34 +- spacy/lang/hu/__init__.py | 34 +- spacy/lang/hy/__init__.py | 28 +- spacy/lang/id/__init__.py | 37 +- spacy/lang/is/__init__.py | 21 +- spacy/lang/it/__init__.py | 33 +- spacy/lang/ja/__init__.py | 327 ++-- spacy/lang/kn/__init__.py | 21 +- spacy/lang/ko/__init__.py | 93 +- spacy/lang/lb/__init__.py | 37 +- spacy/lang/lex_attrs.py | 94 +- spacy/lang/lij/__init__.py | 27 +- spacy/lang/lt/__init__.py | 42 +- spacy/lang/lv/__init__.py | 21 +- spacy/lang/ml/__init__.py | 19 +- spacy/lang/mr/__init__.py | 21 +- spacy/lang/nb/__init__.py | 34 +- spacy/lang/ne/__init__.py | 30 +- spacy/lang/nl/__init__.py | 52 +- spacy/lang/nl/lemmatizer.py | 16 +- spacy/lang/pl/__init__.py | 53 +- spacy/lang/pl/lemmatizer.py | 37 +- spacy/lang/pt/__init__.py | 37 +- spacy/lang/ro/__init__.py | 34 +- spacy/lang/ru/__init__.py | 45 +- spacy/lang/ru/lemmatizer.py | 32 +- spacy/lang/si/__init__.py | 29 +- spacy/lang/sk/__init__.py | 29 +- spacy/lang/sl/__init__.py | 21 +- spacy/lang/sq/__init__.py | 21 +- spacy/lang/sr/__init__.py | 36 +- spacy/lang/sv/__init__.py | 45 +- spacy/lang/ta/__init__.py | 29 +- spacy/lang/te/__init__.py | 29 +- spacy/lang/th/__init__.py | 61 +- spacy/lang/tl/__init__.py | 40 +- spacy/lang/tr/__init__.py | 34 +- spacy/lang/tt/__init__.py | 32 +- spacy/lang/uk/__init__.py | 51 +- spacy/lang/uk/lemmatizer.py | 32 +- spacy/lang/ur/__init__.py | 43 +- spacy/lang/vi/__init__.py | 67 +- spacy/lang/xx/__init__.py | 19 +- spacy/lang/yo/__init__.py | 28 +- spacy/lang/zh/__init__.py | 292 ++-- spacy/language.py | 1052 ++++++++---- spacy/lemmatizer.py | 86 +- spacy/lookups.py | 69 +- spacy/ml/models/entity_linker.py | 4 +- spacy/ml/models/textcat.py | 55 +- spacy/ml/models/tok2vec.py | 4 +- spacy/pipe_analysis.py | 113 +- spacy/pipeline/__init__.py | 31 +- spacy/pipeline/defaults/__init__.py | 93 - .../defaults/entity_linker_defaults.cfg | 13 - .../defaults/morphologizer_defaults.cfg | 14 - .../pipeline/defaults/multitask_defaults.cfg | 15 - spacy/pipeline/defaults/ner_defaults.cfg | 16 - spacy/pipeline/defaults/parser_defaults.cfg | 16 - spacy/pipeline/defaults/senter_defaults.cfg | 13 - .../pipeline/defaults/simple_ner_defaults.cfg | 13 - spacy/pipeline/defaults/tagger_defaults.cfg | 13 - .../defaults/textcat_bow_defaults.cfg | 5 - .../defaults/textcat_cnn_defaults.cfg | 14 - spacy/pipeline/defaults/textcat_defaults.cfg | 10 - spacy/pipeline/defaults/tok2vec_defaults.cfg | 10 - spacy/pipeline/dep_parser.pyx | 104 ++ spacy/pipeline/entity_linker.py | 366 ++++ spacy/pipeline/entityruler.py | 116 +- spacy/pipeline/functions.py | 16 +- spacy/pipeline/hooks.py | 8 +- spacy/pipeline/morphologizer.pyx | 75 +- spacy/pipeline/multitask.pyx | 224 +++ spacy/pipeline/ner.pyx | 90 + spacy/pipeline/pipe.pyx | 172 ++ spacy/pipeline/pipes.pyx | 1504 ----------------- spacy/pipeline/sentencizer.pyx | 173 ++ spacy/pipeline/senter.pyx | 151 ++ spacy/pipeline/simple_ner.py | 98 +- spacy/pipeline/tagger.pyx | 331 ++++ spacy/pipeline/textcat.py | 252 +++ spacy/pipeline/tok2vec.py | 83 +- spacy/schemas.py | 44 +- spacy/syntax/nn_parser.pxd | 5 +- spacy/syntax/nn_parser.pyx | 50 +- spacy/tests/conftest.py | 108 +- spacy/tests/doc/test_add_entities.py | 15 +- spacy/tests/doc/test_doc_api.py | 9 + .../tests/lang/fr/test_prefix_suffix_infix.py | 2 +- spacy/tests/lang/ja/test_serialize.py | 2 +- spacy/tests/lang/ja/test_tokenizer.py | 6 +- spacy/tests/lang/ru/test_lemmatizer.py | 1 + spacy/tests/lang/zh/test_tokenizer.py | 11 +- spacy/tests/parser/test_add_label.py | 32 +- spacy/tests/parser/test_arc_eager_oracle.py | 10 +- spacy/tests/parser/test_ner.py | 59 +- spacy/tests/parser/test_neural_parser.py | 22 +- spacy/tests/parser/test_parse.py | 3 +- spacy/tests/parser/test_preset_sbd.py | 10 +- spacy/tests/pipeline/test_analysis.py | 133 +- spacy/tests/pipeline/test_entity_linker.py | 70 +- spacy/tests/pipeline/test_entity_ruler.py | 47 +- spacy/tests/pipeline/test_factories.py | 47 - spacy/tests/pipeline/test_functions.py | 45 + spacy/tests/pipeline/test_morphologizer.py | 18 +- spacy/tests/pipeline/test_pipe_factories.py | 330 ++++ spacy/tests/pipeline/test_pipe_methods.py | 85 +- spacy/tests/pipeline/test_sentencizer.py | 13 +- spacy/tests/pipeline/test_senter.py | 7 +- spacy/tests/pipeline/test_tagger.py | 14 +- spacy/tests/pipeline/test_textcat.py | 22 +- spacy/tests/regression/test_issue1-1000.py | 16 +- spacy/tests/regression/test_issue1001-1500.py | 4 +- spacy/tests/regression/test_issue1501-2000.py | 33 +- spacy/tests/regression/test_issue2001-2500.py | 8 +- spacy/tests/regression/test_issue2501-3000.py | 6 +- spacy/tests/regression/test_issue3001-3500.py | 29 +- spacy/tests/regression/test_issue3501-4000.py | 49 +- spacy/tests/regression/test_issue4001-4500.py | 54 +- spacy/tests/regression/test_issue4501-5000.py | 60 +- spacy/tests/regression/test_issue5082.py | 10 +- spacy/tests/regression/test_issue5137.py | 18 +- spacy/tests/regression/test_issue5230.py | 18 +- spacy/tests/regression/test_issue5551.py | 14 +- .../tests/serialize/test_serialize_config.py | 178 +- .../serialize/test_serialize_pipeline.py | 73 +- .../serialize/test_serialize_tokenizer.py | 2 +- spacy/tests/test_gold.py | 79 +- spacy/tests/test_language.py | 21 +- spacy/tests/test_lemmatizer.py | 3 +- spacy/tests/test_misc.py | 19 + spacy/tests/test_models.py | 12 +- spacy/tests/tokenizer/test_explain.py | 2 +- spacy/tokenizer.pyx | 36 +- spacy/util.py | 363 ++-- spacy/vocab.pxd | 1 + spacy/vocab.pyx | 83 +- 179 files changed, 6946 insertions(+), 4619 deletions(-) create mode 100644 spacy/default_config.cfg create mode 100644 spacy/lang/en/lemmatizer.py delete mode 100644 spacy/pipeline/defaults/__init__.py delete mode 100644 spacy/pipeline/defaults/entity_linker_defaults.cfg delete mode 100644 spacy/pipeline/defaults/morphologizer_defaults.cfg delete mode 100644 spacy/pipeline/defaults/multitask_defaults.cfg delete mode 100644 spacy/pipeline/defaults/ner_defaults.cfg delete mode 100644 spacy/pipeline/defaults/parser_defaults.cfg delete mode 100644 spacy/pipeline/defaults/senter_defaults.cfg delete mode 100644 spacy/pipeline/defaults/simple_ner_defaults.cfg delete mode 100644 spacy/pipeline/defaults/tagger_defaults.cfg delete mode 100644 spacy/pipeline/defaults/textcat_bow_defaults.cfg delete mode 100644 spacy/pipeline/defaults/textcat_cnn_defaults.cfg delete mode 100644 spacy/pipeline/defaults/textcat_defaults.cfg delete mode 100644 spacy/pipeline/defaults/tok2vec_defaults.cfg create mode 100644 spacy/pipeline/dep_parser.pyx create mode 100644 spacy/pipeline/entity_linker.py create mode 100644 spacy/pipeline/multitask.pyx create mode 100644 spacy/pipeline/ner.pyx create mode 100644 spacy/pipeline/pipe.pyx delete mode 100644 spacy/pipeline/pipes.pyx create mode 100644 spacy/pipeline/sentencizer.pyx create mode 100644 spacy/pipeline/senter.pyx create mode 100644 spacy/pipeline/tagger.pyx create mode 100644 spacy/pipeline/textcat.py delete mode 100644 spacy/tests/pipeline/test_factories.py create mode 100644 spacy/tests/pipeline/test_pipe_factories.py diff --git a/examples/training/train_entity_linker.py b/examples/training/train_entity_linker.py index 22f4af596..8a69ae39c 100644 --- a/examples/training/train_entity_linker.py +++ b/examples/training/train_entity_linker.py @@ -17,7 +17,6 @@ import plac import random from pathlib import Path import spacy -from spacy.kb import KnowledgeBase from spacy.gold import Example from spacy.pipeline import EntityRuler @@ -82,12 +81,16 @@ def main(kb_path, vocab_path, output_dir=None, n_iter=50): # Create the Entity Linker component and add it to the pipeline. if "entity_linker" not in nlp.pipe_names: - kb = KnowledgeBase(vocab=nlp.vocab) - kb.load_bulk(kb_path) - print("Loaded Knowledge Base from '%s'" % kb_path) - - # use only the predicted EL score and not the prior probability (for demo purposes) - cfg = {"kb": kb, "incl_prior": False} + print("Loading Knowledge Base from '%s'" % kb_path) + cfg = { + "kb": { + "@assets": "spacy.KBFromFile.v1", + "vocab_path": vocab_path, + "kb_path": kb_path, + }, + # use only the predicted EL score and not the prior probability (for demo purposes) + "incl_prior": False, + } entity_linker = nlp.create_pipe("entity_linker", cfg) nlp.add_pipe(entity_linker, last=True) diff --git a/pyproject.toml b/pyproject.toml index 0e66ececf..a3e32ca15 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,7 +6,7 @@ requires = [ "cymem>=2.0.2,<2.1.0", "preshed>=3.0.2,<3.1.0", "murmurhash>=0.28.0,<1.1.0", - "thinc>=8.0.0a18,<8.0.0a20", + "thinc>=8.0.0a19,<8.0.0a30", "blis>=0.4.0,<0.5.0", "pytokenizations" ] diff --git a/requirements.txt b/requirements.txt index 7b6e0c9e9..089e4297d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,11 +1,11 @@ # Our libraries cymem>=2.0.2,<2.1.0 preshed>=3.0.2,<3.1.0 -thinc>=8.0.0a18,<8.0.0a20 +thinc>=8.0.0a19,<8.0.0a30 blis>=0.4.0,<0.5.0 ml_datasets>=0.1.1 murmurhash>=0.28.0,<1.1.0 -wasabi>=0.7.0,<1.1.0 +wasabi>=0.7.1,<1.1.0 srsly>=2.1.0,<3.0.0 catalogue>=0.0.7,<1.1.0 typer>=0.3.0,<0.4.0 diff --git a/setup.cfg b/setup.cfg index c7f5ce7f2..2abb1dcb8 100644 --- a/setup.cfg +++ b/setup.cfg @@ -34,15 +34,15 @@ setup_requires = cymem>=2.0.2,<2.1.0 preshed>=3.0.2,<3.1.0 murmurhash>=0.28.0,<1.1.0 - thinc>=8.0.0a18,<8.0.0a20 + thinc>=8.0.0a19,<8.0.0a30 install_requires = # Our libraries murmurhash>=0.28.0,<1.1.0 cymem>=2.0.2,<2.1.0 preshed>=3.0.2,<3.1.0 - thinc>=8.0.0a18,<8.0.0a20 + thinc>=8.0.0a19,<8.0.0a30 blis>=0.4.0,<0.5.0 - wasabi>=0.7.0,<1.1.0 + wasabi>=0.7.1,<1.1.0 srsly>=2.1.0,<3.0.0 catalogue>=0.0.7,<1.1.0 typer>=0.3.0,<0.4.0 diff --git a/setup.py b/setup.py index 7ce46a5bc..6d962ab59 100755 --- a/setup.py +++ b/setup.py @@ -32,8 +32,14 @@ MOD_NAMES = [ "spacy.attrs", "spacy.kb", "spacy.morphology", - "spacy.pipeline.pipes", + "spacy.pipeline.dep_parser", "spacy.pipeline.morphologizer", + "spacy.pipeline.multitask", + "spacy.pipeline.ner", + "spacy.pipeline.pipe", + "spacy.pipeline.sentencizer", + "spacy.pipeline.senter", + "spacy.pipeline.tagger", "spacy.syntax.stateclass", "spacy.syntax._state", "spacy.tokenizer", diff --git a/spacy/__init__.py b/spacy/__init__.py index b788b11ca..e9783b161 100644 --- a/spacy/__init__.py +++ b/spacy/__init__.py @@ -14,7 +14,6 @@ from .about import __version__ from .errors import Errors, Warnings from . import util from .util import registry -from .language import component if sys.maxunicode == 65535: diff --git a/spacy/cli/_util.py b/spacy/cli/_util.py index b87e07870..064d368d7 100644 --- a/spacy/cli/_util.py +++ b/spacy/cli/_util.py @@ -63,7 +63,7 @@ def parse_config_overrides(args: List[str]) -> Dict[str, Any]: result = {} while args: opt = args.pop(0) - err = f"Invalid config override '{opt}'" + err = f"Invalid CLI argument '{opt}'" if opt.startswith("--"): # new argument opt = opt.replace("--", "").replace("-", "_") if "." not in opt: @@ -73,7 +73,7 @@ def parse_config_overrides(args: List[str]) -> Dict[str, Any]: else: value = args.pop(0) # Just like we do in the config, we're calling json.loads on the - # values. But since they come from the CLI, it'd b unintuitive to + # values. But since they come from the CLI, it'd be unintuitive to # explicitly mark strings with escaped quotes. So we're working # around that here by falling back to a string if parsing fails. # TODO: improve logic to handle simple types like list of strings? @@ -82,7 +82,7 @@ def parse_config_overrides(args: List[str]) -> Dict[str, Any]: except ValueError: result[opt] = str(value) else: - msg.fail(f"{err}: options need to start with --", exits=1) + msg.fail(f"{err}: override option should start with --", exits=1) return result diff --git a/spacy/cli/debug_data.py b/spacy/cli/debug_data.py index a1928db5e..1ffceeca1 100644 --- a/spacy/cli/debug_data.py +++ b/spacy/cli/debug_data.py @@ -3,12 +3,12 @@ from pathlib import Path from collections import Counter import sys import srsly -from wasabi import Printer, MESSAGES, msg +from wasabi import Printer, MESSAGES, msg, diff_strings import typer +from thinc.api import Config from ._util import app, Arg, Opt, show_validation_error, parse_config_overrides from ._util import import_code, debug_cli -from ..schemas import ConfigSchema from ..gold import Corpus, Example from ..syntax import nonproj from ..language import Language @@ -33,6 +33,9 @@ def debug_config_cli( ctx: typer.Context, # This is only used to read additional arguments config_path: Path = Arg(..., help="Path to config file", exists=True), code_path: Optional[Path] = Opt(None, "--code-path", "-c", help="Path to Python file with additional code (registered functions) to be imported"), + output_path: Optional[Path] = Opt(None, "--output", "-o", help="Output path for filled config or '-' for standard output", allow_dash=True), + auto_fill: bool = Opt(False, "--auto-fill", "-F", help="Whether or not to auto-fill the config with built-in defaults if possible"), + diff: bool = Opt(False, "--diff", "-D", help="Show a visual diff if config was auto-filled") # fmt: on ): """Debug a config.cfg file and show validation errors. The command will @@ -40,14 +43,37 @@ def debug_config_cli( validation errors are blocking and will prevent the rest of the config from being resolved. This means that you may not see all validation errors at once and some issues are only shown once previous errors have been fixed. + Similar as with the 'train' command, you can override settings from the config + as command line options. For instance, --training.batch_size 128 overrides + the value of "batch_size" in the block "[training]". """ overrides = parse_config_overrides(ctx.args) import_code(code_path) with show_validation_error(): - util.load_config( - config_path, create_objects=False, schema=ConfigSchema, overrides=overrides, - ) - msg.good("Config is valid") + config = Config().from_disk(config_path) + try: + nlp, _ = util.load_model_from_config( + config, overrides=overrides, auto_fill=auto_fill + ) + except ValueError as e: + msg.fail(str(e), exits=1) + is_stdout = output_path is not None and str(output_path) == "-" + if auto_fill: + orig_config = config.to_str() + filled_config = nlp.config.to_str() + if orig_config == filled_config: + msg.good("Original config is valid, no values were auto-filled") + else: + msg.good("Auto-filled config is valid") + if diff: + print(diff_strings(config.to_str(), nlp.config.to_str())) + else: + msg.good("Original config is valid", show=not is_stdout) + if is_stdout: + print(nlp.config.to_str()) + elif output_path is not None: + nlp.config.to_disk(output_path) + msg.good(f"Saved updated config to {output_path}") @debug_cli.command( @@ -117,16 +143,13 @@ def debug_data( if not config_path.exists(): msg.fail("Config file not found", config_path, exists=1) with show_validation_error(): - config = util.load_config( - config_path, - create_objects=False, - schema=ConfigSchema, - overrides=config_overrides, - ) - nlp = util.load_model_from_config(config["nlp"]) + cfg = Config().from_disk(config_path) + nlp, config = util.load_model_from_config(cfg, overrides=config_overrides) + # TODO: handle base model lang = config["nlp"]["lang"] - base_model = config["nlp"]["base_model"] - pipeline = list(config["nlp"]["pipeline"].keys()) + base_model = config["training"]["base_model"] + pipeline = nlp.pipe_names + factory_names = [nlp.get_pipe_meta(pipe).factory for pipe in nlp.pipe_names] tag_map_path = util.ensure_path(config["training"]["tag_map"]) tag_map = {} if tag_map_path is not None: @@ -164,19 +187,17 @@ def debug_data( msg.good("Corpus is loadable") # Create all gold data here to avoid iterating over the train_dataset constantly - gold_train_data = _compile_gold(train_dataset, pipeline, nlp, make_proj=True) + gold_train_data = _compile_gold(train_dataset, factory_names, nlp, make_proj=True) gold_train_unpreprocessed_data = _compile_gold( - train_dataset, pipeline, nlp, make_proj=False + train_dataset, factory_names, nlp, make_proj=False ) - gold_dev_data = _compile_gold(dev_dataset, pipeline, nlp, make_proj=True) + gold_dev_data = _compile_gold(dev_dataset, factory_names, nlp, make_proj=True) train_texts = gold_train_data["texts"] dev_texts = gold_dev_data["texts"] msg.divider("Training stats") msg.text(f"Training pipeline: {', '.join(pipeline)}") - for pipe in [p for p in pipeline if p not in nlp.factories]: - msg.fail(f"Pipeline component '{pipe}' not available in factories") if base_model: msg.text(f"Starting with base model '{base_model}'") else: @@ -244,7 +265,7 @@ def debug_data( else: msg.info("No word vectors present in the model") - if "ner" in pipeline: + if "ner" in factory_names: # Get all unique NER labels present in the data labels = set( label for label in gold_train_data["ner"] if label not in ("O", "-", None) @@ -332,7 +353,7 @@ def debug_data( "with punctuation can not be trained with a noise level > 0." ) - if "textcat" in pipeline: + if "textcat" in factory_names: msg.divider("Text Classification") labels = [label for label in gold_train_data["cats"]] model_labels = _get_labels_from_model(nlp, "textcat") @@ -379,7 +400,7 @@ def debug_data( "contains only instances with mutually-exclusive classes." ) - if "tagger" in pipeline: + if "tagger" in factory_names: msg.divider("Part-of-speech Tagging") labels = [label for label in gold_train_data["tags"]] tag_map = nlp.vocab.morphology.tag_map @@ -394,7 +415,7 @@ def debug_data( for label in non_tagmap: msg.fail(f"Label '{label}' not found in tag map for language '{nlp.lang}'") - if "parser" in pipeline: + if "parser" in factory_names: has_low_data_warning = False msg.divider("Dependency Parsing") @@ -541,7 +562,10 @@ def _load_file(file_path: Path, msg: Printer) -> None: def _compile_gold( - examples: Sequence[Example], pipeline: List[str], nlp: Language, make_proj: bool + examples: Sequence[Example], + factory_names: List[str], + nlp: Language, + make_proj: bool, ) -> Dict[str, Any]: data = { "ner": Counter(), @@ -573,7 +597,7 @@ def _compile_gold( for word in valid_words: if nlp.vocab.strings[word] not in nlp.vocab.vectors: data["words_missing_vectors"].update([word]) - if "ner" in pipeline: + if "ner" in factory_names: for i, label in enumerate(eg.get_aligned_ner()): if label is None: continue @@ -595,14 +619,14 @@ def _compile_gold( data["ner"][combined_label] += 1 elif label == "-": data["ner"]["-"] += 1 - if "textcat" in pipeline: + if "textcat" in factory_names: data["cats"].update(gold.cats) if list(gold.cats.values()).count(1.0) != 1: data["n_cats_multilabel"] += 1 - if "tagger" in pipeline: + if "tagger" in factory_names: tags = eg.get_aligned("TAG", as_string=True) data["tags"].update([x for x in tags if x is not None]) - if "parser" in pipeline: + if "parser" in factory_names: aligned_heads, aligned_deps = eg.get_aligned_parse(projectivize=make_proj) data["deps"].update([x for x in aligned_deps if x is not None]) for i, (dep, head) in enumerate(zip(aligned_deps, aligned_heads)): diff --git a/spacy/cli/debug_model.py b/spacy/cli/debug_model.py index b205f6a19..3007d5de7 100644 --- a/spacy/cli/debug_model.py +++ b/spacy/cli/debug_model.py @@ -1,8 +1,11 @@ +from typing import Dict, Any, Optional from pathlib import Path from wasabi import msg -from thinc.api import require_gpu, fix_random_seed, set_dropout_rate, Adam +from thinc.api import require_gpu, fix_random_seed, set_dropout_rate, Adam, Config +from thinc.api import Model +import typer -from ._util import Arg, Opt, debug_cli +from ._util import Arg, Opt, debug_cli, show_validation_error, parse_config_overrides from .. import util from ..lang.en import English @@ -10,8 +13,10 @@ from ..lang.en import English @debug_cli.command("model") def debug_model_cli( # fmt: off + ctx: typer.Context, # This is only used to read additional arguments config_path: Path = Arg(..., help="Path to config file", exists=True), - layers: str = Opt("", "--layers", "-l", help="Comma-separated names of pipeline components to train"), + section: str = Arg(..., help="Section that defines the model to be analysed"), + layers: str = Opt("", "--layers", "-l", help="Comma-separated names of layer IDs to print"), dimensions: bool = Opt(False, "--dimensions", "-DIM", help="Show dimensions"), parameters: bool = Opt(False, "--parameters", "-PAR", help="Show parameters"), gradients: bool = Opt(False, "--gradients", "-GRAD", help="Show gradients"), @@ -20,14 +25,18 @@ def debug_model_cli( P1: bool = Opt(False, "--print-step1", "-P1", help="Print model after initialization"), P2: bool = Opt(False, "--print-step2", "-P2", help="Print model after training"), P3: bool = Opt(True, "--print-step3", "-P3", help="Print final predictions"), - use_gpu: int = Opt(-1, "--use-gpu", "-g", help="Use GPU"), - seed: int = Opt(None, "--seed", "-s", help="Use GPU"), + use_gpu: int = Opt(-1, "--use-gpu", "-g", help="GPU ID or -1 for CPU") # fmt: on ): """ Analyze a Thinc model implementation. Includes checks for internal structure and activations during training. """ + if use_gpu >= 0: + msg.info("Using GPU") + require_gpu(use_gpu) + else: + msg.info("Using CPU") print_settings = { "dimensions": dimensions, "parameters": parameters, @@ -39,27 +48,47 @@ def debug_model_cli( "print_after_training": P2, "print_prediction": P3, } - + config_overrides = parse_config_overrides(ctx.args) + cfg = Config().from_disk(config_path) + with show_validation_error(): + try: + _, config = util.load_model_from_config(cfg, overrides=config_overrides) + except ValueError as e: + msg.fail(str(e), exits=1) + seed = config["pretraining"]["seed"] if seed is not None: msg.info(f"Fixing random seed: {seed}") fix_random_seed(seed) - if use_gpu >= 0: - msg.info(f"Using GPU: {use_gpu}") - require_gpu(use_gpu) + + component = config + parts = section.split(".") + for item in parts: + try: + component = component[item] + except KeyError: + msg.fail( + f"The section '{section}' is not a valid section in the provided config.", + exits=1, + ) + if hasattr(component, "model"): + model = component.model else: - msg.info(f"Using CPU") - - debug_model( - config_path, print_settings=print_settings, - ) + msg.fail( + f"The section '{section}' does not specify an object that holds a Model.", + exits=1, + ) + debug_model(model, print_settings=print_settings) -def debug_model(config_path: Path, *, print_settings=None): +def debug_model(model: Model, *, print_settings: Optional[Dict[str, Any]] = None): + if not isinstance(model, Model): + msg.fail( + f"Requires a Thinc Model to be analysed, but found {type(model)} instead.", + exits=1, + ) if print_settings is None: print_settings = {} - model = util.load_config(config_path, create_objects=True)["model"] - # STEP 0: Printing before training msg.info(f"Analysing model with ID {model.id}") if print_settings.get("print_before_training"): @@ -67,7 +96,9 @@ def debug_model(config_path: Path, *, print_settings=None): _print_model(model, print_settings) # STEP 1: Initializing the model and printing again - model.initialize(X=_get_docs(), Y=_get_output(model.ops.xp)) + Y = _get_output(model.ops.xp) + _set_output_dim(nO=Y.shape[-1], model=model) + model.initialize(X=_get_docs(), Y=Y) if print_settings.get("print_after_init"): msg.info(f"After initialization:") _print_model(model, print_settings) @@ -110,12 +141,16 @@ def _get_docs(): def _get_output(xp): - return xp.asarray( - [ - xp.asarray([i + 10, i + 20, i + 30], dtype="float32") - for i, _ in enumerate(_get_docs()) - ] - ) + return xp.asarray([i + 10 for i, _ in enumerate(_get_docs())], dtype="float32") + + +def _set_output_dim(model, nO): + # the dim inference doesn't always work 100%, we need this hack like we have it in pipe.pyx + if model.has_dim("nO") is None: + model.set_dim("nO", nO) + if model.has_ref("output_layer"): + if model.get_ref("output_layer").has_dim("nO") is None: + model.get_ref("output_layer").set_dim("nO", nO) def _print_model(model, print_settings): diff --git a/spacy/cli/evaluate.py b/spacy/cli/evaluate.py index 09db389c4..de2e01818 100644 --- a/spacy/cli/evaluate.py +++ b/spacy/cli/evaluate.py @@ -105,9 +105,10 @@ def evaluate( print_textcats_auc_per_cat(msg, scorer.textcats_auc_per_cat) if displacy_path: + factory_names = [nlp.get_pipe_meta(pipe).factory for pipe in nlp.pipe_names] docs = [ex.predicted for ex in dev_dataset] - render_deps = "parser" in nlp.meta.get("pipeline", []) - render_ents = "ner" in nlp.meta.get("pipeline", []) + render_deps = "parser" in factory_names + render_ents = "ner" in factory_names render_parses( docs, displacy_path, diff --git a/spacy/cli/info.py b/spacy/cli/info.py index 98a1efeb8..ca2067edf 100644 --- a/spacy/cli/info.py +++ b/spacy/cli/info.py @@ -84,7 +84,6 @@ def info_model(model: str, *, silent: bool = True) -> Dict[str, Any]: msg.fail("Can't find model meta.json", meta_path, exits=1) meta = srsly.read_json(meta_path) if model_path.resolve() != model_path: - meta["link"] = str(model_path) meta["source"] = str(model_path.resolve()) else: meta["source"] = str(model_path) diff --git a/spacy/cli/package.py b/spacy/cli/package.py index 318c29950..82b9337a4 100644 --- a/spacy/cli/package.py +++ b/spacy/cli/package.py @@ -125,7 +125,6 @@ def get_meta( meta.update(existing_meta) nlp = util.load_model_from_path(Path(model_path)) meta["spacy_version"] = util.get_model_version_range(about.__version__) - meta["pipeline"] = nlp.pipe_names meta["vectors"] = { "width": nlp.vocab.vectors_length, "vectors": len(nlp.vocab.vectors), diff --git a/spacy/cli/pretrain.py b/spacy/cli/pretrain.py index c205bbefa..2b962c0bb 100644 --- a/spacy/cli/pretrain.py +++ b/spacy/cli/pretrain.py @@ -5,7 +5,7 @@ import time import re from collections import Counter from pathlib import Path -from thinc.api import use_pytorch_for_gpu_memory, require_gpu +from thinc.api import use_pytorch_for_gpu_memory, require_gpu, Config from thinc.api import set_dropout_rate, to_categorical, fix_random_seed from thinc.api import CosineDistance, L2Distance from wasabi import msg @@ -15,7 +15,6 @@ import typer from ._util import app, Arg, Opt, parse_config_overrides, show_validation_error from ._util import import_code -from ..schemas import ConfigSchema from ..errors import Errors from ..ml.models.multi_task import build_cloze_multi_task_model from ..ml.models.multi_task import build_cloze_characters_multi_task_model @@ -37,6 +36,7 @@ def pretrain_cli( code_path: Optional[Path] = Opt(None, "--code-path", "-c", help="Path to Python file with additional code (registered functions) to be imported"), resume_path: Optional[Path] = Opt(None, "--resume-path", "-r", help="Path to pretrained weights from which to resume pretraining"), epoch_resume: Optional[int] = Opt(None, "--epoch-resume", "-er", help="The epoch to resume counting from when using '--resume_path'. Prevents unintended overwriting of existing weight files."), + use_gpu: int = Opt(-1, "--use-gpu", "-g", help="GPU ID or -1 for CPU"), # fmt: on ): """ @@ -67,6 +67,7 @@ def pretrain_cli( config_overrides=overrides, resume_path=resume_path, epoch_resume=epoch_resume, + use_gpu=use_gpu, ) @@ -77,40 +78,29 @@ def pretrain( config_overrides: Dict[str, Any] = {}, resume_path: Optional[Path] = None, epoch_resume: Optional[int] = None, + use_gpu: int = -1, ): verify_cli_args(texts_loc, output_dir, config_path, resume_path, epoch_resume) - msg.info(f"Loading config from: {config_path}") - with show_validation_error(): - config = util.load_config( - config_path, - create_objects=False, - validate=True, - schema=ConfigSchema, - overrides=config_overrides, - ) - if not output_dir.exists(): - output_dir.mkdir() - msg.good(f"Created output directory: {output_dir}") - - use_gpu = config["training"]["use_gpu"] if use_gpu >= 0: msg.info("Using GPU") require_gpu(use_gpu) else: msg.info("Using CPU") - + msg.info(f"Loading config from: {config_path}") + config = Config().from_disk(config_path) + with show_validation_error(): + nlp, config = util.load_model_from_config(config, overrides=config_overrides) + # TODO: validate that [pretraining] block exists + if not output_dir.exists(): + output_dir.mkdir() + msg.good(f"Created output directory: {output_dir}") seed = config["pretraining"]["seed"] if seed is not None: fix_random_seed(seed) if use_gpu >= 0 and config["pretraining"]["use_pytorch_for_gpu_memory"]: use_pytorch_for_gpu_memory() - - nlp_config = config["nlp"] - srsly.write_json(output_dir / "config.json", config) + config.to_disk(output_dir / "config.cfg") msg.good("Saved config file in the output directory") - - config = util.load_config(config_path, create_objects=True) - nlp = util.load_model_from_config(nlp_config) pretrain_config = config["pretraining"] if texts_loc != "-": # reading from a file diff --git a/spacy/cli/profile.py b/spacy/cli/profile.py index 7c82f7e5b..14d8435fe 100644 --- a/spacy/cli/profile.py +++ b/spacy/cli/profile.py @@ -25,7 +25,7 @@ def profile_cli( # fmt: on ): """ - Profile a spaCy pipeline, to find out which functions take the most time. + Profile which functions take the most time in a spaCy pipeline. Input should be formatted as one JSON object per line with a key "text". It can either be provided as a JSONL file, or be read from sys.sytdin. If no input file is specified, the IMDB dataset is loaded via Thinc. diff --git a/spacy/cli/train.py b/spacy/cli/train.py index bb0a1d42a..6ff665368 100644 --- a/spacy/cli/train.py +++ b/spacy/cli/train.py @@ -1,4 +1,4 @@ -from typing import Optional, Dict, Any +from typing import Optional, Dict, Any, Tuple, Union, Callable, List from timeit import default_timer as timer import srsly import tqdm @@ -7,6 +7,7 @@ from wasabi import msg import thinc import thinc.schedules from thinc.api import use_pytorch_for_gpu_memory, require_gpu, fix_random_seed +from thinc.api import Config, Optimizer import random import typer @@ -14,18 +15,15 @@ from ._util import app, Arg, Opt, parse_config_overrides, show_validation_error from ._util import import_code from ..gold import Corpus, Example from ..lookups import Lookups +from ..language import Language from .. import util from ..errors import Errors -from ..schemas import ConfigSchema # Don't remove - required to load the built-in architectures from ..ml import models # noqa: F401 -registry = util.registry - - @app.command( "train", context_settings={"allow_extra_args": True, "ignore_unknown_options": True} ) @@ -38,6 +36,8 @@ def train_cli( output_path: Optional[Path] = Opt(None, "--output", "--output-path", "-o", help="Output directory to store model in"), code_path: Optional[Path] = Opt(None, "--code-path", "-c", help="Path to Python file with additional code (registered functions) to be imported"), verbose: bool = Opt(False, "--verbose", "-V", "-VV", help="Display more information for debugging purposes"), + use_gpu: int = Opt(-1, "--use-gpu", "-g", help="GPU ID or -1 for CPU"), + resume: bool = Opt(False, "--resume", "-R", help="Resume training"), # fmt: on ): """ @@ -53,9 +53,7 @@ def train_cli( referenced in the config. """ util.set_env_log(verbose) - verify_cli_args( - train_path=train_path, dev_path=dev_path, config_path=config_path, - ) + verify_cli_args(train_path, dev_path, config_path) overrides = parse_config_overrides(ctx.args) import_code(code_path) train( @@ -63,6 +61,8 @@ def train_cli( {"train": train_path, "dev": dev_path}, output_path=output_path, config_overrides=overrides, + use_gpu=use_gpu, + resume_training=resume, ) @@ -72,63 +72,53 @@ def train( raw_text: Optional[Path] = None, output_path: Optional[Path] = None, config_overrides: Dict[str, Any] = {}, + use_gpu: int = -1, + resume_training: bool = False, ) -> None: - msg.info(f"Loading config from: {config_path}") - # Read the config first without creating objects, to get to the original nlp_config - with show_validation_error(): - config = util.load_config( - config_path, - create_objects=False, - schema=ConfigSchema, - overrides=config_overrides, - ) - use_gpu = config["training"]["use_gpu"] if use_gpu >= 0: msg.info(f"Using GPU: {use_gpu}") require_gpu(use_gpu) else: msg.info("Using CPU") + msg.info(f"Loading config and nlp from: {config_path}") + config = Config().from_disk(config_path) + with show_validation_error(): + nlp, config = util.load_model_from_config(config, overrides=config_overrides) + if config["training"]["base_model"]: + base_nlp = util.load_model(config["training"]["base_model"]) + # TODO: do something to check base_nlp against regular nlp described in config? + nlp = base_nlp + verify_config(nlp) raw_text, tag_map, morph_rules, weights_data = load_from_paths(config) if config["training"]["seed"] is not None: fix_random_seed(config["training"]["seed"]) - if config["training"].get("use_pytorch_for_gpu_memory"): + if config["training"]["use_pytorch_for_gpu_memory"]: # It feels kind of weird to not have a default for this. use_pytorch_for_gpu_memory() - nlp_config = config["nlp"] - config = util.load_config( - config_path, - create_objects=True, - schema=ConfigSchema, - overrides=config_overrides, - ) training = config["training"] - msg.info("Creating nlp from config") - nlp = util.load_model_from_config(nlp_config) optimizer = training["optimizer"] limit = training["limit"] corpus = Corpus(data_paths["train"], data_paths["dev"], limit=limit) - if "textcat" in nlp_config["pipeline"]: - verify_textcat_config(nlp, nlp_config) - if training.get("resume", False): + if resume_training: msg.info("Resuming training") nlp.resume_training() else: msg.info(f"Initializing the nlp pipeline: {nlp.pipe_names}") - train_examples = list( - corpus.train_dataset( - nlp, - shuffle=False, - gold_preproc=training["gold_preproc"], - max_length=training["max_length"], - ) + train_examples = corpus.train_dataset( + nlp, + shuffle=False, + gold_preproc=training["gold_preproc"], + max_length=training["max_length"], ) + train_examples = list(train_examples) nlp.begin_training(lambda: train_examples) - # Replace tag map with provided mapping - nlp.vocab.morphology.load_tag_map(tag_map) - - # Load morph rules - nlp.vocab.morphology.load_morph_exceptions(morph_rules) + if tag_map: + # Replace tag map with provided mapping + nlp.vocab.morphology.load_tag_map(tag_map) + if morph_rules: + # Load morph rules + nlp.vocab.morphology.load_morph_exceptions(morph_rules) # Create empty extra lexeme tables so the data from spacy-lookups-data # isn't loaded if these features are accessed @@ -151,9 +141,8 @@ def train( for subpath in tok2vec_path.split("."): tok2vec = tok2vec.get(subpath) if not tok2vec: - msg.fail( - f"Could not locate the tok2vec model at {tok2vec_path}.", exits=1, - ) + err = f"Could not locate the tok2vec model at {tok2vec_path}" + msg.fail(err, exits=1) tok2vec.from_bytes(weights_data) msg.info("Loading training corpus") @@ -169,12 +158,11 @@ def train( evaluate, dropout=training["dropout"], accumulate_gradient=training["accumulate_gradient"], - patience=training.get("patience", 0), - max_steps=training.get("max_steps", 0), + patience=training["patience"], + max_steps=training["max_steps"], eval_frequency=training["eval_frequency"], raw_text=raw_text, ) - msg.info(f"Training. Initial learn rate: {optimizer.learn_rate}") print_row = setup_printer(training, nlp) @@ -209,8 +197,10 @@ def train( msg.good(f"Saved model to output directory {final_model_path}") -def create_train_batches(nlp, corpus, cfg): - max_epochs = cfg.get("max_epochs", 0) +def create_train_batches( + nlp: Language, corpus: Corpus, cfg: Union[Config, Dict[str, Any]] +): + max_epochs = cfg["max_epochs"] train_examples = list( corpus.train_dataset( nlp, @@ -219,9 +209,8 @@ def create_train_batches(nlp, corpus, cfg): max_length=cfg["max_length"], ) ) - epoch = 0 - batch_strategy = cfg.get("batch_by", "sequences") + batch_strategy = cfg["batch_by"] while True: if len(train_examples) == 0: raise ValueError(Errors.E988) @@ -241,7 +230,6 @@ def create_train_batches(nlp, corpus, cfg): ) else: batches = util.minibatch(train_examples, size=cfg["batch_size"]) - # make sure the minibatch_by_words result is not empty, or we'll have an infinite training loop try: first = next(batches) @@ -255,18 +243,20 @@ def create_train_batches(nlp, corpus, cfg): random.shuffle(train_examples) -def create_evaluation_callback(nlp, optimizer, corpus, cfg): - def evaluate(): - dev_examples = list( - corpus.dev_dataset( - nlp, gold_preproc=cfg["gold_preproc"], ignore_misaligned=True - ) +def create_evaluation_callback( + nlp: Language, + optimizer: Optimizer, + corpus: Corpus, + cfg: Union[Config, Dict[str, Any]], +) -> Callable[[], Tuple[float, Dict[str, float]]]: + def evaluate() -> Tuple[float, Dict[str, float]]: + dev_examples = corpus.dev_dataset( + nlp, gold_preproc=cfg["gold_preproc"], ignore_misaligned=True ) - + dev_examples = list(dev_examples) n_words = sum(len(ex.predicted) for ex in dev_examples) - batch_size = cfg.get("evaluation_batch_size", 128) + batch_size = cfg["eval_batch_size"] start_time = timer() - if optimizer.averages: with nlp.use_params(optimizer.averages): scorer = nlp.evaluate(dev_examples, batch_size=batch_size) @@ -280,12 +270,9 @@ def create_evaluation_callback(nlp, optimizer, corpus, cfg): try: weighted_score = sum(scores[s] * weights.get(s, 0.0) for s in weights) except KeyError as e: - raise KeyError( - Errors.E983.format( - dict="score_weights", key=str(e), keys=list(scores.keys()) - ) - ) - + keys = list(scores.keys()) + err = Errors.E983.format(dict="score_weights", key=str(e), keys=keys) + raise KeyError(err) scores["speed"] = wps return weighted_score, scores @@ -293,17 +280,17 @@ def create_evaluation_callback(nlp, optimizer, corpus, cfg): def train_while_improving( - nlp, - optimizer, + nlp: Language, + optimizer: Optimizer, train_data, evaluate, *, - dropout, - eval_frequency, - accumulate_gradient=1, - patience=0, - max_steps=0, - raw_text=None, + dropout: float, + eval_frequency: int, + accumulate_gradient: int, + patience: int, + max_steps: int, + raw_text: List[Dict[str, str]], ): """Train until an evaluation stops improving. Works as a generator, with each iteration yielding a tuple `(batch, info, is_best_checkpoint)`, @@ -414,7 +401,9 @@ def subdivide_batch(batch, accumulate_gradient): yield subbatch -def setup_printer(training, nlp): +def setup_printer( + training: Union[Dict[str, Any], Config], nlp: Language +) -> Callable[[Dict[str, Any]], None]: score_cols = training["scores"] score_widths = [max(len(col), 6) for col in score_cols] loss_cols = [f"Loss {pipe}" for pipe in nlp.pipe_names] @@ -423,11 +412,10 @@ def setup_printer(training, nlp): table_header = [col.upper() for col in table_header] table_widths = [3, 6] + loss_widths + score_widths + [6] table_aligns = ["r" for _ in table_widths] - msg.row(table_header, widths=table_widths) msg.row(["-" * width for width in table_widths]) - def print_row(info): + def print_row(info: Dict[str, Any]) -> None: try: losses = [ "{0:.2f}".format(float(info["losses"][pipe_name])) @@ -463,7 +451,9 @@ def setup_printer(training, nlp): return print_row -def update_meta(training, nlp, info): +def update_meta( + training: Union[Dict[str, Any], Config], nlp: Language, info: Dict[str, Any] +) -> None: score_cols = training["scores"] nlp.meta["performance"] = {} for metric in score_cols: @@ -472,7 +462,9 @@ def update_meta(training, nlp, info): nlp.meta["performance"][f"{pipe_name}_loss"] = info["losses"][pipe_name] -def load_from_paths(config): +def load_from_paths( + config: Config, +) -> Tuple[List[Dict[str, str]], Dict[str, dict], bytes]: # TODO: separate checks from loading raw_text = util.ensure_path(config["training"]["raw_text"]) if raw_text is not None: @@ -506,7 +498,7 @@ def verify_cli_args( dev_path: Path, config_path: Path, output_path: Optional[Path] = None, -): +) -> None: # Make sure all files and paths exists if they are needed if not config_path or not config_path.exists(): msg.fail("Config file not found", config_path, exits=1) @@ -528,12 +520,23 @@ def verify_cli_args( ) -def verify_textcat_config(nlp, nlp_config): +def verify_config(nlp: Language) -> None: + """Perform additional checks based on the config and loaded nlp object.""" + # TODO: maybe we should validate based on the actual components, the list + # in config["nlp"]["pipeline"] instead? + for pipe_config in nlp.config["components"].values(): + # We can't assume that the component name == the factory + factory = pipe_config["@factories"] + if factory == "textcat": + verify_textcat_config(nlp, pipe_config) + + +def verify_textcat_config(nlp: Language, pipe_config: Dict[str, Any]) -> None: # if 'positive_label' is provided: double check whether it's in the data and # the task is binary - if nlp_config["pipeline"]["textcat"].get("positive_label", None): + if pipe_config.get("positive_label"): textcat_labels = nlp.get_pipe("textcat").cfg.get("labels", []) - pos_label = nlp_config["pipeline"]["textcat"]["positive_label"] + pos_label = pipe_config.get("positive_label") if pos_label not in textcat_labels: msg.fail( f"The textcat's 'positive_label' config setting '{pos_label}' " diff --git a/spacy/default_config.cfg b/spacy/default_config.cfg new file mode 100644 index 000000000..7e6c7a6ec --- /dev/null +++ b/spacy/default_config.cfg @@ -0,0 +1,102 @@ +[nlp] +lang = null +stop_words = [] +lex_attr_getters = {} +pipeline = [] + +[nlp.tokenizer] +@tokenizers = "spacy.Tokenizer.v1" + +[nlp.lemmatizer] +@lemmatizers = "spacy.Lemmatizer.v1" + +[nlp.writing_system] +direction = "ltr" +has_case = true +has_letters = true + +[components] + +# Training hyper-parameters and additional features. +[training] +# Whether to train on sequences with 'gold standard' sentence boundaries +# and tokens. If you set this to true, take care to ensure your run-time +# data is passed in sentence-by-sentence via some prior preprocessing. +gold_preproc = false +# Limitations on training document length or number of examples. +max_length = 5000 +limit = 0 +# Data augmentation +orth_variant_level = 0.0 +dropout = 0.1 +# Controls early-stopping. 0 or -1 mean unlimited. +patience = 1600 +max_epochs = 0 +max_steps = 20000 +eval_frequency = 200 +eval_batch_size = 128 +# Other settings +seed = 0 +accumulate_gradient = 1 +use_pytorch_for_gpu_memory = false +# Control how scores are printed and checkpoints are evaluated. +scores = ["speed", "tags_acc", "uas", "las", "ents_f"] +score_weights = {"tags_acc": 0.2, "las": 0.4, "ents_f": 0.4} +# These settings are invalid for the transformer models. +init_tok2vec = null +discard_oversize = false +omit_extra_lookups = false +batch_by = "sequences" +raw_text = null +tag_map = null +morph_rules = null +base_model = null +vectors = null + +[training.batch_size] +@schedules = "compounding.v1" +start = 1000 +stop = 1000 +compound = 1.001 + +[training.optimizer] +@optimizers = "Adam.v1" +beta1 = 0.9 +beta2 = 0.999 +L2_is_weight_decay = true +L2 = 0.01 +grad_clip = 1.0 +use_averages = false +eps = 1e-8 + +[training.optimizer.learn_rate] +@schedules = "warmup_linear.v1" +warmup_steps = 250 +total_steps = 20000 +initial_rate = 0.001 + +[pretraining] +max_epochs = 1000 +min_length = 5 +max_length = 500 +dropout = 0.2 +n_save_every = null +batch_size = 3000 +seed = ${training:seed} +use_pytorch_for_gpu_memory = ${training:use_pytorch_for_gpu_memory} +tok2vec_model = "components.tok2vec.model" + +[pretraining.objective] +type = "characters" +n_characters = 4 + +[pretraining.optimizer] +@optimizers = "Adam.v1" +beta1 = 0.9 +beta2 = 0.999 +L2_is_weight_decay = true +L2 = 0.01 +grad_clip = 1.0 +use_averages = true +eps = 1e-8 +learn_rate = 0.001 diff --git a/spacy/errors.py b/spacy/errors.py index 4e3ca2a9b..f6c7a569f 100644 --- a/spacy/errors.py +++ b/spacy/errors.py @@ -124,20 +124,24 @@ class Warnings: @add_codes class Errors: E001 = ("No component '{name}' found in pipeline. Available names: {opts}") - E002 = ("Can't find factory for '{name}'. This usually happens when spaCy " - "calls `nlp.create_pipe` with a component name that's not built " - "in - for example, when constructing the pipeline from a model's " - "meta.json. If you're using a custom component, you can write to " - "`Language.factories['{name}']` or remove it from the model meta " - "and add it via `nlp.add_pipe` instead.") + E002 = ("Can't find factory for '{name}' for language {lang} ({lang_code}). " + "This usually happens when spaCy calls nlp.{method} with custom " + "component name that's not registered on the current language class. " + "If you're using a custom component, make sure you've added the " + "decorator @Language.component (for function components) or " + "@Language.factory (for class components).\n\nAvailable " + "factories: {opts}") E003 = ("Not a valid pipeline component. Expected callable, but " - "got {component} (name: '{name}').") - E004 = ("If you meant to add a built-in component, use `create_pipe`: " - "`nlp.add_pipe(nlp.create_pipe('{component}'))`") + "got {component} (name: '{name}'). If you're using a custom " + "component factory, double-check that it correctly returns your " + "initialized component.") + E004 = ("Can't set up pipeline component: a factory for '{name}' already exists.") E005 = ("Pipeline component '{name}' returned None. If you're using a " "custom component, maybe you forgot to return the processed Doc?") - E006 = ("Invalid constraints. You can only set one of the following: " - "before, after, first, last.") + E006 = ("Invalid constraints for adding pipeline component. You can only " + "set one of the following: before (component name or index), " + "after (component name or index), first (True) or last (True). " + "Invalid configuration: {args}. Existing components: {opts}") E007 = ("'{name}' already exists in pipeline. Existing names: {opts}") E008 = ("Some current components would be lost when restoring previous " "pipeline state. If you added components after calling " @@ -184,7 +188,7 @@ class Errors: "the documentation:\nhttps://spacy.io/usage/models") E030 = ("Sentence boundaries unset. You can add the 'sentencizer' " "component to the pipeline with: " - "nlp.add_pipe(nlp.create_pipe('sentencizer')). " + "nlp.add_pipe('sentencizer'). " "Alternatively, add the dependency parser, or set sentence " "boundaries by setting doc[i].is_sent_start.") E031 = ("Invalid token: empty string ('') at position {i}.") @@ -365,8 +369,6 @@ class Errors: E133 = ("The sum of prior probabilities for alias '{alias}' should not " "exceed 1, but found {sum}.") E134 = ("Entity '{entity}' is not defined in the Knowledge Base.") - E135 = ("If you meant to replace a built-in component, use `create_pipe`: " - "`nlp.replace_pipe('{name}', nlp.create_pipe('{name}'))`") E137 = ("Expected 'dict' type, but got '{type}' from '{line}'. Make sure " "to provide a valid JSON object as input with either the `text` " "or `tokens` key. For more info, see the docs:\n" @@ -484,6 +486,62 @@ class Errors: E199 = ("Unable to merge 0-length span at doc[{start}:{end}].") # TODO: fix numbering after merging develop into master + E956 = ("Can't find component '{name}' in [components] block in the config. " + "Available components: {opts}") + E957 = ("Writing directly to Language.factories isn't needed anymore in " + "spaCy v3. Instead, you can use the @Language.factory decorator " + "to register your custom component factory or @Language.component " + "to register a simple stateless function component that just takes " + "a Doc and returns it.") + E958 = ("Language code defined in config ({bad_lang_code}) does not match " + "language code of current Language subclass {lang} ({lang_code})") + E959 = ("Can't insert component {dir} index {idx}. Existing components: {opts}") + E960 = ("No config data found for component '{name}'. This is likely a bug " + "in spaCy.") + E961 = ("Found non-serializable Python object in config. Configs should " + "only include values that can be serialized to JSON. If you need " + "to pass models or other objects to your component, use a reference " + "to a registered function or initialize the object in your " + "component.\n\n{config}") + E962 = ("Received incorrect {style} for pipe '{name}'. Expected dict, " + "got: {cfg_type}.") + E963 = ("Can't read component info from @Language.{decorator} decorator. " + "Maybe you forgot to call it? Make sure you're using " + "@Language.{decorator}() instead of @Language.{decorator}.") + E964 = ("The pipeline component factory for '{name}' needs to have the " + "following named arguments, which are passed in by spaCy:\n- nlp: " + "receives the current nlp object and lets you access the vocab\n- " + "name: the name of the component instance, can be used to identify " + "the component, output losses etc.") + E965 = ("It looks like you're using the @Language.component decorator to " + "register '{name}' on a class instead of a function component. If " + "you need to register a class or function that *returns* a component " + "function, use the @Language.factory decorator instead.") + E966 = ("nlp.add_pipe now takes the string name of the registered component " + "factory, not a callable component. Expected string, but got " + "{component} (name: '{name}').\n\n- If you created your component " + "with nlp.create_pipe('name'): remove nlp.create_pipe and call " + "nlp.add_pipe('name') instead.\n\n- If you passed in a component " + "like TextCategorizer(): call nlp.add_pipe with the string name " + "instead, e.g. nlp.add_pipe('textcat').\n\n- If you're using a custom " + "component: Add the decorator @Language.component (for function " + "components) or @Language.factory (for class components / factories) " + "to your custom component and assign it a name, e.g. " + "@Language.component('your_name'). You can then run " + "nlp.add_pipe('your_name') to add it to the pipeline.") + E967 = ("No {meta} meta information found for '{name}'. This is likely a bug in spaCy.") + E968 = ("nlp.replace_pipe now takes the string name of the registered component " + "factory, not a callable component. Expected string, but got " + "{component}.\n\n- If you created your component with" + "with nlp.create_pipe('name'): remove nlp.create_pipe and call " + "nlp.replace_pipe('{name}', 'name') instead.\n\n- If you passed in a " + "component like TextCategorizer(): call nlp.replace_pipe with the " + "string name instead, e.g. nlp.replace_pipe('{name}', 'textcat').\n\n" + "- If you're using a custom component: Add the decorator " + "@Language.component (for function components) or @Language.factory " + "(for class components / factories) to your custom component and " + "assign it a name, e.g. @Language.component('your_name'). You can " + "then run nlp.replace_pipe('{name}', 'your_name').") E969 = ("Expected string values for field '{field}', but received {types} instead. ") E970 = ("Can not execute command '{str_command}'. Do you have '{tool}' installed?") E971 = ("Found incompatible lengths in Doc.from_array: {array_length} for the " @@ -506,10 +564,12 @@ class Errors: "into {values}, but found {value}.") E983 = ("Invalid key for '{dict}': {key}. Available keys: " "{keys}") - E985 = ("The pipeline component '{component}' is already available in the base " - "model. The settings in the component block in the config file are " - "being ignored. If you want to replace this component instead, set " - "'replace' to True in the training configuration.") + E984 = ("Invalid component config for '{name}': no @factories key " + "specifying the registered function used to initialize the " + "component. For example, @factories = \"ner\" will use the 'ner' " + "factory and all other settings in the block will be passed " + "to it as arguments.\n\n{config}") + E985 = ("Can't load model from config file: no 'nlp' section found.\n\n{config}") E986 = ("Could not create any training batches: check your input. " "Perhaps discard_oversize should be set to False ?") E987 = ("The text of an example training instance is either a Doc or " @@ -530,9 +590,9 @@ class Errors: E992 = ("The function `select_pipes` was called with `enable`={enable} " "and `disable`={disable} but that information is conflicting " "for the `nlp` pipeline with components {names}.") - E993 = ("The config for 'nlp' should include either a key 'name' to " - "refer to an existing model by name or path, or a key 'lang' " - "to create a new blank model.") + E993 = ("The config for 'nlp' needs to include a key 'lang' specifying " + "the code of the language to initialize it with (for example " + "'en' for English).\n\n{config}") E996 = ("Could not parse {file}: {msg}") E997 = ("Tokenizer special cases are not allowed to modify the text. " "This would map '{chunk}' to '{orth}' given token attributes " @@ -540,9 +600,9 @@ class Errors: E999 = ("Unable to merge the `Doc` objects because they do not all share " "the same `Vocab`.") E1000 = ("No pkuseg model available. Provide a pkuseg model when " - "initializing the pipeline: " - '`cfg = {"segmenter": "pkuseg", "pkuseg_model": name_or_path}; ' - 'nlp = Chinese(meta={"tokenizer": {"config": cfg}})`') + "initializing the pipeline:\n" + 'cfg = {"tokenizer": {"segmenter": "pkuseg", "pkuseg_model": name_or_path}}\m' + 'nlp = Chinese(config=cfg)') @add_codes diff --git a/spacy/gold/converters/conllu2docs.py b/spacy/gold/converters/conllu2docs.py index b591d3218..11ee86182 100644 --- a/spacy/gold/converters/conllu2docs.py +++ b/spacy/gold/converters/conllu2docs.py @@ -1,10 +1,9 @@ import re from .conll_ner2docs import n_sents_info -from ...gold import Example from ...gold import iob_to_biluo, spans_from_biluo_tags -from ...language import Language from ...tokens import Doc, Token, Span +from ...vocab import Vocab from wasabi import Printer @@ -73,7 +72,7 @@ def read_conllx( ner_map=None, ): """ Yield docs, one for each sentence """ - vocab = Language.Defaults.create_vocab() # need vocab to make a minimal Doc + vocab = Vocab() # need vocab to make a minimal Doc for sent in input_data.strip().split("\n\n"): lines = sent.strip().split("\n") if lines: diff --git a/spacy/lang/af/__init__.py b/spacy/lang/af/__init__.py index 0da123419..ee187ae5a 100644 --- a/spacy/lang/af/__init__.py +++ b/spacy/lang/af/__init__.py @@ -1,17 +1,26 @@ +from typing import Set +from thinc.api import Config + from .stop_words import STOP_WORDS from ...language import Language -from ...attrs import LANG +from ...util import registry -class AfrikaansDefaults(Language.Defaults): - lex_attr_getters = dict(Language.Defaults.lex_attr_getters) - lex_attr_getters[LANG] = lambda text: "af" - stop_words = STOP_WORDS +DEFAULT_CONFIG = """ +[nlp] +lang = "af" +stop_words = {"@language_data": "spacy.af.stop_words"} +""" + + +@registry.language_data("spacy.af.stop_words") +def stop_words() -> Set[str]: + return STOP_WORDS class Afrikaans(Language): lang = "af" - Defaults = AfrikaansDefaults + default_config = Config().from_str(DEFAULT_CONFIG) __all__ = ["Afrikaans"] diff --git a/spacy/lang/ar/__init__.py b/spacy/lang/ar/__init__.py index 6a1a8af3a..d46b18b6f 100644 --- a/spacy/lang/ar/__init__.py +++ b/spacy/lang/ar/__init__.py @@ -1,31 +1,48 @@ +from typing import Set, Dict, Callable, Any +from thinc.api import Config + from .stop_words import STOP_WORDS from .lex_attrs import LEX_ATTRS from .punctuation import TOKENIZER_SUFFIXES from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS from ..tokenizer_exceptions import BASE_EXCEPTIONS -from ..norm_exceptions import BASE_NORMS from ...language import Language -from ...attrs import LANG, NORM -from ...util import update_exc, add_lookups +from ...util import update_exc, registry + + +DEFAULT_CONFIG = """ +[nlp] +lang = "ar" +stop_words = {"@language_data": "spacy.ar.stop_words"} +lex_attr_getters = {"@language_data": "spacy.ar.lex_attr_getters"} + +[nlp.writing_system] +direction = "rtl" +has_case = false +has_letters = true +""" + + +@registry.language_data("spacy.ar.stop_words") +def stop_words() -> Set[str]: + return STOP_WORDS + + +@registry.language_data("spacy.ar.lex_attr_getters") +def lex_attr_getters() -> Dict[int, Callable[[str], Any]]: + return LEX_ATTRS class ArabicDefaults(Language.Defaults): - lex_attr_getters = dict(Language.Defaults.lex_attr_getters) - lex_attr_getters.update(LEX_ATTRS) - lex_attr_getters[LANG] = lambda text: "ar" - lex_attr_getters[NORM] = add_lookups( - Language.Defaults.lex_attr_getters[NORM], BASE_NORMS - ) tokenizer_exceptions = update_exc(BASE_EXCEPTIONS, TOKENIZER_EXCEPTIONS) - stop_words = STOP_WORDS suffixes = TOKENIZER_SUFFIXES - writing_system = {"direction": "rtl", "has_case": False, "has_letters": True} class Arabic(Language): lang = "ar" Defaults = ArabicDefaults + default_config = Config().from_str(DEFAULT_CONFIG) __all__ = ["Arabic"] diff --git a/spacy/lang/bg/__init__.py b/spacy/lang/bg/__init__.py index 437feb9ed..4a31a3653 100644 --- a/spacy/lang/bg/__init__.py +++ b/spacy/lang/bg/__init__.py @@ -1,17 +1,26 @@ +from typing import Set +from thinc.api import Config + from .stop_words import STOP_WORDS from ...language import Language -from ...attrs import LANG +from ...util import registry -class BulgarianDefaults(Language.Defaults): - lex_attr_getters = dict(Language.Defaults.lex_attr_getters) - lex_attr_getters[LANG] = lambda text: "bg" - stop_words = STOP_WORDS +DEFAULT_CONFIG = """ +[nlp] +lang = "bg" +stop_words = {"@language_data": "spacy.bg.stop_words"} +""" + + +@registry.language_data("spacy.bg.stop_words") +def stop_words() -> Set[str]: + return STOP_WORDS class Bulgarian(Language): lang = "bg" - Defaults = BulgarianDefaults + default_config = Config().from_str(DEFAULT_CONFIG) __all__ = ["Bulgarian"] diff --git a/spacy/lang/bn/__init__.py b/spacy/lang/bn/__init__.py index 399d64c73..2ac771537 100644 --- a/spacy/lang/bn/__init__.py +++ b/spacy/lang/bn/__init__.py @@ -1,18 +1,35 @@ +from typing import Set +from thinc.api import Config + from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS from .punctuation import TOKENIZER_PREFIXES, TOKENIZER_SUFFIXES, TOKENIZER_INFIXES from .stop_words import STOP_WORDS - from ..tokenizer_exceptions import BASE_EXCEPTIONS from ...language import Language -from ...attrs import LANG -from ...util import update_exc +from ...util import update_exc, registry + + +DEFAULT_CONFIG = """ +[nlp] +lang = "bn" +stop_words = {"@language_data": "spacy.bn.stop_words"} + +[nlp.lemmatizer] +@lemmatizers = "spacy.Lemmatizer.v1" + +[nlp.lemmatizer.data_paths] +@language_data = "spacy-lookups-data" +lang = ${nlp:lang} +""" + + +@registry.language_data("spacy.bn.stop_words") +def stop_words() -> Set[str]: + return STOP_WORDS class BengaliDefaults(Language.Defaults): - lex_attr_getters = dict(Language.Defaults.lex_attr_getters) - lex_attr_getters[LANG] = lambda text: "bn" tokenizer_exceptions = update_exc(BASE_EXCEPTIONS, TOKENIZER_EXCEPTIONS) - stop_words = STOP_WORDS prefixes = TOKENIZER_PREFIXES suffixes = TOKENIZER_SUFFIXES infixes = TOKENIZER_INFIXES @@ -21,6 +38,7 @@ class BengaliDefaults(Language.Defaults): class Bengali(Language): lang = "bn" Defaults = BengaliDefaults + default_config = Config().from_str(DEFAULT_CONFIG) __all__ = ["Bengali"] diff --git a/spacy/lang/ca/__init__.py b/spacy/lang/ca/__init__.py index a1ff2f2df..d2924e902 100644 --- a/spacy/lang/ca/__init__.py +++ b/spacy/lang/ca/__init__.py @@ -1,31 +1,49 @@ +from typing import Set, Dict, Callable, Any +from thinc.api import Config + from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS from .stop_words import STOP_WORDS from .lex_attrs import LEX_ATTRS - from ..tokenizer_exceptions import BASE_EXCEPTIONS -from ..norm_exceptions import BASE_NORMS from ...language import Language -from ...attrs import LANG, NORM -from ...util import update_exc, add_lookups - +from ...util import update_exc, registry from .punctuation import TOKENIZER_INFIXES +DEFAULT_CONFIG = """ +[nlp] +lang = "ca" +stop_words = {"@language_data": "spacy.ca.stop_words"} +lex_attr_getters = {"@language_data": "spacy.ca.lex_attr_getters"} + +[nlp.lemmatizer] +@lemmatizers = "spacy.Lemmatizer.v1" + +[nlp.lemmatizer.data_paths] +@language_data = "spacy-lookups-data" +lang = ${nlp:lang} +""" + + +@registry.language_data("spacy.ca.stop_words") +def stop_words() -> Set[str]: + return STOP_WORDS + + +@registry.language_data("spacy.ca.lex_attr_getters") +def lex_attr_getters() -> Dict[int, Callable[[str], Any]]: + return LEX_ATTRS + + class CatalanDefaults(Language.Defaults): - lex_attr_getters = dict(Language.Defaults.lex_attr_getters) - lex_attr_getters[LANG] = lambda text: "ca" - lex_attr_getters[NORM] = add_lookups( - Language.Defaults.lex_attr_getters[NORM], BASE_NORMS - ) - lex_attr_getters.update(LEX_ATTRS) tokenizer_exceptions = update_exc(BASE_EXCEPTIONS, TOKENIZER_EXCEPTIONS) - stop_words = STOP_WORDS infixes = TOKENIZER_INFIXES class Catalan(Language): lang = "ca" Defaults = CatalanDefaults + default_config = Config().from_str(DEFAULT_CONFIG) __all__ = ["Catalan"] diff --git a/spacy/lang/cs/__init__.py b/spacy/lang/cs/__init__.py index a27e3339d..f424c83fa 100644 --- a/spacy/lang/cs/__init__.py +++ b/spacy/lang/cs/__init__.py @@ -1,17 +1,26 @@ +from typing import Set +from thinc.api import Config + from .stop_words import STOP_WORDS from ...language import Language -from ...attrs import LANG +from ...util import registry -class CzechDefaults(Language.Defaults): - lex_attr_getters = dict(Language.Defaults.lex_attr_getters) - lex_attr_getters[LANG] = lambda text: "cs" - stop_words = STOP_WORDS +DEFAULT_CONFIG = """ +[nlp] +lang = "cs" +stop_words = {"@language_data": "spacy.cs.stop_words"} +""" + + +@registry.language_data("spacy.cs.stop_words") +def stop_words() -> Set[str]: + return STOP_WORDS class Czech(Language): lang = "cs" - Defaults = CzechDefaults + default_config = Config().from_str(DEFAULT_CONFIG) __all__ = ["Czech"] diff --git a/spacy/lang/da/__init__.py b/spacy/lang/da/__init__.py index 10f4e9afc..82ed5ed34 100644 --- a/spacy/lang/da/__init__.py +++ b/spacy/lang/da/__init__.py @@ -1,27 +1,50 @@ +from typing import Set, Dict, Callable, Any +from thinc.api import Config + from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS from .punctuation import TOKENIZER_INFIXES, TOKENIZER_SUFFIXES from .stop_words import STOP_WORDS from .lex_attrs import LEX_ATTRS - from ..tokenizer_exceptions import BASE_EXCEPTIONS from ...language import Language -from ...attrs import LANG -from ...util import update_exc +from ...util import update_exc, registry + + +DEFAULT_CONFIG = """ +[nlp] +lang = "da" +stop_words = {"@language_data": "spacy.da.stop_words"} +lex_attr_getters = {"@language_data": "spacy.da.lex_attr_getters"} + +[nlp.lemmatizer] +@lemmatizers = "spacy.Lemmatizer.v1" + +[nlp.lemmatizer.data_paths] +@language_data = "spacy-lookups-data" +lang = ${nlp:lang} +""" + + +@registry.language_data("spacy.da.stop_words") +def stop_words() -> Set[str]: + return STOP_WORDS + + +@registry.language_data("spacy.da.lex_attr_getters") +def lex_attr_getters() -> Dict[int, Callable[[str], Any]]: + return LEX_ATTRS class DanishDefaults(Language.Defaults): - lex_attr_getters = dict(Language.Defaults.lex_attr_getters) - lex_attr_getters.update(LEX_ATTRS) - lex_attr_getters[LANG] = lambda text: "da" tokenizer_exceptions = update_exc(BASE_EXCEPTIONS, TOKENIZER_EXCEPTIONS) infixes = TOKENIZER_INFIXES suffixes = TOKENIZER_SUFFIXES - stop_words = STOP_WORDS class Danish(Language): lang = "da" Defaults = DanishDefaults + default_config = Config().from_str(DEFAULT_CONFIG) __all__ = ["Danish"] diff --git a/spacy/lang/de/__init__.py b/spacy/lang/de/__init__.py index 825c3172f..a5c38bd39 100644 --- a/spacy/lang/de/__init__.py +++ b/spacy/lang/de/__init__.py @@ -1,23 +1,40 @@ +from typing import Set +from thinc.api import Config + from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS from .punctuation import TOKENIZER_PREFIXES, TOKENIZER_SUFFIXES from .punctuation import TOKENIZER_INFIXES from .stop_words import STOP_WORDS from .syntax_iterators import SYNTAX_ITERATORS - from ..tokenizer_exceptions import BASE_EXCEPTIONS from ...language import Language -from ...attrs import LANG -from ...util import update_exc +from ...util import update_exc, registry + + +DEFAULT_CONFIG = """ +[nlp] +lang = "de" +stop_words = {"@language_data": "spacy.de.stop_words"} + +[nlp.lemmatizer] +@lemmatizers = "spacy.Lemmatizer.v1" + +[nlp.lemmatizer.data_paths] +@language_data = "spacy-lookups-data" +lang = ${nlp:lang} +""" + + +@registry.language_data("spacy.de.stop_words") +def stop_words() -> Set[str]: + return STOP_WORDS class GermanDefaults(Language.Defaults): - lex_attr_getters = dict(Language.Defaults.lex_attr_getters) - lex_attr_getters[LANG] = lambda text: "de" tokenizer_exceptions = update_exc(BASE_EXCEPTIONS, TOKENIZER_EXCEPTIONS) prefixes = TOKENIZER_PREFIXES suffixes = TOKENIZER_SUFFIXES infixes = TOKENIZER_INFIXES - stop_words = STOP_WORDS syntax_iterators = SYNTAX_ITERATORS single_orth_variants = [ {"tags": ["$("], "variants": ["…", "..."]}, @@ -38,6 +55,7 @@ class GermanDefaults(Language.Defaults): class German(Language): lang = "de" Defaults = GermanDefaults + default_config = Config().from_str(DEFAULT_CONFIG) __all__ = ["German"] diff --git a/spacy/lang/el/__init__.py b/spacy/lang/el/__init__.py index 37b4f8f01..2fd8647fb 100644 --- a/spacy/lang/el/__init__.py +++ b/spacy/lang/el/__init__.py @@ -1,3 +1,6 @@ +from typing import Set, Dict, Callable, Any +from thinc.api import Config + from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS from .stop_words import STOP_WORDS from .lex_attrs import LEX_ATTRS @@ -6,32 +9,51 @@ from .syntax_iterators import SYNTAX_ITERATORS from .punctuation import TOKENIZER_PREFIXES, TOKENIZER_SUFFIXES, TOKENIZER_INFIXES from ..tokenizer_exceptions import BASE_EXCEPTIONS from ...language import Language -from ...lookups import Lookups -from ...attrs import LANG -from ...util import update_exc +from ...util import update_exc, registry + + +DEFAULT_CONFIG = """ +[nlp] +lang = "el" +stop_words = {"@language_data": "spacy.el.stop_words"} +lex_attr_getters = {"@language_data": "spacy.el.lex_attr_getters"} + +[nlp.lemmatizer] +@lemmatizers = "spacy.GreekLemmatizer.v1" + +[nlp.lemmatizer.data_paths] +@language_data = "spacy-lookups-data" +lang = ${nlp:lang} +""" + + +@registry.lemmatizers("spacy.GreekLemmatizer.v1") +def create_greek_lemmatizer(data_paths: dict = {}) -> GreekLemmatizer: + return GreekLemmatizer(data_paths=data_paths) + + +@registry.language_data("spacy.el.stop_words") +def stop_words() -> Set[str]: + return STOP_WORDS + + +@registry.language_data("spacy.el.lex_attr_getters") +def lex_attr_getters() -> Dict[int, Callable[[str], Any]]: + return LEX_ATTRS class GreekDefaults(Language.Defaults): - lex_attr_getters = dict(Language.Defaults.lex_attr_getters) - lex_attr_getters.update(LEX_ATTRS) - lex_attr_getters[LANG] = lambda text: "el" tokenizer_exceptions = update_exc(BASE_EXCEPTIONS, TOKENIZER_EXCEPTIONS) - stop_words = STOP_WORDS prefixes = TOKENIZER_PREFIXES suffixes = TOKENIZER_SUFFIXES infixes = TOKENIZER_INFIXES syntax_iterators = SYNTAX_ITERATORS - @classmethod - def create_lemmatizer(cls, nlp=None, lookups=None): - if lookups is None: - lookups = Lookups() - return GreekLemmatizer(lookups) - class Greek(Language): lang = "el" Defaults = GreekDefaults + default_config = Config().from_str(DEFAULT_CONFIG) __all__ = ["Greek"] diff --git a/spacy/lang/el/lemmatizer.py b/spacy/lang/el/lemmatizer.py index cf3a7fe97..809a23485 100644 --- a/spacy/lang/el/lemmatizer.py +++ b/spacy/lang/el/lemmatizer.py @@ -1,3 +1,5 @@ +from typing import Dict, List + from ...lemmatizer import Lemmatizer @@ -12,7 +14,13 @@ class GreekLemmatizer(Lemmatizer): not applicable for Greek language. """ - def lemmatize(self, string, index, exceptions, rules): + def lemmatize( + self, + string: str, + index: Dict[str, List[str]], + exceptions: Dict[str, Dict[str, List[str]]], + rules: Dict[str, List[List[str]]], + ) -> List[str]: string = string.lower() forms = [] if string in index: diff --git a/spacy/lang/en/__init__.py b/spacy/lang/en/__init__.py index 6aeda6108..4a69b2a41 100644 --- a/spacy/lang/en/__init__.py +++ b/spacy/lang/en/__init__.py @@ -1,25 +1,50 @@ +from typing import Set, Dict, Callable, Any +from thinc.api import Config + from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS from .stop_words import STOP_WORDS from .lex_attrs import LEX_ATTRS from .syntax_iterators import SYNTAX_ITERATORS - +from .lemmatizer import is_base_form from .punctuation import TOKENIZER_INFIXES from ..tokenizer_exceptions import BASE_EXCEPTIONS from ...language import Language -from ...attrs import LANG -from ...util import update_exc +from ...lemmatizer import Lemmatizer +from ...util import update_exc, registry -def _return_en(_): - return "en" +DEFAULT_CONFIG = """ +[nlp] +lang = "en" +stop_words = {"@language_data": "spacy.en.stop_words"} +lex_attr_getters = {"@language_data": "spacy.en.lex_attr_getters"} + +[nlp.lemmatizer] +@lemmatizers = "spacy.EnglishLemmatizer.v1" + +[nlp.lemmatizer.data_paths] +@language_data = "spacy-lookups-data" +lang = ${nlp:lang} +""" + + +@registry.language_data("spacy.en.stop_words") +def stop_words() -> Set[str]: + return STOP_WORDS + + +@registry.language_data("spacy.en.lex_attr_getters") +def lex_attr_getters() -> Dict[int, Callable[[str], Any]]: + return LEX_ATTRS + + +@registry.lemmatizers("spacy.EnglishLemmatizer.v1") +def create_lemmatizer(data_paths: dict = {}) -> "Lemmatizer": + return Lemmatizer(data_paths=data_paths, is_base_form=is_base_form) class EnglishDefaults(Language.Defaults): - lex_attr_getters = dict(Language.Defaults.lex_attr_getters) - lex_attr_getters.update(LEX_ATTRS) - lex_attr_getters[LANG] = _return_en tokenizer_exceptions = update_exc(BASE_EXCEPTIONS, TOKENIZER_EXCEPTIONS) - stop_words = STOP_WORDS syntax_iterators = SYNTAX_ITERATORS infixes = TOKENIZER_INFIXES single_orth_variants = [ @@ -31,45 +56,11 @@ class EnglishDefaults(Language.Defaults): {"tags": ["``", "''"], "variants": [('"', '"'), ("“", "”")]}, ] - @classmethod - def is_base_form(cls, univ_pos, morphology=None): - """ - Check whether we're dealing with an uninflected paradigm, so we can - avoid lemmatization entirely. - - univ_pos (unicode / int): The token's universal part-of-speech tag. - morphology (dict): The token's morphological features following the - Universal Dependencies scheme. - """ - if morphology is None: - morphology = {} - if univ_pos == "noun" and morphology.get("Number") == "sing": - return True - elif univ_pos == "verb" and morphology.get("VerbForm") == "inf": - return True - # This maps 'VBP' to base form -- probably just need 'IS_BASE' - # morphology - elif univ_pos == "verb" and ( - morphology.get("VerbForm") == "fin" - and morphology.get("Tense") == "pres" - and morphology.get("Number") is None - ): - return True - elif univ_pos == "adj" and morphology.get("Degree") == "pos": - return True - elif morphology.get("VerbForm") == "inf": - return True - elif morphology.get("VerbForm") == "none": - return True - elif morphology.get("Degree") == "pos": - return True - else: - return False - class English(Language): lang = "en" Defaults = EnglishDefaults + default_config = Config().from_str(DEFAULT_CONFIG) __all__ = ["English"] diff --git a/spacy/lang/en/lemmatizer.py b/spacy/lang/en/lemmatizer.py new file mode 100644 index 000000000..6d5db9e1e --- /dev/null +++ b/spacy/lang/en/lemmatizer.py @@ -0,0 +1,36 @@ +from typing import Optional + + +def is_base_form(univ_pos: str, morphology: Optional[dict] = None) -> bool: + """ + Check whether we're dealing with an uninflected paradigm, so we can + avoid lemmatization entirely. + + univ_pos (unicode / int): The token's universal part-of-speech tag. + morphology (dict): The token's morphological features following the + Universal Dependencies scheme. + """ + if morphology is None: + morphology = {} + if univ_pos == "noun" and morphology.get("Number") == "sing": + return True + elif univ_pos == "verb" and morphology.get("VerbForm") == "inf": + return True + # This maps 'VBP' to base form -- probably just need 'IS_BASE' + # morphology + elif univ_pos == "verb" and ( + morphology.get("VerbForm") == "fin" + and morphology.get("Tense") == "pres" + and morphology.get("Number") is None + ): + return True + elif univ_pos == "adj" and morphology.get("Degree") == "pos": + return True + elif morphology.get("VerbForm") == "inf": + return True + elif morphology.get("VerbForm") == "none": + return True + elif morphology.get("Degree") == "pos": + return True + else: + return False diff --git a/spacy/lang/en/lex_attrs.py b/spacy/lang/en/lex_attrs.py index 96fb4c9fa..975e6b392 100644 --- a/spacy/lang/en/lex_attrs.py +++ b/spacy/lang/en/lex_attrs.py @@ -1,47 +1,17 @@ from ...attrs import LIKE_NUM - +# fmt: off _num_words = [ - "zero", - "one", - "two", - "three", - "four", - "five", - "six", - "seven", - "eight", - "nine", - "ten", - "eleven", - "twelve", - "thirteen", - "fourteen", - "fifteen", - "sixteen", - "seventeen", - "eighteen", - "nineteen", - "twenty", - "thirty", - "forty", - "fifty", - "sixty", - "seventy", - "eighty", - "ninety", - "hundred", - "thousand", - "million", - "billion", - "trillion", - "quadrillion", - "gajillion", - "bazillion", + "zero", "one", "two", "three", "four", "five", "six", "seven", "eight", + "nine", "ten", "eleven", "twelve", "thirteen", "fourteen", "fifteen", + "sixteen", "seventeen", "eighteen", "nineteen", "twenty", "thirty", "forty", + "fifty", "sixty", "seventy", "eighty", "ninety", "hundred", "thousand", + "million", "billion", "trillion", "quadrillion", "gajillion", "bazillion" ] +# fmt: on -def like_num(text): +def like_num(text: str) -> bool: if text.startswith(("+", "-", "±", "~")): text = text[1:] text = text.replace(",", "").replace(".", "") diff --git a/spacy/lang/es/__init__.py b/spacy/lang/es/__init__.py index 10bade878..4425bfc01 100644 --- a/spacy/lang/es/__init__.py +++ b/spacy/lang/es/__init__.py @@ -1,33 +1,52 @@ +from typing import Set, Dict, Callable, Any +from thinc.config import Config + from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS from .stop_words import STOP_WORDS from .lex_attrs import LEX_ATTRS from .syntax_iterators import SYNTAX_ITERATORS from .punctuation import TOKENIZER_INFIXES, TOKENIZER_SUFFIXES - from ..tokenizer_exceptions import BASE_EXCEPTIONS -from ..norm_exceptions import BASE_NORMS from ...language import Language -from ...attrs import LANG, NORM -from ...util import update_exc, add_lookups +from ...util import update_exc, registry + + +DEFAULT_CONFIG = """ +[nlp] +lang = "es" +stop_words = {"@language_data": "spacy.es.stop_words"} +lex_attr_getters = {"@language_data": "spacy.es.lex_attr_getters"} + +[nlp.lemmatizer] +@lemmatizers = "spacy.Lemmatizer.v1" + +[nlp.lemmatizer.data_paths] +@language_data = "spacy-lookups-data" +lang = ${nlp:lang} +""" + + +@registry.language_data("spacy.es.stop_words") +def stop_words() -> Set[str]: + return STOP_WORDS + + +@registry.language_data("spacy.es.lex_attr_getters") +def lex_attr_getters() -> Dict[int, Callable[[str], Any]]: + return LEX_ATTRS class SpanishDefaults(Language.Defaults): - lex_attr_getters = dict(Language.Defaults.lex_attr_getters) - lex_attr_getters.update(LEX_ATTRS) - lex_attr_getters[LANG] = lambda text: "es" - lex_attr_getters[NORM] = add_lookups( - Language.Defaults.lex_attr_getters[NORM], BASE_NORMS - ) tokenizer_exceptions = update_exc(BASE_EXCEPTIONS, TOKENIZER_EXCEPTIONS) infixes = TOKENIZER_INFIXES suffixes = TOKENIZER_SUFFIXES - stop_words = STOP_WORDS syntax_iterators = SYNTAX_ITERATORS class Spanish(Language): lang = "es" Defaults = SpanishDefaults + default_config = Config().from_str(DEFAULT_CONFIG) __all__ = ["Spanish"] diff --git a/spacy/lang/et/__init__.py b/spacy/lang/et/__init__.py index e0b0a8a87..38da9ab1e 100644 --- a/spacy/lang/et/__init__.py +++ b/spacy/lang/et/__init__.py @@ -1,17 +1,26 @@ +from typing import Set +from thinc.api import Config + from .stop_words import STOP_WORDS from ...language import Language -from ...attrs import LANG +from ...util import registry -class EstonianDefaults(Language.Defaults): - lex_attr_getters = dict(Language.Defaults.lex_attr_getters) - lex_attr_getters[LANG] = lambda text: "et" - stop_words = STOP_WORDS +DEFAULT_CONFIG = """ +[nlp] +lang = "et" +stop_words = {"@language_data": "spacy.et.stop_words"} +""" + + +@registry.language_data("spacy.et.stop_words") +def stop_words() -> Set[str]: + return STOP_WORDS class Estonian(Language): lang = "et" - Defaults = EstonianDefaults + default_config = Config().from_str(DEFAULT_CONFIG) __all__ = ["Estonian"] diff --git a/spacy/lang/eu/__init__.py b/spacy/lang/eu/__init__.py index 0ad0b7ccc..4df50bca5 100644 --- a/spacy/lang/eu/__init__.py +++ b/spacy/lang/eu/__init__.py @@ -1,25 +1,41 @@ +from typing import Set, Dict, Callable, Any +from thinc.api import Config + from .stop_words import STOP_WORDS from .lex_attrs import LEX_ATTRS from .punctuation import TOKENIZER_SUFFIXES - from ..tokenizer_exceptions import BASE_EXCEPTIONS from ...language import Language -from ...attrs import LANG +from ...util import registry + + +DEFAULT_CONFIG = """ +[nlp] +lang = "eu" +stop_words = {"@language_data": "spacy.eu.stop_words"} +lex_attr_getters = {"@language_data": "spacy.eu.lex_attr_getters"} +""" + + +@registry.language_data("spacy.eu.stop_words") +def stop_words() -> Set[str]: + return STOP_WORDS + + +@registry.language_data("spacy.eu.lex_attr_getters") +def lex_attr_getters() -> Dict[int, Callable[[str], Any]]: + return LEX_ATTRS class BasqueDefaults(Language.Defaults): - lex_attr_getters = dict(Language.Defaults.lex_attr_getters) - lex_attr_getters.update(LEX_ATTRS) - lex_attr_getters[LANG] = lambda text: "eu" - tokenizer_exceptions = BASE_EXCEPTIONS - stop_words = STOP_WORDS suffixes = TOKENIZER_SUFFIXES class Basque(Language): lang = "eu" Defaults = BasqueDefaults + default_config = Config().from_str(DEFAULT_CONFIG) __all__ = ["Basque"] diff --git a/spacy/lang/fa/__init__.py b/spacy/lang/fa/__init__.py index ab3f6b358..085f400a4 100644 --- a/spacy/lang/fa/__init__.py +++ b/spacy/lang/fa/__init__.py @@ -1,7 +1,8 @@ +from typing import Set, Dict, Callable, Any +from thinc.api import Config + from ...language import Language -from ...attrs import LANG, NORM -from ...util import update_exc, add_lookups -from ..norm_exceptions import BASE_NORMS +from ...util import update_exc, registry from .stop_words import STOP_WORDS from .lex_attrs import LEX_ATTRS from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS @@ -9,23 +10,46 @@ from .punctuation import TOKENIZER_SUFFIXES from .syntax_iterators import SYNTAX_ITERATORS +DEFAULT_CONFIG = """ +[nlp] +lang = "fa" +stop_words = {"@language_data": "spacy.fa.stop_words"} +lex_attr_getters = {"@language_data": "spacy.fa.lex_attr_getters"} + +[nlp.writing_system] +direction = "rtl" +has_case = false +has_letters = true + +[nlp.lemmatizer] +@lemmatizers = "spacy.Lemmatizer.v1" + +[nlp.lemmatizer.data_paths] +@language_data = "spacy-lookups-data" +lang = ${nlp:lang} +""" + + +@registry.language_data("spacy.fa.stop_words") +def stop_words() -> Set[str]: + return STOP_WORDS + + +@registry.language_data("spacy.fa.lex_attr_getters") +def lex_attr_getters() -> Dict[int, Callable[[str], Any]]: + return LEX_ATTRS + + class PersianDefaults(Language.Defaults): - lex_attr_getters = dict(Language.Defaults.lex_attr_getters) - lex_attr_getters.update(LEX_ATTRS) - lex_attr_getters[NORM] = add_lookups( - Language.Defaults.lex_attr_getters[NORM], BASE_NORMS - ) - lex_attr_getters[LANG] = lambda text: "fa" tokenizer_exceptions = update_exc(TOKENIZER_EXCEPTIONS) - stop_words = STOP_WORDS suffixes = TOKENIZER_SUFFIXES - writing_system = {"direction": "rtl", "has_case": False, "has_letters": True} syntax_iterators = SYNTAX_ITERATORS class Persian(Language): lang = "fa" Defaults = PersianDefaults + default_config = Config().from_str(DEFAULT_CONFIG) __all__ = ["Persian"] diff --git a/spacy/lang/fi/__init__.py b/spacy/lang/fi/__init__.py index db58ad3ba..69a6412f0 100644 --- a/spacy/lang/fi/__init__.py +++ b/spacy/lang/fi/__init__.py @@ -1,31 +1,43 @@ +from typing import Set, Dict, Callable, Any +from thinc.api import Config + from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS from .stop_words import STOP_WORDS from .lex_attrs import LEX_ATTRS from .punctuation import TOKENIZER_INFIXES, TOKENIZER_SUFFIXES - from ..tokenizer_exceptions import BASE_EXCEPTIONS -from ..norm_exceptions import BASE_NORMS from ...language import Language -from ...attrs import LANG, NORM -from ...util import update_exc, add_lookups +from ...util import update_exc, registry + + +DEFAULT_CONFIG = """ +[nlp] +lang = "fi" +stop_words = {"@language_data": "spacy.fi.stop_words"} +lex_attr_getters = {"@language_data": "spacy.fi.lex_attr_getters"} +""" + + +@registry.language_data("spacy.fi.stop_words") +def stop_words() -> Set[str]: + return STOP_WORDS + + +@registry.language_data("spacy.fi.lex_attr_getters") +def lex_attr_getters() -> Dict[int, Callable[[str], Any]]: + return LEX_ATTRS class FinnishDefaults(Language.Defaults): - lex_attr_getters = dict(Language.Defaults.lex_attr_getters) - lex_attr_getters.update(LEX_ATTRS) - lex_attr_getters[LANG] = lambda text: "fi" - lex_attr_getters[NORM] = add_lookups( - Language.Defaults.lex_attr_getters[NORM], BASE_NORMS - ) infixes = TOKENIZER_INFIXES suffixes = TOKENIZER_SUFFIXES tokenizer_exceptions = update_exc(BASE_EXCEPTIONS, TOKENIZER_EXCEPTIONS) - stop_words = STOP_WORDS class Finnish(Language): lang = "fi" Defaults = FinnishDefaults + default_config = Config().from_str(DEFAULT_CONFIG) __all__ = ["Finnish"] diff --git a/spacy/lang/fr/__init__.py b/spacy/lang/fr/__init__.py index a2813ec80..8140a21b6 100644 --- a/spacy/lang/fr/__init__.py +++ b/spacy/lang/fr/__init__.py @@ -1,44 +1,61 @@ +from typing import Set, Dict, Callable, Any +from thinc.api import Config + from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS, TOKEN_MATCH from .punctuation import TOKENIZER_PREFIXES, TOKENIZER_INFIXES from .punctuation import TOKENIZER_SUFFIXES from .stop_words import STOP_WORDS from .lex_attrs import LEX_ATTRS -from .lemmatizer import FrenchLemmatizer +from .lemmatizer import FrenchLemmatizer, is_base_form from .syntax_iterators import SYNTAX_ITERATORS - from ..tokenizer_exceptions import BASE_EXCEPTIONS -from ..norm_exceptions import BASE_NORMS from ...language import Language -from ...lookups import Lookups -from ...attrs import LANG, NORM -from ...util import update_exc, add_lookups +from ...util import update_exc, registry + + +DEFAULT_CONFIG = """ +[nlp] +lang = "fr" +stop_words = {"@language_data": "spacy.fr.stop_words"} +lex_attr_getters = {"@language_data": "spacy.fr.lex_attr_getters"} + +[nlp.lemmatizer] +@lemmatizers = "spacy.FrenchLemmatizer.v1" + +[nlp.lemmatizer.data_paths] +@language_data = "spacy-lookups-data" +lang = ${nlp:lang} +""" + + +@registry.lemmatizers("spacy.FrenchLemmatizer.v1") +def create_french_lemmatizer(data_paths: dict = {}) -> FrenchLemmatizer: + return FrenchLemmatizer(data_paths=data_paths, is_base_form=is_base_form) + + +@registry.language_data("spacy.fr.stop_words") +def stop_words() -> Set[str]: + return STOP_WORDS + + +@registry.language_data("spacy.fr.lex_attr_getters") +def lex_attr_getters() -> Dict[int, Callable[[str], Any]]: + return LEX_ATTRS class FrenchDefaults(Language.Defaults): - lex_attr_getters = dict(Language.Defaults.lex_attr_getters) - lex_attr_getters.update(LEX_ATTRS) - lex_attr_getters[LANG] = lambda text: "fr" - lex_attr_getters[NORM] = add_lookups( - Language.Defaults.lex_attr_getters[NORM], BASE_NORMS - ) tokenizer_exceptions = update_exc(BASE_EXCEPTIONS, TOKENIZER_EXCEPTIONS) - stop_words = STOP_WORDS prefixes = TOKENIZER_PREFIXES infixes = TOKENIZER_INFIXES suffixes = TOKENIZER_SUFFIXES token_match = TOKEN_MATCH syntax_iterators = SYNTAX_ITERATORS - @classmethod - def create_lemmatizer(cls, nlp=None, lookups=None): - if lookups is None: - lookups = Lookups() - return FrenchLemmatizer(lookups) - class French(Language): lang = "fr" Defaults = FrenchDefaults + default_config = Config().from_str(DEFAULT_CONFIG) __all__ = ["French"] diff --git a/spacy/lang/fr/lemmatizer.py b/spacy/lang/fr/lemmatizer.py index 04982bb08..e46ec1682 100644 --- a/spacy/lang/fr/lemmatizer.py +++ b/spacy/lang/fr/lemmatizer.py @@ -1,3 +1,5 @@ +from typing import Optional, List, Dict + from ...lemmatizer import Lemmatizer from ...symbols import POS, NOUN, VERB, ADJ, ADV, PRON, DET, AUX, PUNCT, ADP from ...symbols import SCONJ, CCONJ @@ -13,7 +15,9 @@ class FrenchLemmatizer(Lemmatizer): the lookup table. """ - def __call__(self, string, univ_pos, morphology=None): + def __call__( + self, string: str, univ_pos: str, morphology: Optional[dict] = None + ) -> List[str]: lookup_table = self.lookups.get_table("lemma_lookup", {}) if "lemma_rules" not in self.lookups: return [lookup_table.get(string, string)] @@ -52,62 +56,19 @@ class FrenchLemmatizer(Lemmatizer): ) return lemmas - def is_base_form(self, univ_pos, morphology=None): - """ - Check whether we're dealing with an uninflected paradigm, so we can - avoid lemmatization entirely. - """ - morphology = {} if morphology is None else morphology - others = [ - key - for key in morphology - if key not in (POS, "Number", "POS", "VerbForm", "Tense") - ] - if univ_pos == "noun" and morphology.get("Number") == "sing": - return True - elif univ_pos == "verb" and morphology.get("VerbForm") == "inf": - return True - # This maps 'VBP' to base form -- probably just need 'IS_BASE' - # morphology - elif univ_pos == "verb" and ( - morphology.get("VerbForm") == "fin" - and morphology.get("Tense") == "pres" - and morphology.get("Number") is None - and not others - ): - return True - elif univ_pos == "adj" and morphology.get("Degree") == "pos": - return True - elif "VerbForm=inf" in morphology: - return True - elif "VerbForm=none" in morphology: - return True - elif "Number=sing" in morphology: - return True - elif "Degree=pos" in morphology: - return True - else: - return False - - def noun(self, string, morphology=None): - return self(string, "noun", morphology) - - def verb(self, string, morphology=None): - return self(string, "verb", morphology) - - def adj(self, string, morphology=None): - return self(string, "adj", morphology) - - def punct(self, string, morphology=None): - return self(string, "punct", morphology) - - def lookup(self, string, orth=None): + def lookup(self, string: str, orth: Optional[int] = None) -> str: lookup_table = self.lookups.get_table("lemma_lookup", {}) if orth is not None and orth in lookup_table: return lookup_table[orth][0] return string - def lemmatize(self, string, index, exceptions, rules): + def lemmatize( + self, + string: str, + index: Dict[str, List[str]], + exceptions: Dict[str, Dict[str, List[str]]], + rules: Dict[str, List[List[str]]], + ) -> List[str]: lookup_table = self.lookups.get_table("lemma_lookup", {}) string = string.lower() forms = [] @@ -133,3 +94,41 @@ class FrenchLemmatizer(Lemmatizer): if not forms: forms.append(string) return list(set(forms)) + + +def is_base_form(univ_pos: str, morphology: Optional[dict] = None) -> bool: + """ + Check whether we're dealing with an uninflected paradigm, so we can + avoid lemmatization entirely. + """ + morphology = {} if morphology is None else morphology + others = [ + key + for key in morphology + if key not in (POS, "Number", "POS", "VerbForm", "Tense") + ] + if univ_pos == "noun" and morphology.get("Number") == "sing": + return True + elif univ_pos == "verb" and morphology.get("VerbForm") == "inf": + return True + # This maps 'VBP' to base form -- probably just need 'IS_BASE' + # morphology + elif univ_pos == "verb" and ( + morphology.get("VerbForm") == "fin" + and morphology.get("Tense") == "pres" + and morphology.get("Number") is None + and not others + ): + return True + elif univ_pos == "adj" and morphology.get("Degree") == "pos": + return True + elif "VerbForm=inf" in morphology: + return True + elif "VerbForm=none" in morphology: + return True + elif "Number=sing" in morphology: + return True + elif "Degree=pos" in morphology: + return True + else: + return False diff --git a/spacy/lang/ga/__init__.py b/spacy/lang/ga/__init__.py index cea7c0e94..d88051a65 100644 --- a/spacy/lang/ga/__init__.py +++ b/spacy/lang/ga/__init__.py @@ -1,23 +1,33 @@ +from typing import Set +from thinc.api import Config + from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS from .stop_words import STOP_WORDS - from ..tokenizer_exceptions import BASE_EXCEPTIONS from ...language import Language -from ...attrs import LANG -from ...util import update_exc +from ...util import update_exc, registry + + +DEFAULT_CONFIG = """ +[nlp] +lang = "ga" +stop_words = {"@language_data": "spacy.ga.stop_words"} +""" + + +@registry.language_data("spacy.ga.stop_words") +def stop_words() -> Set[str]: + return STOP_WORDS class IrishDefaults(Language.Defaults): - lex_attr_getters = dict(Language.Defaults.lex_attr_getters) - lex_attr_getters[LANG] = lambda text: "ga" - tokenizer_exceptions = update_exc(BASE_EXCEPTIONS, TOKENIZER_EXCEPTIONS) - stop_words = set(STOP_WORDS) class Irish(Language): lang = "ga" Defaults = IrishDefaults + default_config = Config().from_str(DEFAULT_CONFIG) __all__ = ["Irish"] diff --git a/spacy/lang/gu/__init__.py b/spacy/lang/gu/__init__.py index bc8fc260c..3ca8bbd4c 100644 --- a/spacy/lang/gu/__init__.py +++ b/spacy/lang/gu/__init__.py @@ -1,15 +1,26 @@ +from typing import Set +from thinc.api import Config + from .stop_words import STOP_WORDS - from ...language import Language +from ...util import registry -class GujaratiDefaults(Language.Defaults): - stop_words = STOP_WORDS +DEFAULT_CONFIG = """ +[nlp] +lang = "gu" +stop_words = {"@language_data": "spacy.gu.stop_words"} +""" + + +@registry.language_data("spacy.gu.stop_words") +def stop_words() -> Set[str]: + return STOP_WORDS class Gujarati(Language): lang = "gu" - Defaults = GujaratiDefaults + default_config = Config().from_str(DEFAULT_CONFIG) __all__ = ["Gujarati"] diff --git a/spacy/lang/he/__init__.py b/spacy/lang/he/__init__.py index 0d324f64c..f979879a1 100644 --- a/spacy/lang/he/__init__.py +++ b/spacy/lang/he/__init__.py @@ -1,22 +1,37 @@ -from .stop_words import STOP_WORDS +from typing import Set +from thinc.api import Config +from .stop_words import STOP_WORDS from ..tokenizer_exceptions import BASE_EXCEPTIONS from ...language import Language -from ...attrs import LANG -from ...util import update_exc +from ...util import update_exc, registry + + +DEFAULT_CONFIG = """ +[nlp] +lang = "he" +stop_words = {"@language_data": "spacy.he.stop_words"} + +[nlp.writing_system] +direction = "rtl" +has_case = false +has_letters = true +""" + + +@registry.language_data("spacy.he.stop_words") +def stop_words() -> Set[str]: + return STOP_WORDS class HebrewDefaults(Language.Defaults): - lex_attr_getters = dict(Language.Defaults.lex_attr_getters) - lex_attr_getters[LANG] = lambda text: "he" tokenizer_exceptions = update_exc(BASE_EXCEPTIONS) - stop_words = STOP_WORDS - writing_system = {"direction": "rtl", "has_case": False, "has_letters": True} class Hebrew(Language): lang = "he" Defaults = HebrewDefaults + default_config = Config().from_str(DEFAULT_CONFIG) __all__ = ["Hebrew"] diff --git a/spacy/lang/hi/__init__.py b/spacy/lang/hi/__init__.py index 9a96de95c..48890c4f9 100644 --- a/spacy/lang/hi/__init__.py +++ b/spacy/lang/hi/__init__.py @@ -1,20 +1,33 @@ +from typing import Set, Dict, Callable, Any +from thinc.api import Config + from .stop_words import STOP_WORDS from .lex_attrs import LEX_ATTRS - from ...language import Language -from ...attrs import LANG +from ...util import registry -class HindiDefaults(Language.Defaults): - lex_attr_getters = dict(Language.Defaults.lex_attr_getters) - lex_attr_getters.update(LEX_ATTRS) - lex_attr_getters[LANG] = lambda text: "hi" - stop_words = STOP_WORDS +DEFAULT_CONFIG = """ +[nlp] +lang = "hi" +stop_words = {"@language_data": "spacy.hi.stop_words"} +lex_attr_getters = {"@language_data": "spacy.hi.lex_attr_getters"} +""" + + +@registry.language_data("spacy.hi.stop_words") +def stop_words() -> Set[str]: + return STOP_WORDS + + +@registry.language_data("spacy.hi.lex_attr_getters") +def lex_attr_getters() -> Dict[int, Callable[[str], Any]]: + return LEX_ATTRS class Hindi(Language): lang = "hi" - Defaults = HindiDefaults + default_config = Config().from_str(DEFAULT_CONFIG) __all__ = ["Hindi"] diff --git a/spacy/lang/hr/__init__.py b/spacy/lang/hr/__init__.py index fbc66ece0..648186093 100644 --- a/spacy/lang/hr/__init__.py +++ b/spacy/lang/hr/__init__.py @@ -1,25 +1,39 @@ -from .stop_words import STOP_WORDS +from typing import Set +from thinc.api import Config +from .stop_words import STOP_WORDS from ..tokenizer_exceptions import BASE_EXCEPTIONS -from ..norm_exceptions import BASE_NORMS from ...language import Language -from ...attrs import LANG, NORM -from ...util import update_exc, add_lookups +from ...util import update_exc, registry + + +DEFAULT_CONFIG = """ +[nlp] +lang = "hr" +stop_words = {"@language_data": "spacy.hr.stop_words"} + +[nlp.lemmatizer] +@lemmatizers = "spacy.Lemmatizer.v1" + +[nlp.lemmatizer.data_paths] +@language_data = "spacy-lookups-data" +lang = ${nlp:lang} +""" + + +@registry.language_data("spacy.hr.stop_words") +def stop_words() -> Set[str]: + return STOP_WORDS class CroatianDefaults(Language.Defaults): - lex_attr_getters = dict(Language.Defaults.lex_attr_getters) - lex_attr_getters[LANG] = lambda text: "hr" - lex_attr_getters[NORM] = add_lookups( - Language.Defaults.lex_attr_getters[NORM], BASE_NORMS - ) tokenizer_exceptions = update_exc(BASE_EXCEPTIONS) - stop_words = STOP_WORDS class Croatian(Language): lang = "hr" Defaults = CroatianDefaults + default_config = Config().from_str(DEFAULT_CONFIG) __all__ = ["Croatian"] diff --git a/spacy/lang/hu/__init__.py b/spacy/lang/hu/__init__.py index df3fe4a44..3e83e971a 100644 --- a/spacy/lang/hu/__init__.py +++ b/spacy/lang/hu/__init__.py @@ -1,22 +1,35 @@ +from typing import Set +from thinc.api import Config + from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS, TOKEN_MATCH from .punctuation import TOKENIZER_PREFIXES, TOKENIZER_SUFFIXES, TOKENIZER_INFIXES from .stop_words import STOP_WORDS - from ..tokenizer_exceptions import BASE_EXCEPTIONS -from ..norm_exceptions import BASE_NORMS from ...language import Language -from ...attrs import LANG, NORM -from ...util import update_exc, add_lookups +from ...util import update_exc, registry + + +DEFAULT_CONFIG = """ +[nlp] +lang = "hu" +stop_words = {"@language_data": "spacy.hu.stop_words"} + +[nlp.lemmatizer] +@lemmatizers = "spacy.Lemmatizer.v1" + +[nlp.lemmatizer.data_paths] +@language_data = "spacy-lookups-data" +lang = ${nlp:lang} +""" + + +@registry.language_data("spacy.hu.stop_words") +def stop_words() -> Set[str]: + return STOP_WORDS class HungarianDefaults(Language.Defaults): - lex_attr_getters = dict(Language.Defaults.lex_attr_getters) - lex_attr_getters[LANG] = lambda text: "hu" - lex_attr_getters[NORM] = add_lookups( - Language.Defaults.lex_attr_getters[NORM], BASE_NORMS - ) tokenizer_exceptions = update_exc(BASE_EXCEPTIONS, TOKENIZER_EXCEPTIONS) - stop_words = STOP_WORDS prefixes = TOKENIZER_PREFIXES suffixes = TOKENIZER_SUFFIXES infixes = TOKENIZER_INFIXES @@ -26,6 +39,7 @@ class HungarianDefaults(Language.Defaults): class Hungarian(Language): lang = "hu" Defaults = HungarianDefaults + default_config = Config().from_str(DEFAULT_CONFIG) __all__ = ["Hungarian"] diff --git a/spacy/lang/hy/__init__.py b/spacy/lang/hy/__init__.py index 863fde004..33bb8d08a 100644 --- a/spacy/lang/hy/__init__.py +++ b/spacy/lang/hy/__init__.py @@ -1,21 +1,33 @@ +from typing import Set, Dict, Callable, Any +from thinc.api import Config + from .stop_words import STOP_WORDS from .lex_attrs import LEX_ATTRS - -from ...attrs import LANG from ...language import Language +from ...util import registry -class ArmenianDefaults(Language.Defaults): - lex_attr_getters = dict(Language.Defaults.lex_attr_getters) - lex_attr_getters[LANG] = lambda text: "hy" +DEFAULT_CONFIG = """ +[nlp] +lang = "hy" +stop_words = {"@language_data": "spacy.hy.stop_words"} +lex_attr_getters = {"@language_data": "spacy.hy.lex_attr_getters"} +""" - lex_attr_getters.update(LEX_ATTRS) - stop_words = STOP_WORDS + +@registry.language_data("spacy.hy.stop_words") +def stop_words() -> Set[str]: + return STOP_WORDS + + +@registry.language_data("spacy.hy.lex_attr_getters") +def lex_attr_getters() -> Dict[int, Callable[[str], Any]]: + return LEX_ATTRS class Armenian(Language): lang = "hy" - Defaults = ArmenianDefaults + default_config = Config().from_str(DEFAULT_CONFIG) __all__ = ["Armenian"] diff --git a/spacy/lang/id/__init__.py b/spacy/lang/id/__init__.py index 4bb0af704..b8b34aa26 100644 --- a/spacy/lang/id/__init__.py +++ b/spacy/lang/id/__init__.py @@ -1,21 +1,43 @@ +from typing import Set, Dict, Callable, Any +from thinc.config import Config + from .stop_words import STOP_WORDS from .punctuation import TOKENIZER_SUFFIXES, TOKENIZER_PREFIXES, TOKENIZER_INFIXES from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS from .lex_attrs import LEX_ATTRS from .syntax_iterators import SYNTAX_ITERATORS - from ..tokenizer_exceptions import BASE_EXCEPTIONS from ...language import Language -from ...attrs import LANG -from ...util import update_exc +from ...util import update_exc, registry + + +DEFAULT_CONFIG = """ +[nlp] +lang = "id" +stop_words = {"@language_data": "spacy.id.stop_words"} +lex_attr_getters = {"@language_data": "spacy.id.lex_attr_getters"} + +[nlp.lemmatizer] +@lemmatizers = "spacy.Lemmatizer.v1" + +[nlp.lemmatizer.data_paths] +@language_data = "spacy-lookups-data" +lang = ${nlp:lang} +""" + + +@registry.language_data("spacy.id.stop_words") +def stop_words() -> Set[str]: + return STOP_WORDS + + +@registry.language_data("spacy.id.lex_attr_getters") +def lex_attr_getters() -> Dict[int, Callable[[str], Any]]: + return LEX_ATTRS class IndonesianDefaults(Language.Defaults): - lex_attr_getters = dict(Language.Defaults.lex_attr_getters) - lex_attr_getters[LANG] = lambda text: "id" - lex_attr_getters.update(LEX_ATTRS) tokenizer_exceptions = update_exc(BASE_EXCEPTIONS, TOKENIZER_EXCEPTIONS) - stop_words = STOP_WORDS prefixes = TOKENIZER_PREFIXES suffixes = TOKENIZER_SUFFIXES infixes = TOKENIZER_INFIXES @@ -25,6 +47,7 @@ class IndonesianDefaults(Language.Defaults): class Indonesian(Language): lang = "id" Defaults = IndonesianDefaults + default_config = Config().from_str(DEFAULT_CONFIG) __all__ = ["Indonesian"] diff --git a/spacy/lang/is/__init__.py b/spacy/lang/is/__init__.py index cdcfd6e71..82fc7e0c2 100644 --- a/spacy/lang/is/__init__.py +++ b/spacy/lang/is/__init__.py @@ -1,17 +1,26 @@ +from typing import Set +from thinc.api import Config + from .stop_words import STOP_WORDS from ...language import Language -from ...attrs import LANG +from ...util import registry -class IcelandicDefaults(Language.Defaults): - lex_attr_getters = dict(Language.Defaults.lex_attr_getters) - lex_attr_getters[LANG] = lambda text: "is" - stop_words = STOP_WORDS +DEFAULT_CONFIG = """ +[nlp] +lang = "is" +stop_words = {"@language_data": "spacy.is.stop_words"} +""" + + +@registry.language_data("spacy.is.stop_words") +def stop_words() -> Set[str]: + return STOP_WORDS class Icelandic(Language): lang = "is" - Defaults = IcelandicDefaults + default_config = Config().from_str(DEFAULT_CONFIG) __all__ = ["Icelandic"] diff --git a/spacy/lang/it/__init__.py b/spacy/lang/it/__init__.py index dbdfc1b5a..1b0a15348 100644 --- a/spacy/lang/it/__init__.py +++ b/spacy/lang/it/__init__.py @@ -1,20 +1,34 @@ +from typing import Set +from thinc.api import Config + from .stop_words import STOP_WORDS from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS from .punctuation import TOKENIZER_PREFIXES, TOKENIZER_INFIXES - from ..tokenizer_exceptions import BASE_EXCEPTIONS -from ..norm_exceptions import BASE_NORMS from ...language import Language -from ...attrs import LANG, NORM -from ...util import update_exc, add_lookups +from ...util import update_exc, registry + + +DEFAULT_CONFIG = """ +[nlp] +lang = "it" +stop_words = {"@language_data": "spacy.it.stop_words"} + +[nlp.lemmatizer] +@lemmatizers = "spacy.Lemmatizer.v1" + +[nlp.lemmatizer.data_paths] +@language_data = "spacy-lookups-data" +lang = ${nlp:lang} +""" + + +@registry.language_data("spacy.it.stop_words") +def stop_words() -> Set[str]: + return STOP_WORDS class ItalianDefaults(Language.Defaults): - lex_attr_getters = dict(Language.Defaults.lex_attr_getters) - lex_attr_getters[LANG] = lambda text: "it" - lex_attr_getters[NORM] = add_lookups( - Language.Defaults.lex_attr_getters[NORM], BASE_NORMS - ) tokenizer_exceptions = update_exc(BASE_EXCEPTIONS, TOKENIZER_EXCEPTIONS) stop_words = STOP_WORDS prefixes = TOKENIZER_PREFIXES @@ -24,6 +38,7 @@ class ItalianDefaults(Language.Defaults): class Italian(Language): lang = "it" Defaults = ItalianDefaults + default_config = Config().from_str(DEFAULT_CONFIG) __all__ = ["Italian"] diff --git a/spacy/lang/ja/__init__.py b/spacy/lang/ja/__init__.py index 220c81057..2e56c08d8 100644 --- a/spacy/lang/ja/__init__.py +++ b/spacy/lang/ja/__init__.py @@ -1,21 +1,187 @@ +from typing import Optional, Union, Dict, Any, Set +from pathlib import Path import srsly -from collections import namedtuple, OrderedDict +from collections import namedtuple +from thinc.api import Config from .stop_words import STOP_WORDS from .syntax_iterators import SYNTAX_ITERATORS from .tag_map import TAG_MAP from .tag_orth_map import TAG_ORTH_MAP from .tag_bigram_map import TAG_BIGRAM_MAP -from ...attrs import LANG from ...compat import copy_reg from ...errors import Errors from ...language import Language from ...symbols import POS from ...tokens import Doc -from ...util import DummyTokenizer +from ...util import DummyTokenizer, registry from ... import util +DEFAULT_CONFIG = """ +[nlp] +lang = "ja" +stop_words = {"@language_data": "spacy.ja.stop_words"} + +[nlp.tokenizer] +@tokenizers = "spacy.JapaneseTokenizer.v1" +split_mode = null + +[nlp.writing_system] +direction = "ltr" +has_case = false +has_letters = false +""" + + +@registry.language_data("spacy.ja.stop_words") +def stop_words() -> Set[str]: + return STOP_WORDS + + +@registry.tokenizers("spacy.JapaneseTokenizer.v1") +def create_japanese_tokenizer(split_mode: Optional[str] = None): + def japanese_tokenizer_factory(nlp): + return JapaneseTokenizer(nlp, split_mode=split_mode) + + return japanese_tokenizer_factory + + +class JapaneseTokenizer(DummyTokenizer): + def __init__(self, nlp: Language, split_mode: Optional[str] = None) -> None: + self.vocab = nlp.vocab + self.split_mode = split_mode + self.tokenizer = try_sudachi_import(self.split_mode) + + def __call__(self, text: str) -> Doc: + # convert sudachipy.morpheme.Morpheme to DetailedToken and merge continuous spaces + sudachipy_tokens = self.tokenizer.tokenize(text) + dtokens = self._get_dtokens(sudachipy_tokens) + dtokens, spaces = get_dtokens_and_spaces(dtokens, text) + + # create Doc with tag bi-gram based part-of-speech identification rules + words, tags, inflections, lemmas, readings, sub_tokens_list = ( + zip(*dtokens) if dtokens else [[]] * 6 + ) + sub_tokens_list = list(sub_tokens_list) + doc = Doc(self.vocab, words=words, spaces=spaces) + next_pos = None # for bi-gram rules + for idx, (token, dtoken) in enumerate(zip(doc, dtokens)): + token.tag_ = dtoken.tag + if next_pos: # already identified in previous iteration + token.pos = next_pos + next_pos = None + else: + token.pos, next_pos = resolve_pos( + token.orth_, + dtoken.tag, + tags[idx + 1] if idx + 1 < len(tags) else None, + ) + # if there's no lemma info (it's an unk) just use the surface + token.lemma_ = dtoken.lemma if dtoken.lemma else dtoken.surface + doc.user_data["inflections"] = inflections + doc.user_data["reading_forms"] = readings + doc.user_data["sub_tokens"] = sub_tokens_list + return doc + + def _get_dtokens(self, sudachipy_tokens, need_sub_tokens: bool = True): + sub_tokens_list = ( + self._get_sub_tokens(sudachipy_tokens) if need_sub_tokens else None + ) + dtokens = [ + DetailedToken( + token.surface(), # orth + "-".join([xx for xx in token.part_of_speech()[:4] if xx != "*"]), # tag + ",".join([xx for xx in token.part_of_speech()[4:] if xx != "*"]), # inf + token.dictionary_form(), # lemma + token.reading_form(), # user_data['reading_forms'] + sub_tokens_list[idx] + if sub_tokens_list + else None, # user_data['sub_tokens'] + ) + for idx, token in enumerate(sudachipy_tokens) + if len(token.surface()) > 0 + # remove empty tokens which can be produced with characters like … that + ] + # Sudachi normalizes internally and outputs each space char as a token. + # This is the preparation for get_dtokens_and_spaces() to merge the continuous space tokens + return [ + t + for idx, t in enumerate(dtokens) + if idx == 0 + or not t.surface.isspace() + or t.tag != "空白" + or not dtokens[idx - 1].surface.isspace() + or dtokens[idx - 1].tag != "空白" + ] + + def _get_sub_tokens(self, sudachipy_tokens): + if ( + self.split_mode is None or self.split_mode == "A" + ): # do nothing for default split mode + return None + + sub_tokens_list = [] # list of (list of list of DetailedToken | None) + for token in sudachipy_tokens: + sub_a = token.split(self.tokenizer.SplitMode.A) + if len(sub_a) == 1: # no sub tokens + sub_tokens_list.append(None) + elif self.split_mode == "B": + sub_tokens_list.append([self._get_dtokens(sub_a, False)]) + else: # "C" + sub_b = token.split(self.tokenizer.SplitMode.B) + if len(sub_a) == len(sub_b): + dtokens = self._get_dtokens(sub_a, False) + sub_tokens_list.append([dtokens, dtokens]) + else: + sub_tokens_list.append( + [ + self._get_dtokens(sub_a, False), + self._get_dtokens(sub_b, False), + ] + ) + return sub_tokens_list + + def _get_config(self) -> Dict[str, Any]: + return {"split_mode": self.split_mode} + + def _set_config(self, config: Dict[str, Any] = {}) -> None: + self.split_mode = config.get("split_mode", None) + + def to_bytes(self, **kwargs) -> bytes: + serializers = {"cfg": lambda: srsly.json_dumps(self._get_config())} + return util.to_bytes(serializers, []) + + def from_bytes(self, data: bytes, **kwargs) -> "JapaneseTokenizer": + deserializers = {"cfg": lambda b: self._set_config(srsly.json_loads(b))} + util.from_bytes(data, deserializers, []) + self.tokenizer = try_sudachi_import(self.split_mode) + return self + + def to_disk(self, path: Union[str, Path], **kwargs) -> None: + path = util.ensure_path(path) + serializers = {"cfg": lambda p: srsly.write_json(p, self._get_config())} + return util.to_disk(path, serializers, []) + + def from_disk(self, path: Union[str, Path], **kwargs) -> "JapaneseTokenizer": + path = util.ensure_path(path) + serializers = {"cfg": lambda p: self._set_config(srsly.read_json(p))} + util.from_disk(path, serializers, []) + self.tokenizer = try_sudachi_import(self.split_mode) + return self + + +class JapaneseDefaults(Language.Defaults): + tag_map = TAG_MAP + syntax_iterators = SYNTAX_ITERATORS + + +class Japanese(Language): + lang = "ja" + Defaults = JapaneseDefaults + default_config = Config().from_str(DEFAULT_CONFIG) + + # Hold the attributes we need with convenient names DetailedToken = namedtuple( "DetailedToken", ["surface", "tag", "inf", "lemma", "reading", "sub_tokens"] @@ -133,161 +299,6 @@ def get_dtokens_and_spaces(dtokens, text, gap_tag="空白"): return text_dtokens, text_spaces -class JapaneseTokenizer(DummyTokenizer): - def __init__(self, cls, nlp=None, config={}): - self.vocab = nlp.vocab if nlp is not None else cls.create_vocab(nlp) - self.split_mode = config.get("split_mode", None) - self.tokenizer = try_sudachi_import(self.split_mode) - - def __call__(self, text): - # convert sudachipy.morpheme.Morpheme to DetailedToken and merge continuous spaces - sudachipy_tokens = self.tokenizer.tokenize(text) - dtokens = self._get_dtokens(sudachipy_tokens) - dtokens, spaces = get_dtokens_and_spaces(dtokens, text) - - # create Doc with tag bi-gram based part-of-speech identification rules - words, tags, inflections, lemmas, readings, sub_tokens_list = ( - zip(*dtokens) if dtokens else [[]] * 6 - ) - sub_tokens_list = list(sub_tokens_list) - doc = Doc(self.vocab, words=words, spaces=spaces) - next_pos = None # for bi-gram rules - for idx, (token, dtoken) in enumerate(zip(doc, dtokens)): - token.tag_ = dtoken.tag - if next_pos: # already identified in previous iteration - token.pos = next_pos - next_pos = None - else: - token.pos, next_pos = resolve_pos( - token.orth_, - dtoken.tag, - tags[idx + 1] if idx + 1 < len(tags) else None, - ) - # if there's no lemma info (it's an unk) just use the surface - token.lemma_ = dtoken.lemma if dtoken.lemma else dtoken.surface - - doc.user_data["inflections"] = inflections - doc.user_data["reading_forms"] = readings - doc.user_data["sub_tokens"] = sub_tokens_list - - return doc - - def _get_dtokens(self, sudachipy_tokens, need_sub_tokens=True): - sub_tokens_list = ( - self._get_sub_tokens(sudachipy_tokens) if need_sub_tokens else None - ) - dtokens = [ - DetailedToken( - token.surface(), # orth - "-".join([xx for xx in token.part_of_speech()[:4] if xx != "*"]), # tag - ",".join([xx for xx in token.part_of_speech()[4:] if xx != "*"]), # inf - token.dictionary_form(), # lemma - token.reading_form(), # user_data['reading_forms'] - sub_tokens_list[idx] - if sub_tokens_list - else None, # user_data['sub_tokens'] - ) - for idx, token in enumerate(sudachipy_tokens) - if len(token.surface()) > 0 - # remove empty tokens which can be produced with characters like … that - ] - # Sudachi normalizes internally and outputs each space char as a token. - # This is the preparation for get_dtokens_and_spaces() to merge the continuous space tokens - return [ - t - for idx, t in enumerate(dtokens) - if idx == 0 - or not t.surface.isspace() - or t.tag != "空白" - or not dtokens[idx - 1].surface.isspace() - or dtokens[idx - 1].tag != "空白" - ] - - def _get_sub_tokens(self, sudachipy_tokens): - if ( - self.split_mode is None or self.split_mode == "A" - ): # do nothing for default split mode - return None - - sub_tokens_list = [] # list of (list of list of DetailedToken | None) - for token in sudachipy_tokens: - sub_a = token.split(self.tokenizer.SplitMode.A) - if len(sub_a) == 1: # no sub tokens - sub_tokens_list.append(None) - elif self.split_mode == "B": - sub_tokens_list.append([self._get_dtokens(sub_a, False)]) - else: # "C" - sub_b = token.split(self.tokenizer.SplitMode.B) - if len(sub_a) == len(sub_b): - dtokens = self._get_dtokens(sub_a, False) - sub_tokens_list.append([dtokens, dtokens]) - else: - sub_tokens_list.append( - [ - self._get_dtokens(sub_a, False), - self._get_dtokens(sub_b, False), - ] - ) - return sub_tokens_list - - def _get_config(self): - config = OrderedDict((("split_mode", self.split_mode),)) - return config - - def _set_config(self, config={}): - self.split_mode = config.get("split_mode", None) - - def to_bytes(self, **kwargs): - serializers = OrderedDict( - (("cfg", lambda: srsly.json_dumps(self._get_config())),) - ) - return util.to_bytes(serializers, []) - - def from_bytes(self, data, **kwargs): - deserializers = OrderedDict( - (("cfg", lambda b: self._set_config(srsly.json_loads(b))),) - ) - util.from_bytes(data, deserializers, []) - self.tokenizer = try_sudachi_import(self.split_mode) - return self - - def to_disk(self, path, **kwargs): - path = util.ensure_path(path) - serializers = OrderedDict( - (("cfg", lambda p: srsly.write_json(p, self._get_config())),) - ) - return util.to_disk(path, serializers, []) - - def from_disk(self, path, **kwargs): - path = util.ensure_path(path) - serializers = OrderedDict( - (("cfg", lambda p: self._set_config(srsly.read_json(p))),) - ) - util.from_disk(path, serializers, []) - self.tokenizer = try_sudachi_import(self.split_mode) - - -class JapaneseDefaults(Language.Defaults): - lex_attr_getters = dict(Language.Defaults.lex_attr_getters) - lex_attr_getters[LANG] = lambda _text: "ja" - stop_words = STOP_WORDS - tag_map = TAG_MAP - syntax_iterators = SYNTAX_ITERATORS - writing_system = {"direction": "ltr", "has_case": False, "has_letters": False} - - @classmethod - def create_tokenizer(cls, nlp=None, config={}): - return JapaneseTokenizer(cls, nlp, config) - - -class Japanese(Language): - lang = "ja" - Defaults = JapaneseDefaults - - def make_doc(self, text): - return self.tokenizer(text) - - def pickle_japanese(instance): return Japanese, tuple() diff --git a/spacy/lang/kn/__init__.py b/spacy/lang/kn/__init__.py index ef3b10f81..c323ca5c7 100644 --- a/spacy/lang/kn/__init__.py +++ b/spacy/lang/kn/__init__.py @@ -1,17 +1,26 @@ +from typing import Set +from thinc.api import Config + from .stop_words import STOP_WORDS from ...language import Language -from ...attrs import LANG +from ...util import registry -class KannadaDefaults(Language.Defaults): - lex_attr_getters = dict(Language.Defaults.lex_attr_getters) - lex_attr_getters[LANG] = lambda text: "kn" - stop_words = STOP_WORDS +DEFAULT_CONFIG = """ +[nlp] +lang = "kn" +stop_words = {"@language_data": "spacy.kn.stop_words"} +""" + + +@registry.language_data("spacy.kn.stop_words") +def stop_words() -> Set[str]: + return STOP_WORDS class Kannada(Language): lang = "kn" - Defaults = KannadaDefaults + default_config = Config().from_str(DEFAULT_CONFIG) __all__ = ["Kannada"] diff --git a/spacy/lang/ko/__init__.py b/spacy/lang/ko/__init__.py index 8b45c602c..f55660745 100644 --- a/spacy/lang/ko/__init__.py +++ b/spacy/lang/ko/__init__.py @@ -1,51 +1,52 @@ +from typing import Set, Optional, Any, Dict +from thinc.api import Config + from .stop_words import STOP_WORDS from .tag_map import TAG_MAP -from ...attrs import LANG from ...language import Language from ...tokens import Doc from ...compat import copy_reg -from ...util import DummyTokenizer +from ...util import DummyTokenizer, registry -def try_mecab_import(): - try: - from natto import MeCab +DEFAULT_CONFIG = """ +[nlp] +lang = "ko" +stop_words = {"@language_data": "spacy.ko.stop_words"} - return MeCab - except ImportError: - raise ImportError( - "Korean support requires [mecab-ko](https://bitbucket.org/eunjeon/mecab-ko/src/master/README.md), " - "[mecab-ko-dic](https://bitbucket.org/eunjeon/mecab-ko-dic), " - "and [natto-py](https://github.com/buruzaemon/natto-py)" - ) +[nlp.tokenizer] +@tokenizers = "spacy.KoreanTokenizer.v1" + +[nlp.writing_system] +direction = "ltr" +has_case = false +has_letters = false +""" -# fmt: on +@registry.language_data("spacy.ko.stop_words") +def stop_words() -> Set[str]: + return STOP_WORDS -def check_spaces(text, tokens): - prev_end = -1 - start = 0 - for token in tokens: - idx = text.find(token, start) - if prev_end > 0: - yield prev_end != idx - prev_end = idx + len(token) - start = prev_end - if start > 0: - yield False +@registry.tokenizers("spacy.KoreanTokenizer.v1") +def create_korean_tokenizer(): + def korean_tokenizer_factory(nlp): + return KoreanTokenizer(nlp) + + return korean_tokenizer_factory class KoreanTokenizer(DummyTokenizer): - def __init__(self, cls, nlp=None): - self.vocab = nlp.vocab if nlp is not None else cls.create_vocab(nlp) + def __init__(self, nlp: Optional[Language] = None): + self.vocab = nlp.vocab MeCab = try_mecab_import() self.mecab_tokenizer = MeCab("-F%f[0],%f[7]") def __del__(self): self.mecab_tokenizer.__del__() - def __call__(self, text): + def __call__(self, text: str) -> Doc: dtokens = list(self.detailed_tokens(text)) surfaces = [dt["surface"] for dt in dtokens] doc = Doc(self.vocab, words=surfaces, spaces=list(check_spaces(text, surfaces))) @@ -56,7 +57,7 @@ class KoreanTokenizer(DummyTokenizer): doc.user_data["full_tags"] = [dt["tag"] for dt in dtokens] return doc - def detailed_tokens(self, text): + def detailed_tokens(self, text: str) -> Dict[str, Any]: # 품사 태그(POS)[0], 의미 부류(semantic class)[1], 종성 유무(jongseong)[2], 읽기(reading)[3], # 타입(type)[4], 첫번째 품사(start pos)[5], 마지막 품사(end pos)[6], 표현(expression)[7], * for node in self.mecab_tokenizer.parse(text, as_nodes=True): @@ -72,23 +73,39 @@ class KoreanTokenizer(DummyTokenizer): class KoreanDefaults(Language.Defaults): - lex_attr_getters = dict(Language.Defaults.lex_attr_getters) - lex_attr_getters[LANG] = lambda _text: "ko" - stop_words = STOP_WORDS tag_map = TAG_MAP - writing_system = {"direction": "ltr", "has_case": False, "has_letters": False} - - @classmethod - def create_tokenizer(cls, nlp=None): - return KoreanTokenizer(cls, nlp) class Korean(Language): lang = "ko" Defaults = KoreanDefaults + default_config = Config().from_str(DEFAULT_CONFIG) - def make_doc(self, text): - return self.tokenizer(text) + +def try_mecab_import() -> None: + try: + from natto import MeCab + + return MeCab + except ImportError: + raise ImportError( + "Korean support requires [mecab-ko](https://bitbucket.org/eunjeon/mecab-ko/src/master/README.md), " + "[mecab-ko-dic](https://bitbucket.org/eunjeon/mecab-ko-dic), " + "and [natto-py](https://github.com/buruzaemon/natto-py)" + ) + + +def check_spaces(text, tokens): + prev_end = -1 + start = 0 + for token in tokens: + idx = text.find(token, start) + if prev_end > 0: + yield prev_end != idx + prev_end = idx + len(token) + start = prev_end + if start > 0: + yield False def pickle_korean(instance): diff --git a/spacy/lang/lb/__init__.py b/spacy/lang/lb/__init__.py index 5ea74066a..54e4e82c0 100644 --- a/spacy/lang/lb/__init__.py +++ b/spacy/lang/lb/__init__.py @@ -1,26 +1,49 @@ +from typing import Set, Dict, Callable, Any +from thinc.api import Config + from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS from .punctuation import TOKENIZER_INFIXES from .lex_attrs import LEX_ATTRS from .stop_words import STOP_WORDS - from ..tokenizer_exceptions import BASE_EXCEPTIONS from ...language import Language -from ...attrs import LANG -from ...util import update_exc +from ...util import update_exc, registry + + +DEFAULT_CONFIG = """ +[nlp] +lang = "lb" +stop_words = {"@language_data": "spacy.lb.stop_words"} +lex_attr_getters = {"@language_data": "spacy.lb.lex_attr_getters"} + +[nlp.lemmatizer] +@lemmatizers = "spacy.Lemmatizer.v1" + +[nlp.lemmatizer.data_paths] +@language_data = "spacy-lookups-data" +lang = ${nlp:lang} +""" + + +@registry.language_data("spacy.lb.stop_words") +def stop_words() -> Set[str]: + return STOP_WORDS + + +@registry.language_data("spacy.lb.lex_attr_getters") +def lex_attr_getters() -> Dict[int, Callable[[str], Any]]: + return LEX_ATTRS class LuxembourgishDefaults(Language.Defaults): - lex_attr_getters = dict(Language.Defaults.lex_attr_getters) - lex_attr_getters.update(LEX_ATTRS) - lex_attr_getters[LANG] = lambda text: "lb" tokenizer_exceptions = update_exc(BASE_EXCEPTIONS, TOKENIZER_EXCEPTIONS) - stop_words = STOP_WORDS infixes = TOKENIZER_INFIXES class Luxembourgish(Language): lang = "lb" Defaults = LuxembourgishDefaults + default_config = Config().from_str(DEFAULT_CONFIG) __all__ = ["Luxembourgish"] diff --git a/spacy/lang/lex_attrs.py b/spacy/lang/lex_attrs.py index 0310b2b36..088a05ef4 100644 --- a/spacy/lang/lex_attrs.py +++ b/spacy/lang/lex_attrs.py @@ -1,3 +1,4 @@ +from typing import Set import unicodedata import re @@ -21,21 +22,21 @@ _tlds = set( ) -def is_punct(text): +def is_punct(text: str) -> bool: for char in text: if not unicodedata.category(char).startswith("P"): return False return True -def is_ascii(text): +def is_ascii(text: str) -> bool: for char in text: if ord(char) >= 128: return False return True -def like_num(text): +def like_num(text: str) -> bool: if text.startswith(("+", "-", "±", "~")): text = text[1:] # can be overwritten by lang with list of number words @@ -49,64 +50,31 @@ def like_num(text): return False -def is_bracket(text): +def is_bracket(text: str) -> bool: brackets = ("(", ")", "[", "]", "{", "}", "<", ">") return text in brackets -def is_quote(text): - quotes = ( - '"', - "'", - "`", - "«", - "»", - "‘", - "’", - "‚", - "‛", - "“", - "”", - "„", - "‟", - "‹", - "›", - "❮", - "❯", - "''", - "``", - ) +def is_quote(text: str) -> bool: + # fmt: off + quotes = ('"', "'", "`", "«", "»", "‘", "’", "‚", "‛", "“", "”", "„", "‟", "‹", "›", "❮", "❯", "''", "``") + # fmt: on return text in quotes -def is_left_punct(text): - left_punct = ( - "(", - "[", - "{", - "<", - '"', - "'", - "«", - "‘", - "‚", - "‛", - "“", - "„", - "‟", - "‹", - "❮", - "``", - ) +def is_left_punct(text: str) -> bool: + # fmt: off + left_punct = ("(", "[", "{", "<", '"', "'", "«", "‘", "‚", "‛", "“", "„", "‟", "‹", "❮", "``") + # fmt: on return text in left_punct -def is_right_punct(text): +def is_right_punct(text: str) -> bool: right_punct = (")", "]", "}", ">", '"', "'", "»", "’", "”", "›", "❯", "''") return text in right_punct -def is_currency(text): +def is_currency(text: str) -> bool: # can be overwritten by lang with list of currency words, e.g. dollar, euro for char in text: if unicodedata.category(char) != "Sc": @@ -114,11 +82,11 @@ def is_currency(text): return True -def like_email(text): +def like_email(text: str) -> bool: return bool(_like_email(text)) -def like_url(text): +def like_url(text: str) -> bool: # We're looking for things that function in text like URLs. So, valid URL # or not, anything they say http:// is going to be good. if text.startswith("http://") or text.startswith("https://"): @@ -144,7 +112,7 @@ def like_url(text): return False -def word_shape(text): +def word_shape(text: str) -> str: if len(text) >= 100: return "LONG" shape = [] @@ -171,46 +139,52 @@ def word_shape(text): return "".join(shape) -def lower(string): +def lower(string: str) -> str: return string.lower() -def prefix(string): +def prefix(string: str) -> str: return string[0] -def suffix(string): +def suffix(string: str) -> str: return string[-3:] -def is_alpha(string): +def is_alpha(string: str) -> bool: return string.isalpha() -def is_digit(string): +def is_digit(string: str) -> bool: return string.isdigit() -def is_lower(string): +def is_lower(string: str) -> bool: return string.islower() -def is_space(string): +def is_space(string: str) -> bool: return string.isspace() -def is_title(string): +def is_title(string: str) -> bool: return string.istitle() -def is_upper(string): +def is_upper(string: str) -> bool: return string.isupper() -def is_stop(string, stops=set()): +def is_stop(string: str, stops: Set[str] = set()) -> bool: return string.lower() in stops +def get_lang(text: str, lang: str = "") -> str: + # This function is partially applied so lang code can be passed in + # automatically while still allowing pickling + return lang + + LEX_ATTRS = { attrs.LOWER: lower, attrs.NORM: lower, diff --git a/spacy/lang/lij/__init__.py b/spacy/lang/lij/__init__.py index a75f081bf..c7a17bee9 100644 --- a/spacy/lang/lij/__init__.py +++ b/spacy/lang/lij/__init__.py @@ -1,28 +1,35 @@ +from typing import Set +from thinc.api import Config + from .stop_words import STOP_WORDS from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS from .punctuation import TOKENIZER_INFIXES - from ..tokenizer_exceptions import BASE_EXCEPTIONS -from ..norm_exceptions import BASE_NORMS from ...language import Language -from ...attrs import LANG, NORM -from ...util import update_exc, add_lookups +from ...util import update_exc, registry + + +DEFAULT_CONFIG = """ +[nlp] +lang = "lij" +stop_words = {"@language_data": "spacy.lij.stop_words"} +""" + + +@registry.language_data("spacy.lij.stop_words") +def stop_words() -> Set[str]: + return STOP_WORDS class LigurianDefaults(Language.Defaults): - lex_attr_getters = dict(Language.Defaults.lex_attr_getters) - lex_attr_getters[LANG] = lambda text: "lij" - lex_attr_getters[NORM] = add_lookups( - Language.Defaults.lex_attr_getters[NORM], BASE_NORMS - ) tokenizer_exceptions = update_exc(BASE_EXCEPTIONS, TOKENIZER_EXCEPTIONS) - stop_words = STOP_WORDS infixes = TOKENIZER_INFIXES class Ligurian(Language): lang = "lij" Defaults = LigurianDefaults + default_config = Config().from_str(DEFAULT_CONFIG) __all__ = ["Ligurian"] diff --git a/spacy/lang/lt/__init__.py b/spacy/lang/lt/__init__.py index fa3c87e21..656df79c9 100644 --- a/spacy/lang/lt/__init__.py +++ b/spacy/lang/lt/__init__.py @@ -1,27 +1,41 @@ +from typing import Set, Dict, Callable, Any +from thinc.api import Config + from .punctuation import TOKENIZER_INFIXES, TOKENIZER_SUFFIXES from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS from .stop_words import STOP_WORDS from .lex_attrs import LEX_ATTRS - from ..tokenizer_exceptions import BASE_EXCEPTIONS -from ..norm_exceptions import BASE_NORMS from ...language import Language -from ...attrs import LANG, NORM -from ...util import update_exc, add_lookups +from ...util import update_exc, registry -def _return_lt(_): - return "lt" +DEFAULT_CONFIG = """ +[nlp] +lang = "lt" +stop_words = {"@language_data": "spacy.lt.stop_words"} +lex_attr_getters = {"@language_data": "spacy.lt.lex_attr_getters"} + +[nlp.lemmatizer] +@lemmatizers = "spacy.Lemmatizer.v1" + +[nlp.lemmatizer.data_paths] +@language_data = "spacy-lookups-data" +lang = ${nlp:lang} +""" + + +@registry.language_data("spacy.lt.stop_words") +def stop_words() -> Set[str]: + return STOP_WORDS + + +@registry.language_data("spacy.lt.lex_attr_getters") +def lex_attr_getters() -> Dict[int, Callable[[str], Any]]: + return LEX_ATTRS class LithuanianDefaults(Language.Defaults): - lex_attr_getters = dict(Language.Defaults.lex_attr_getters) - lex_attr_getters[LANG] = _return_lt - lex_attr_getters[NORM] = add_lookups( - Language.Defaults.lex_attr_getters[NORM], BASE_NORMS - ) - lex_attr_getters.update(LEX_ATTRS) - infixes = TOKENIZER_INFIXES suffixes = TOKENIZER_SUFFIXES mod_base_exceptions = { @@ -29,12 +43,12 @@ class LithuanianDefaults(Language.Defaults): } del mod_base_exceptions["8)"] tokenizer_exceptions = update_exc(mod_base_exceptions, TOKENIZER_EXCEPTIONS) - stop_words = STOP_WORDS class Lithuanian(Language): lang = "lt" Defaults = LithuanianDefaults + default_config = Config().from_str(DEFAULT_CONFIG) __all__ = ["Lithuanian"] diff --git a/spacy/lang/lv/__init__.py b/spacy/lang/lv/__init__.py index dd8919b73..e37b44b0d 100644 --- a/spacy/lang/lv/__init__.py +++ b/spacy/lang/lv/__init__.py @@ -1,17 +1,26 @@ +from typing import Set +from thinc.api import Config + from .stop_words import STOP_WORDS from ...language import Language -from ...attrs import LANG +from ...util import registry -class LatvianDefaults(Language.Defaults): - lex_attr_getters = dict(Language.Defaults.lex_attr_getters) - lex_attr_getters[LANG] = lambda text: "lv" - stop_words = STOP_WORDS +DEFAULT_CONFIG = """ +[nlp] +lang = "lv" +stop_words = {"@language_data": "spacy.lv.stop_words"} +""" + + +@registry.language_data("spacy.lv.stop_words") +def stop_words() -> Set[str]: + return STOP_WORDS class Latvian(Language): lang = "lv" - Defaults = LatvianDefaults + default_config = Config().from_str(DEFAULT_CONFIG) __all__ = ["Latvian"] diff --git a/spacy/lang/ml/__init__.py b/spacy/lang/ml/__init__.py index e92a7617f..e2ac0a641 100644 --- a/spacy/lang/ml/__init__.py +++ b/spacy/lang/ml/__init__.py @@ -1,15 +1,26 @@ +from typing import Set +from thinc.api import Config + from .stop_words import STOP_WORDS - from ...language import Language +from ...util import registry -class MalayalamDefaults(Language.Defaults): - stop_words = STOP_WORDS +DEFAULT_CONFIG = """ +[nlp] +lang = "ml" +stop_words = {"@language_data": "spacy.ml.stop_words"} +""" + + +@registry.language_data("spacy.ml.stop_words") +def stop_words() -> Set[str]: + return STOP_WORDS class Malayalam(Language): lang = "ml" - Defaults = MalayalamDefaults + default_config = Config().from_str(DEFAULT_CONFIG) __all__ = ["Malayalam"] diff --git a/spacy/lang/mr/__init__.py b/spacy/lang/mr/__init__.py index eb52a3935..3d7c621cb 100644 --- a/spacy/lang/mr/__init__.py +++ b/spacy/lang/mr/__init__.py @@ -1,17 +1,26 @@ +from typing import Set +from thinc.api import Config + from .stop_words import STOP_WORDS from ...language import Language -from ...attrs import LANG +from ...util import registry -class MarathiDefaults(Language.Defaults): - lex_attr_getters = dict(Language.Defaults.lex_attr_getters) - lex_attr_getters[LANG] = lambda text: "mr" - stop_words = STOP_WORDS +DEFAULT_CONFIG = """ +[nlp] +lang = "af" +stop_words = {"@language_data": "spacy.mr.stop_words"} +""" + + +@registry.language_data("spacy.mr.stop_words") +def stop_words() -> Set[str]: + return STOP_WORDS class Marathi(Language): lang = "mr" - Defaults = MarathiDefaults + default_config = Config().from_str(DEFAULT_CONFIG) __all__ = ["Marathi"] diff --git a/spacy/lang/nb/__init__.py b/spacy/lang/nb/__init__.py index 39df2e857..e472b0c60 100644 --- a/spacy/lang/nb/__init__.py +++ b/spacy/lang/nb/__init__.py @@ -1,33 +1,47 @@ +from typing import Set +from thinc.api import Config + from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS from .punctuation import TOKENIZER_PREFIXES, TOKENIZER_INFIXES from .punctuation import TOKENIZER_SUFFIXES from .stop_words import STOP_WORDS from .syntax_iterators import SYNTAX_ITERATORS - from ..tokenizer_exceptions import BASE_EXCEPTIONS -from ..norm_exceptions import BASE_NORMS from ...language import Language -from ...attrs import LANG, NORM -from ...util import update_exc, add_lookups +from ...util import update_exc, registry + + +DEFAULT_CONFIG = """ +[nlp] +lang = "nb" +stop_words = {"@language_data": "spacy.nb.stop_words"} + +[nlp.lemmatizer] +@lemmatizers = "spacy.Lemmatizer.v1" + +[nlp.lemmatizer.data_paths] +@language_data = "spacy-lookups-data" +lang = ${nlp:lang} +""" + + +@registry.language_data("spacy.nb.stop_words") +def stop_words() -> Set[str]: + return STOP_WORDS class NorwegianDefaults(Language.Defaults): - lex_attr_getters = dict(Language.Defaults.lex_attr_getters) - lex_attr_getters[LANG] = lambda text: "nb" - lex_attr_getters[NORM] = add_lookups( - Language.Defaults.lex_attr_getters[NORM], BASE_NORMS - ) tokenizer_exceptions = update_exc(BASE_EXCEPTIONS, TOKENIZER_EXCEPTIONS) prefixes = TOKENIZER_PREFIXES infixes = TOKENIZER_INFIXES suffixes = TOKENIZER_SUFFIXES - stop_words = STOP_WORDS syntax_iterators = SYNTAX_ITERATORS class Norwegian(Language): lang = "nb" Defaults = NorwegianDefaults + default_config = Config().from_str(DEFAULT_CONFIG) __all__ = ["Norwegian"] diff --git a/spacy/lang/ne/__init__.py b/spacy/lang/ne/__init__.py index 21556277d..b72af86e4 100644 --- a/spacy/lang/ne/__init__.py +++ b/spacy/lang/ne/__init__.py @@ -1,23 +1,33 @@ -# coding: utf8 -from __future__ import unicode_literals +from typing import Set, Dict, Callable, Any +from thinc.api import Config from .stop_words import STOP_WORDS from .lex_attrs import LEX_ATTRS - from ...language import Language -from ...attrs import LANG +from ...util import registry -class NepaliDefaults(Language.Defaults): - lex_attr_getters = dict(Language.Defaults.lex_attr_getters) - lex_attr_getters.update(LEX_ATTRS) - lex_attr_getters[LANG] = lambda text: "ne" # Nepali language ISO code - stop_words = STOP_WORDS +DEFAULT_CONFIG = """ +[nlp] +lang = "ne" +stop_words = {"@language_data": "spacy.ne.stop_words"} +lex_attr_getters = {"@language_data": "spacy.ne.lex_attr_getters"} +""" + + +@registry.language_data("spacy.ne.stop_words") +def stop_words() -> Set[str]: + return STOP_WORDS + + +@registry.language_data("spacy.ne.lex_attr_getters") +def lex_attr_getters() -> Dict[int, Callable[[str], Any]]: + return LEX_ATTRS class Nepali(Language): lang = "ne" - Defaults = NepaliDefaults + default_config = Config().from_str(DEFAULT_CONFIG) __all__ = ["Nepali"] diff --git a/spacy/lang/nl/__init__.py b/spacy/lang/nl/__init__.py index 355cc78db..7e9806bc3 100644 --- a/spacy/lang/nl/__init__.py +++ b/spacy/lang/nl/__init__.py @@ -1,3 +1,6 @@ +from typing import Set, Dict, Callable, Any +from thinc.api import Config + from .stop_words import STOP_WORDS from .lex_attrs import LEX_ATTRS from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS @@ -5,36 +8,51 @@ from .punctuation import TOKENIZER_PREFIXES, TOKENIZER_INFIXES from .punctuation import TOKENIZER_SUFFIXES from .lemmatizer import DutchLemmatizer from ..tokenizer_exceptions import BASE_EXCEPTIONS -from ..norm_exceptions import BASE_NORMS from ...language import Language -from ...lookups import Lookups -from ...attrs import LANG, NORM -from ...util import update_exc, add_lookups +from ...util import update_exc, registry + + +DEFAULT_CONFIG = """ +[nlp] +lang = "nl" +stop_words = {"@language_data": "spacy.nl.stop_words"} +lex_attr_getters = {"@language_data": "spacy.nl.lex_attr_getters"} + +[nlp.lemmatizer] +@lemmatizers = "spacy.DutchLemmatizer.v1" + +[nlp.lemmatizer.data_paths] +@language_data = "spacy-lookups-data" +lang = ${nlp:lang} +""" + + +@registry.language_data("spacy.nl.stop_words") +def stop_words() -> Set[str]: + return STOP_WORDS + + +@registry.language_data("spacy.nl.lex_attr_getters") +def lex_attr_getters() -> Dict[int, Callable[[str], Any]]: + return LEX_ATTRS + + +@registry.lemmatizers("spacy.DutchLemmatizer.v1") +def create_dutch_lemmatizer(data_paths: dict = {}) -> DutchLemmatizer: + return DutchLemmatizer(data_paths=data_paths) class DutchDefaults(Language.Defaults): - lex_attr_getters = dict(Language.Defaults.lex_attr_getters) - lex_attr_getters.update(LEX_ATTRS) - lex_attr_getters[LANG] = lambda text: "nl" - lex_attr_getters[NORM] = add_lookups( - Language.Defaults.lex_attr_getters[NORM], BASE_NORMS - ) tokenizer_exceptions = update_exc(BASE_EXCEPTIONS, TOKENIZER_EXCEPTIONS) - stop_words = STOP_WORDS prefixes = TOKENIZER_PREFIXES infixes = TOKENIZER_INFIXES suffixes = TOKENIZER_SUFFIXES - @classmethod - def create_lemmatizer(cls, nlp=None, lookups=None): - if lookups is None: - lookups = Lookups() - return DutchLemmatizer(lookups) - class Dutch(Language): lang = "nl" Defaults = DutchDefaults + default_config = Config().from_str(DEFAULT_CONFIG) __all__ = ["Dutch"] diff --git a/spacy/lang/nl/lemmatizer.py b/spacy/lang/nl/lemmatizer.py index e7501ec52..b01debaa9 100644 --- a/spacy/lang/nl/lemmatizer.py +++ b/spacy/lang/nl/lemmatizer.py @@ -1,3 +1,5 @@ +from typing import Optional, List, Dict, Tuple + from ...lemmatizer import Lemmatizer from ...symbols import NOUN, VERB, ADJ, NUM, DET, PRON, ADP, AUX, ADV @@ -34,7 +36,9 @@ class DutchLemmatizer(Lemmatizer): "num": "num", } - def __call__(self, string, univ_pos, morphology=None): + def __call__( + self, string: str, univ_pos: str, morphology: Optional[dict] = None + ) -> List[str]: # Difference 1: self.rules is assumed to be non-None, so no # 'is None' check required. # String lowercased from the get-go. All lemmatization results in @@ -92,7 +96,7 @@ class DutchLemmatizer(Lemmatizer): # Overrides parent method so that a lowercased version of the string is # used to search the lookup table. This is necessary because our lookup # table consists entirely of lowercase keys. - def lookup(self, string, orth=None): + def lookup(self, string: str, orth: Optional[int] = None) -> str: lookup_table = self.lookups.get_table("lemma_lookup", {}) string = string.lower() if orth is not None: @@ -102,7 +106,13 @@ class DutchLemmatizer(Lemmatizer): # Reimplemented to focus more on application of suffix rules and to return # as early as possible. - def lemmatize(self, string, index, exceptions, rules): + def lemmatize( + self, + string: str, + index: Dict[str, List[str]], + exceptions: Dict[str, Dict[str, List[str]]], + rules: Dict[str, List[List[str]]], + ) -> Tuple[List[str], bool]: # returns (forms, is_known: bool) oov_forms = [] for old, new in rules: diff --git a/spacy/lang/pl/__init__.py b/spacy/lang/pl/__init__.py index a6b82282f..87a174ec8 100644 --- a/spacy/lang/pl/__init__.py +++ b/spacy/lang/pl/__init__.py @@ -1,43 +1,60 @@ +from typing import Set, Dict, Callable, Any +from thinc.api import Config + from .punctuation import TOKENIZER_PREFIXES, TOKENIZER_INFIXES from .punctuation import TOKENIZER_SUFFIXES from .stop_words import STOP_WORDS from .lex_attrs import LEX_ATTRS from .lemmatizer import PolishLemmatizer - from ..tokenizer_exceptions import BASE_EXCEPTIONS -from ..norm_exceptions import BASE_NORMS from ...language import Language -from ...attrs import LANG, NORM -from ...util import add_lookups -from ...lookups import Lookups +from ...util import registry + + +DEFAULT_CONFIG = """ +[nlp] +lang = "pl" +stop_words = {"@language_data": "spacy.pl.stop_words"} +lex_attr_getters = {"@language_data": "spacy.pl.lex_attr_getters"} + +[nlp.lemmatizer] +@lemmatizers = "spacy.PolishLemmatizer.v1" + +[nlp.lemmatizer.data_paths] +@language_data = "spacy-lookups-data" +lang = ${nlp:lang} +""" + + +@registry.language_data("spacy.pl.stop_words") +def stop_words() -> Set[str]: + return STOP_WORDS + + +@registry.language_data("spacy.pl.lex_attr_getters") +def lex_attr_getters() -> Dict[int, Callable[[str], Any]]: + return LEX_ATTRS + + +@registry.lemmatizers("spacy.PolishLemmatizer.v1") +def create_polish_lemmatizer(data_paths: dict = {}) -> PolishLemmatizer: + return PolishLemmatizer(data_paths=data_paths) class PolishDefaults(Language.Defaults): - lex_attr_getters = dict(Language.Defaults.lex_attr_getters) - lex_attr_getters.update(LEX_ATTRS) - lex_attr_getters[LANG] = lambda text: "pl" - lex_attr_getters[NORM] = add_lookups( - Language.Defaults.lex_attr_getters[NORM], BASE_NORMS - ) mod_base_exceptions = { exc: val for exc, val in BASE_EXCEPTIONS.items() if not exc.endswith(".") } tokenizer_exceptions = mod_base_exceptions - stop_words = STOP_WORDS prefixes = TOKENIZER_PREFIXES infixes = TOKENIZER_INFIXES suffixes = TOKENIZER_SUFFIXES - @classmethod - def create_lemmatizer(cls, nlp=None, lookups=None): - if lookups is None: - lookups = Lookups() - return PolishLemmatizer(lookups) - class Polish(Language): lang = "pl" Defaults = PolishDefaults + default_config = Config().from_str(DEFAULT_CONFIG) __all__ = ["Polish"] diff --git a/spacy/lang/pl/lemmatizer.py b/spacy/lang/pl/lemmatizer.py index b80a1a143..8e96dd75b 100644 --- a/spacy/lang/pl/lemmatizer.py +++ b/spacy/lang/pl/lemmatizer.py @@ -1,3 +1,5 @@ +from typing import Optional, List, Dict + from ...lemmatizer import Lemmatizer from ...parts_of_speech import NAMES @@ -7,30 +9,29 @@ class PolishLemmatizer(Lemmatizer): # dictionary (morfeusz.sgjp.pl/en) by Institute of Computer Science PAS. # It utilizes some prefix based improvements for verb and adjectives # lemmatization, as well as case-sensitive lemmatization for nouns. - def __call__(self, string, univ_pos, morphology=None): + def __call__( + self, string: str, univ_pos: str, morphology: Optional[dict] = None + ) -> List[str]: if isinstance(univ_pos, int): univ_pos = NAMES.get(univ_pos, "X") univ_pos = univ_pos.upper() - lookup_pos = univ_pos.lower() if univ_pos == "PROPN": lookup_pos = "noun" lookup_table = self.lookups.get_table("lemma_lookup_" + lookup_pos, {}) - if univ_pos == "NOUN": return self.lemmatize_noun(string, morphology, lookup_table) - if univ_pos != "PROPN": string = string.lower() - if univ_pos == "ADJ": return self.lemmatize_adj(string, morphology, lookup_table) elif univ_pos == "VERB": return self.lemmatize_verb(string, morphology, lookup_table) - return [lookup_table.get(string, string.lower())] - def lemmatize_adj(self, string, morphology, lookup_table): + def lemmatize_adj( + self, string: str, morphology: dict, lookup_table: Dict[str, str] + ) -> List[str]: # this method utilizes different procedures for adjectives # with 'nie' and 'naj' prefixes if string[:3] == "nie": @@ -41,25 +42,26 @@ class PolishLemmatizer(Lemmatizer): return [lookup_table[naj_search_string]] if search_string in lookup_table: return [lookup_table[search_string]] - if string[:3] == "naj": naj_search_string = string[3:] if naj_search_string in lookup_table: return [lookup_table[naj_search_string]] - return [lookup_table.get(string, string)] - def lemmatize_verb(self, string, morphology, lookup_table): + def lemmatize_verb( + self, string: str, morphology: dict, lookup_table: Dict[str, str] + ) -> List[str]: # this method utilizes a different procedure for verbs # with 'nie' prefix if string[:3] == "nie": search_string = string[3:] if search_string in lookup_table: return [lookup_table[search_string]] - return [lookup_table.get(string, string)] - def lemmatize_noun(self, string, morphology, lookup_table): + def lemmatize_noun( + self, string: str, morphology: dict, lookup_table: Dict[str, str] + ) -> List[str]: # this method is case-sensitive, in order to work # for incorrectly tagged proper names if string != string.lower(): @@ -68,11 +70,16 @@ class PolishLemmatizer(Lemmatizer): elif string in lookup_table: return [lookup_table[string]] return [string.lower()] - return [lookup_table.get(string, string)] - def lookup(self, string, orth=None): + def lookup(self, string: str, orth: Optional[int] = None) -> str: return string.lower() - def lemmatize(self, string, index, exceptions, rules): + def lemmatize( + self, + string: str, + index: Dict[str, List[str]], + exceptions: Dict[str, Dict[str, List[str]]], + rules: Dict[str, List[List[str]]], + ) -> List[str]: raise NotImplementedError diff --git a/spacy/lang/pt/__init__.py b/spacy/lang/pt/__init__.py index c86cdcd48..6dc22ed61 100644 --- a/spacy/lang/pt/__init__.py +++ b/spacy/lang/pt/__init__.py @@ -1,20 +1,42 @@ +from typing import Set, Dict, Callable, Any +from thinc.api import Config + from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS from .stop_words import STOP_WORDS from .lex_attrs import LEX_ATTRS - from ..tokenizer_exceptions import BASE_EXCEPTIONS from .punctuation import TOKENIZER_INFIXES, TOKENIZER_PREFIXES from ...language import Language -from ...attrs import LANG -from ...util import update_exc +from ...util import update_exc, registry + + +DEFAULT_CONFIG = """ +[nlp] +lang = "pt" +stop_words = {"@language_data": "spacy.pt.stop_words"} +lex_attr_getters = {"@language_data": "spacy.pt.lex_attr_getters"} + +[nlp.lemmatizer] +@lemmatizers = "spacy.Lemmatizer.v1" + +[nlp.lemmatizer.data_paths] +@language_data = "spacy-lookups-data" +lang = ${nlp:lang} +""" + + +@registry.language_data("spacy.pt.stop_words") +def stop_words() -> Set[str]: + return STOP_WORDS + + +@registry.language_data("spacy.pt.lex_attr_getters") +def lex_attr_getters() -> Dict[int, Callable[[str], Any]]: + return LEX_ATTRS class PortugueseDefaults(Language.Defaults): - lex_attr_getters = dict(Language.Defaults.lex_attr_getters) - lex_attr_getters[LANG] = lambda text: "pt" - lex_attr_getters.update(LEX_ATTRS) tokenizer_exceptions = update_exc(BASE_EXCEPTIONS, TOKENIZER_EXCEPTIONS) - stop_words = STOP_WORDS infixes = TOKENIZER_INFIXES prefixes = TOKENIZER_PREFIXES @@ -22,6 +44,7 @@ class PortugueseDefaults(Language.Defaults): class Portuguese(Language): lang = "pt" Defaults = PortugueseDefaults + default_config = Config().from_str(DEFAULT_CONFIG) __all__ = ["Portuguese"] diff --git a/spacy/lang/ro/__init__.py b/spacy/lang/ro/__init__.py index a8afae945..b66b7767c 100644 --- a/spacy/lang/ro/__init__.py +++ b/spacy/lang/ro/__init__.py @@ -1,27 +1,40 @@ +from typing import Set +from thinc.api import Config + from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS from .stop_words import STOP_WORDS from .punctuation import TOKENIZER_PREFIXES, TOKENIZER_INFIXES from .punctuation import TOKENIZER_SUFFIXES - from ..tokenizer_exceptions import BASE_EXCEPTIONS -from ..norm_exceptions import BASE_NORMS from ...language import Language -from ...attrs import LANG, NORM -from ...util import update_exc, add_lookups +from ...util import update_exc, registry # Lemma data note: # Original pairs downloaded from http://www.lexiconista.com/datasets/lemmatization/ # Replaced characters using cedillas with the correct ones (ș and ț) +DEFAULT_CONFIG = """ +[nlp] +lang = "ro" +stop_words = {"@language_data": "spacy.ro.stop_words"} + +[nlp.lemmatizer] +@lemmatizers = "spacy.Lemmatizer.v1" + +[nlp.lemmatizer.data_paths] +@language_data = "spacy-lookups-data" +lang = ${nlp:lang} +""" + + +@registry.language_data("spacy.ro.stop_words") +def stop_words() -> Set[str]: + return STOP_WORDS + + class RomanianDefaults(Language.Defaults): - lex_attr_getters = dict(Language.Defaults.lex_attr_getters) - lex_attr_getters[LANG] = lambda text: "ro" - lex_attr_getters[NORM] = add_lookups( - Language.Defaults.lex_attr_getters[NORM], BASE_NORMS - ) tokenizer_exceptions = update_exc(BASE_EXCEPTIONS, TOKENIZER_EXCEPTIONS) - stop_words = STOP_WORDS prefixes = TOKENIZER_PREFIXES suffixes = TOKENIZER_SUFFIXES infixes = TOKENIZER_INFIXES @@ -30,6 +43,7 @@ class RomanianDefaults(Language.Defaults): class Romanian(Language): lang = "ro" Defaults = RomanianDefaults + default_config = Config().from_str(DEFAULT_CONFIG) __all__ = ["Romanian"] diff --git a/spacy/lang/ru/__init__.py b/spacy/lang/ru/__init__.py index 885e30529..004a8d83a 100644 --- a/spacy/lang/ru/__init__.py +++ b/spacy/lang/ru/__init__.py @@ -1,32 +1,49 @@ +from typing import Set, Dict, Callable, Any +from thinc.api import Config + from .stop_words import STOP_WORDS from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS from .lex_attrs import LEX_ATTRS from .lemmatizer import RussianLemmatizer - from ..tokenizer_exceptions import BASE_EXCEPTIONS -from ...util import update_exc +from ...util import update_exc, registry from ...language import Language -from ...lookups import Lookups -from ...attrs import LANG + + +DEFAULT_CONFIG = """ +[nlp] +lang = "ru" +stop_words = {"@language_data": "spacy.ru.stop_words"} +lex_attr_getters = {"@language_data": "spacy.ru.lex_attr_getters"} + +[nlp.lemmatizer] +@lemmatizers = "spacy.RussianLemmatizer.v1" +""" + + +@registry.language_data("spacy.ru.stop_words") +def stop_words() -> Set[str]: + return STOP_WORDS + + +@registry.language_data("spacy.ru.lex_attr_getters") +def lex_attr_getters() -> Dict[int, Callable[[str], Any]]: + return LEX_ATTRS + + +@registry.lemmatizers("spacy.RussianLemmatizer.v1") +def create_russian_lemmatizer() -> RussianLemmatizer: + return RussianLemmatizer() class RussianDefaults(Language.Defaults): - lex_attr_getters = dict(Language.Defaults.lex_attr_getters) - lex_attr_getters.update(LEX_ATTRS) - lex_attr_getters[LANG] = lambda text: "ru" tokenizer_exceptions = update_exc(BASE_EXCEPTIONS, TOKENIZER_EXCEPTIONS) - stop_words = STOP_WORDS - - @classmethod - def create_lemmatizer(cls, nlp=None, lookups=None): - if lookups is None: - lookups = Lookups() - return RussianLemmatizer(lookups) class Russian(Language): lang = "ru" Defaults = RussianDefaults + default_config = Config().from_str(DEFAULT_CONFIG) __all__ = ["Russian"] diff --git a/spacy/lang/ru/lemmatizer.py b/spacy/lang/ru/lemmatizer.py index ed0e858f5..a9a7ad80f 100644 --- a/spacy/lang/ru/lemmatizer.py +++ b/spacy/lang/ru/lemmatizer.py @@ -1,11 +1,17 @@ +from typing import Optional, Tuple, Dict, List + from ...symbols import ADJ, DET, NOUN, NUM, PRON, PROPN, PUNCT, VERB, POS from ...lemmatizer import Lemmatizer +from ...lookups import Lookups + + +PUNCT_RULES = {"«": '"', "»": '"'} class RussianLemmatizer(Lemmatizer): _morph = None - def __init__(self, lookups=None): + def __init__(self, lookups: Optional[Lookups] = None) -> None: super(RussianLemmatizer, self).__init__(lookups) try: from pymorphy2 import MorphAnalyzer @@ -19,15 +25,15 @@ class RussianLemmatizer(Lemmatizer): if RussianLemmatizer._morph is None: RussianLemmatizer._morph = MorphAnalyzer() - def __call__(self, string, univ_pos, morphology=None): + def __call__( + self, string: str, univ_pos: str, morphology: Optional[dict] = None + ) -> List[str]: univ_pos = self.normalize_univ_pos(univ_pos) if univ_pos == "PUNCT": return [PUNCT_RULES.get(string, string)] - if univ_pos not in ("ADJ", "DET", "NOUN", "NUM", "PRON", "PROPN", "VERB"): # Skip unchangeable pos return [string.lower()] - analyses = self._morph.parse(string) filtered_analyses = [] for analysis in analyses: @@ -39,12 +45,10 @@ class RussianLemmatizer(Lemmatizer): analysis_pos in ("NOUN", "PROPN") and univ_pos in ("NOUN", "PROPN") ): filtered_analyses.append(analysis) - if not len(filtered_analyses): return [string.lower()] if morphology is None or (len(morphology) == 1 and POS in morphology): return list(set([analysis.normal_form for analysis in filtered_analyses])) - if univ_pos in ("ADJ", "DET", "NOUN", "PROPN"): features_to_compare = ["Case", "Number", "Gender"] elif univ_pos == "NUM": @@ -61,7 +65,6 @@ class RussianLemmatizer(Lemmatizer): "VerbForm", "Voice", ] - analyses, filtered_analyses = filtered_analyses, [] for analysis in analyses: _, analysis_morph = oc2ud(str(analysis.tag)) @@ -74,16 +77,14 @@ class RussianLemmatizer(Lemmatizer): break else: filtered_analyses.append(analysis) - if not len(filtered_analyses): return [string.lower()] return list(set([analysis.normal_form for analysis in filtered_analyses])) @staticmethod - def normalize_univ_pos(univ_pos): + def normalize_univ_pos(univ_pos: str) -> Optional[str]: if isinstance(univ_pos, str): return univ_pos.upper() - symbols_to_str = { ADJ: "ADJ", DET: "DET", @@ -98,14 +99,14 @@ class RussianLemmatizer(Lemmatizer): return symbols_to_str[univ_pos] return None - def lookup(self, string, orth=None): + def lookup(self, string: str, orth: Optional[int] = None) -> str: analyses = self._morph.parse(string) if len(analyses) == 1: return analyses[0].normal_form return string -def oc2ud(oc_tag): +def oc2ud(oc_tag: str) -> Tuple[str, Dict[str, str]]: gram_map = { "_POS": { "ADJF": "ADJ", @@ -160,11 +161,9 @@ def oc2ud(oc_tag): "Voice": {"actv": "Act", "pssv": "Pass"}, "Abbr": {"Abbr": "Yes"}, } - pos = "X" morphology = dict() unmatched = set() - grams = oc_tag.replace(" ", ",").split(",") for gram in grams: match = False @@ -177,7 +176,6 @@ def oc2ud(oc_tag): morphology[categ] = gmap[gram] if not match: unmatched.add(gram) - while len(unmatched) > 0: gram = unmatched.pop() if gram in ("Name", "Patr", "Surn", "Geox", "Orgn"): @@ -186,8 +184,4 @@ def oc2ud(oc_tag): pos = "AUX" elif gram == "Pltm": morphology["Number"] = "Ptan" - return pos, morphology - - -PUNCT_RULES = {"«": '"', "»": '"'} diff --git a/spacy/lang/si/__init__.py b/spacy/lang/si/__init__.py index 3b065860c..69c4718c0 100644 --- a/spacy/lang/si/__init__.py +++ b/spacy/lang/si/__init__.py @@ -1,20 +1,33 @@ +from typing import Set, Dict, Callable, Any +from thinc.api import Config + from .stop_words import STOP_WORDS from .lex_attrs import LEX_ATTRS - from ...language import Language -from ...attrs import LANG +from ...util import registry -class SinhalaDefaults(Language.Defaults): - lex_attr_getters = dict(Language.Defaults.lex_attr_getters) - lex_attr_getters.update(LEX_ATTRS) - lex_attr_getters[LANG] = lambda text: "si" - stop_words = STOP_WORDS +DEFAULT_CONFIG = """ +[nlp] +lang = "si" +stop_words = {"@language_data": "spacy.si.stop_words"} +lex_attr_getters = {"@language_data": "spacy.si.lex_attr_getters"} +""" + + +@registry.language_data("spacy.si.stop_words") +def stop_words() -> Set[str]: + return STOP_WORDS + + +@registry.language_data("spacy.si.lex_attr_getters") +def lex_attr_getters() -> Dict[int, Callable[[str], Any]]: + return LEX_ATTRS class Sinhala(Language): lang = "si" - Defaults = SinhalaDefaults + default_config = Config().from_str(DEFAULT_CONFIG) __all__ = ["Sinhala"] diff --git a/spacy/lang/sk/__init__.py b/spacy/lang/sk/__init__.py index c10e186d1..c9493e829 100644 --- a/spacy/lang/sk/__init__.py +++ b/spacy/lang/sk/__init__.py @@ -1,20 +1,33 @@ +from typing import Set, Dict, Callable, Any +from thinc.api import Config + from .stop_words import STOP_WORDS from .lex_attrs import LEX_ATTRS - from ...language import Language -from ...attrs import LANG +from ...util import registry -class SlovakDefaults(Language.Defaults): - lex_attr_getters = dict(Language.Defaults.lex_attr_getters) - lex_attr_getters.update(LEX_ATTRS) - lex_attr_getters[LANG] = lambda text: "sk" - stop_words = STOP_WORDS +DEFAULT_CONFIG = """ +[nlp] +lang = "sk" +stop_words = {"@language_data": "spacy.sk.stop_words"} +lex_attr_getters = {"@language_data": "spacy.sk.lex_attr_getters"} +""" + + +@registry.language_data("spacy.sk.stop_words") +def stop_words() -> Set[str]: + return STOP_WORDS + + +@registry.language_data("spacy.sk.lex_attr_getters") +def lex_attr_getters() -> Dict[int, Callable[[str], Any]]: + return LEX_ATTRS class Slovak(Language): lang = "sk" - Defaults = SlovakDefaults + default_config = Config().from_str(DEFAULT_CONFIG) __all__ = ["Slovak"] diff --git a/spacy/lang/sl/__init__.py b/spacy/lang/sl/__init__.py index ce46e92dc..4f1954669 100644 --- a/spacy/lang/sl/__init__.py +++ b/spacy/lang/sl/__init__.py @@ -1,17 +1,26 @@ +from typing import Set +from thinc.api import Config + from .stop_words import STOP_WORDS from ...language import Language -from ...attrs import LANG +from ...util import registry -class SlovenianDefaults(Language.Defaults): - lex_attr_getters = dict(Language.Defaults.lex_attr_getters) - lex_attr_getters[LANG] = lambda text: "sl" - stop_words = STOP_WORDS +DEFAULT_CONFIG = """ +[nlp] +lang = "sl" +stop_words = {"@language_data": "spacy.sl.stop_words"} +""" + + +@registry.language_data("spacy.sl.stop_words") +def stop_words() -> Set[str]: + return STOP_WORDS class Slovenian(Language): lang = "sl" - Defaults = SlovenianDefaults + default_config = Config().from_str(DEFAULT_CONFIG) __all__ = ["Slovenian"] diff --git a/spacy/lang/sq/__init__.py b/spacy/lang/sq/__init__.py index 034604838..a3da6b354 100644 --- a/spacy/lang/sq/__init__.py +++ b/spacy/lang/sq/__init__.py @@ -1,17 +1,26 @@ +from typing import Set +from thinc.api import Config + from .stop_words import STOP_WORDS from ...language import Language -from ...attrs import LANG +from ...util import registry -class AlbanianDefaults(Language.Defaults): - lex_attr_getters = dict(Language.Defaults.lex_attr_getters) - lex_attr_getters[LANG] = lambda text: "sq" - stop_words = STOP_WORDS +DEFAULT_CONFIG = """ +[nlp] +lang = "sq" +stop_words = {"@language_data": "spacy.sq.stop_words"} +""" + + +@registry.language_data("spacy.sq.stop_words") +def stop_words() -> Set[str]: + return STOP_WORDS class Albanian(Language): lang = "sq" - Defaults = AlbanianDefaults + default_config = Config().from_str(DEFAULT_CONFIG) __all__ = ["Albanian"] diff --git a/spacy/lang/sr/__init__.py b/spacy/lang/sr/__init__.py index 7f2172707..fd53d3826 100644 --- a/spacy/lang/sr/__init__.py +++ b/spacy/lang/sr/__init__.py @@ -1,23 +1,47 @@ +from typing import Set, Dict, Callable, Any +from thinc.api import Config + from .stop_words import STOP_WORDS from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS from .lex_attrs import LEX_ATTRS from ..tokenizer_exceptions import BASE_EXCEPTIONS from ...language import Language -from ...attrs import LANG -from ...util import update_exc +from ...util import update_exc, registry + + +DEFAULT_CONFIG = """ +[nlp] +lang = "sr" +stop_words = {"@language_data": "spacy.sr.stop_words"} +lex_attr_getters = {"@language_data": "spacy.sr.lex_attr_getters"} + +[nlp.lemmatizer] +@lemmatizers = "spacy.Lemmatizer.v1" + +[nlp.lemmatizer.data_paths] +@language_data = "spacy-lookups-data" +lang = ${nlp:lang} +""" + + +@registry.language_data("spacy.sr.stop_words") +def stop_words() -> Set[str]: + return STOP_WORDS + + +@registry.language_data("spacy.sr.lex_attr_getters") +def lex_attr_getters() -> Dict[int, Callable[[str], Any]]: + return LEX_ATTRS class SerbianDefaults(Language.Defaults): - lex_attr_getters = dict(Language.Defaults.lex_attr_getters) - lex_attr_getters.update(LEX_ATTRS) - lex_attr_getters[LANG] = lambda text: "sr" tokenizer_exceptions = update_exc(BASE_EXCEPTIONS, TOKENIZER_EXCEPTIONS) - stop_words = STOP_WORDS class Serbian(Language): lang = "sr" Defaults = SerbianDefaults + default_config = Config().from_str(DEFAULT_CONFIG) __all__ = ["Serbian"] diff --git a/spacy/lang/sv/__init__.py b/spacy/lang/sv/__init__.py index 9dcdc543d..5c376fd51 100644 --- a/spacy/lang/sv/__init__.py +++ b/spacy/lang/sv/__init__.py @@ -1,35 +1,54 @@ +from typing import Set, Dict, Callable, Any +from thinc.api import Config + from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS from .stop_words import STOP_WORDS from .lex_attrs import LEX_ATTRS +from ..tokenizer_exceptions import BASE_EXCEPTIONS +from ...language import Language +from ...util import update_exc, registry +from .syntax_iterators import SYNTAX_ITERATORS # Punctuation stolen from Danish from ..da.punctuation import TOKENIZER_INFIXES, TOKENIZER_SUFFIXES -from ..tokenizer_exceptions import BASE_EXCEPTIONS -from ..norm_exceptions import BASE_NORMS -from ...language import Language -from ...attrs import LANG, NORM -from ...util import update_exc, add_lookups -from .syntax_iterators import SYNTAX_ITERATORS + +DEFAULT_CONFIG = """ +[nlp] +lang = "sv" +stop_words = {"@language_data": "spacy.sv.stop_words"} +lex_attr_getters = {"@language_data": "spacy.sv.lex_attr_getters"} + +[nlp.lemmatizer] +@lemmatizers = "spacy.Lemmatizer.v1" + +[nlp.lemmatizer.data_paths] +@language_data = "spacy-lookups-data" +lang = ${nlp:lang} +""" + + +@registry.language_data("spacy.sv.stop_words") +def stop_words() -> Set[str]: + return STOP_WORDS + + +@registry.language_data("spacy.sv.lex_attr_getters") +def lex_attr_getters() -> Dict[int, Callable[[str], Any]]: + return LEX_ATTRS class SwedishDefaults(Language.Defaults): - lex_attr_getters = dict(Language.Defaults.lex_attr_getters) - lex_attr_getters.update(LEX_ATTRS) - lex_attr_getters[LANG] = lambda text: "sv" - lex_attr_getters[NORM] = add_lookups( - Language.Defaults.lex_attr_getters[NORM], BASE_NORMS - ) tokenizer_exceptions = update_exc(BASE_EXCEPTIONS, TOKENIZER_EXCEPTIONS) infixes = TOKENIZER_INFIXES suffixes = TOKENIZER_SUFFIXES - stop_words = STOP_WORDS syntax_iterators = SYNTAX_ITERATORS class Swedish(Language): lang = "sv" Defaults = SwedishDefaults + default_config = Config().from_str(DEFAULT_CONFIG) __all__ = ["Swedish"] diff --git a/spacy/lang/ta/__init__.py b/spacy/lang/ta/__init__.py index d7a04afea..983bd5de4 100644 --- a/spacy/lang/ta/__init__.py +++ b/spacy/lang/ta/__init__.py @@ -1,20 +1,33 @@ +from typing import Set, Dict, Callable, Any +from thinc.api import Config + from .stop_words import STOP_WORDS from .lex_attrs import LEX_ATTRS - from ...language import Language -from ...attrs import LANG +from ...util import registry -class TamilDefaults(Language.Defaults): - lex_attr_getters = dict(Language.Defaults.lex_attr_getters) - lex_attr_getters[LANG] = lambda text: "ta" - lex_attr_getters.update(LEX_ATTRS) - stop_words = STOP_WORDS +DEFAULT_CONFIG = """ +[nlp] +lang = "ta" +stop_words = {"@language_data": "spacy.ta.stop_words"} +lex_attr_getters = {"@language_data": "spacy.ta.lex_attr_getters"} +""" + + +@registry.language_data("spacy.ta.stop_words") +def stop_words() -> Set[str]: + return STOP_WORDS + + +@registry.language_data("spacy.ta.lex_attr_getters") +def lex_attr_getters() -> Dict[int, Callable[[str], Any]]: + return LEX_ATTRS class Tamil(Language): lang = "ta" - Defaults = TamilDefaults + default_config = Config().from_str(DEFAULT_CONFIG) __all__ = ["Tamil"] diff --git a/spacy/lang/te/__init__.py b/spacy/lang/te/__init__.py index 424164cc7..d012d418d 100644 --- a/spacy/lang/te/__init__.py +++ b/spacy/lang/te/__init__.py @@ -1,20 +1,33 @@ +from typing import Set, Dict, Callable, Any +from thinc.api import Config + from .stop_words import STOP_WORDS from .lex_attrs import LEX_ATTRS - from ...language import Language -from ...attrs import LANG +from ...util import registry -class TeluguDefaults(Language.Defaults): - lex_attr_getters = dict(Language.Defaults.lex_attr_getters) - lex_attr_getters.update(LEX_ATTRS) - lex_attr_getters[LANG] = lambda text: "te" - stop_words = STOP_WORDS +DEFAULT_CONFIG = """ +[nlp] +lang = "te" +stop_words = {"@language_data": "spacy.te.stop_words"} +lex_attr_getters = {"@language_data": "spacy.te.lex_attr_getters"} +""" + + +@registry.language_data("spacy.te.stop_words") +def stop_words() -> Set[str]: + return STOP_WORDS + + +@registry.language_data("spacy.te.lex_attr_getters") +def lex_attr_getters() -> Dict[int, Callable[[str], Any]]: + return LEX_ATTRS class Telugu(Language): lang = "te" - Defaults = TeluguDefaults + default_config = Config().from_str(DEFAULT_CONFIG) __all__ = ["Telugu"] diff --git a/spacy/lang/th/__init__.py b/spacy/lang/th/__init__.py index 6c7a56693..116355342 100644 --- a/spacy/lang/th/__init__.py +++ b/spacy/lang/th/__init__.py @@ -1,15 +1,44 @@ -from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS +from typing import Set, Dict, Callable, Any +from thinc.api import Config + from .stop_words import STOP_WORDS from .lex_attrs import LEX_ATTRS - -from ...attrs import LANG from ...language import Language from ...tokens import Doc -from ...util import DummyTokenizer +from ...util import DummyTokenizer, registry + + +DEFAULT_CONFIG = """ +[nlp] +lang = "th" +stop_words = {"@language_data": "spacy.th.stop_words"} +lex_attr_getters = {"@language_data": "spacy.th.lex_attr_getters"} + +[nlp.tokenizer] +@tokenizers = "spacy.ThaiTokenizer.v1" +""" + + +@registry.language_data("spacy.th.stop_words") +def stop_words() -> Set[str]: + return STOP_WORDS + + +@registry.language_data("spacy.th.lex_attr_getters") +def lex_attr_getters() -> Dict[int, Callable[[str], Any]]: + return LEX_ATTRS + + +@registry.tokenizers("spacy.ThaiTokenizer.v1") +def create_thai_tokenizer(): + def thai_tokenizer_factory(nlp): + return ThaiTokenizer(nlp) + + return thai_tokenizer_factory class ThaiTokenizer(DummyTokenizer): - def __init__(self, cls, nlp=None): + def __init__(self, nlp: Language) -> None: try: from pythainlp.tokenize import word_tokenize except ImportError: @@ -17,34 +46,18 @@ class ThaiTokenizer(DummyTokenizer): "The Thai tokenizer requires the PyThaiNLP library: " "https://github.com/PyThaiNLP/pythainlp" ) - self.word_tokenize = word_tokenize - self.vocab = nlp.vocab if nlp is not None else cls.create_vocab(nlp) + self.vocab = nlp.vocab - def __call__(self, text): + def __call__(self, text: str) -> Doc: words = list(self.word_tokenize(text)) spaces = [False] * len(words) return Doc(self.vocab, words=words, spaces=spaces) -class ThaiDefaults(Language.Defaults): - lex_attr_getters = dict(Language.Defaults.lex_attr_getters) - lex_attr_getters.update(LEX_ATTRS) - lex_attr_getters[LANG] = lambda _text: "th" - tokenizer_exceptions = dict(TOKENIZER_EXCEPTIONS) - stop_words = STOP_WORDS - - @classmethod - def create_tokenizer(cls, nlp=None): - return ThaiTokenizer(cls, nlp) - - class Thai(Language): lang = "th" - Defaults = ThaiDefaults - - def make_doc(self, text): - return self.tokenizer(text) + default_config = Config().from_str(DEFAULT_CONFIG) __all__ = ["Thai"] diff --git a/spacy/lang/tl/__init__.py b/spacy/lang/tl/__init__.py index f477029f7..c52adb046 100644 --- a/spacy/lang/tl/__init__.py +++ b/spacy/lang/tl/__init__.py @@ -1,31 +1,47 @@ +from typing import Set, Dict, Callable, Any +from thinc.api import Config + from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS from .stop_words import STOP_WORDS from .lex_attrs import LEX_ATTRS from ..tokenizer_exceptions import BASE_EXCEPTIONS -from ..norm_exceptions import BASE_NORMS from ...language import Language -from ...attrs import LANG, NORM -from ...util import update_exc, add_lookups +from ...util import update_exc, registry -def _return_tl(_): - return "tl" +DEFAULT_CONFIG = """ +[nlp] +lang = "tl" +stop_words = {"@language_data": "spacy.tl.stop_words"} +lex_attr_getters = {"@language_data": "spacy.tl.lex_attr_getters"} + +[nlp.lemmatizer] +@lemmatizers = "spacy.Lemmatizer.v1" + +[nlp.lemmatizer.data_paths] +@language_data = "spacy-lookups-data" +lang = ${nlp:lang} +""" + + +@registry.language_data("spacy.tl.stop_words") +def stop_words() -> Set[str]: + return STOP_WORDS + + +@registry.language_data("spacy.tl.lex_attr_getters") +def lex_attr_getters() -> Dict[int, Callable[[str], Any]]: + return LEX_ATTRS class TagalogDefaults(Language.Defaults): - lex_attr_getters = dict(Language.Defaults.lex_attr_getters) - lex_attr_getters[LANG] = _return_tl - lex_attr_getters[NORM] = add_lookups( - Language.Defaults.lex_attr_getters[NORM], BASE_NORMS - ) - lex_attr_getters.update(LEX_ATTRS) tokenizer_exceptions = update_exc(BASE_EXCEPTIONS, TOKENIZER_EXCEPTIONS) - stop_words = STOP_WORDS class Tagalog(Language): lang = "tl" Defaults = TagalogDefaults + default_config = Config().from_str(DEFAULT_CONFIG) __all__ = ["Tagalog"] diff --git a/spacy/lang/tr/__init__.py b/spacy/lang/tr/__init__.py index a29d78261..f6782b419 100644 --- a/spacy/lang/tr/__init__.py +++ b/spacy/lang/tr/__init__.py @@ -1,26 +1,40 @@ +from typing import Set +from thinc.api import Config + from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS from .stop_words import STOP_WORDS - from ..tokenizer_exceptions import BASE_EXCEPTIONS -from ..norm_exceptions import BASE_NORMS from ...language import Language -from ...attrs import LANG, NORM -from ...util import update_exc, add_lookups +from ...util import update_exc, registry + + +DEFAULT_CONFIG = """ +[nlp] +lang = "tr" +stop_words = {"@language_data": "spacy.tr.stop_words"} + +[nlp.lemmatizer] +@lemmatizers = "spacy.Lemmatizer.v1" + +[nlp.lemmatizer.data_paths] +@language_data = "spacy-lookups-data" +lang = ${nlp:lang} +""" + + +@registry.language_data("spacy.tr.stop_words") +def stop_words() -> Set[str]: + return STOP_WORDS class TurkishDefaults(Language.Defaults): - lex_attr_getters = dict(Language.Defaults.lex_attr_getters) - lex_attr_getters[LANG] = lambda text: "tr" - lex_attr_getters[NORM] = add_lookups( - Language.Defaults.lex_attr_getters[NORM], BASE_NORMS - ) tokenizer_exceptions = update_exc(BASE_EXCEPTIONS, TOKENIZER_EXCEPTIONS) - stop_words = STOP_WORDS class Turkish(Language): lang = "tr" Defaults = TurkishDefaults + default_config = Config().from_str(DEFAULT_CONFIG) __all__ = ["Turkish"] diff --git a/spacy/lang/tt/__init__.py b/spacy/lang/tt/__init__.py index 80574a70d..45f9a24b0 100644 --- a/spacy/lang/tt/__init__.py +++ b/spacy/lang/tt/__init__.py @@ -1,28 +1,42 @@ +from typing import Set, Dict, Callable, Any +from thinc.api import Config + from .lex_attrs import LEX_ATTRS from .punctuation import TOKENIZER_INFIXES from .stop_words import STOP_WORDS from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS from ..tokenizer_exceptions import BASE_EXCEPTIONS -from ...attrs import LANG from ...language import Language -from ...util import update_exc +from ...util import update_exc, registry + + +DEFAULT_CONFIG = """ +[nlp] +lang = "tt" +stop_words = {"@language_data": "spacy.tt.stop_words"} +lex_attr_getters = {"@language_data": "spacy.tt.lex_attr_getters"} +""" + + +@registry.language_data("spacy.tt.stop_words") +def stop_words() -> Set[str]: + return STOP_WORDS + + +@registry.language_data("spacy.tt.lex_attr_getters") +def lex_attr_getters() -> Dict[int, Callable[[str], Any]]: + return LEX_ATTRS class TatarDefaults(Language.Defaults): - lex_attr_getters = dict(Language.Defaults.lex_attr_getters) - lex_attr_getters[LANG] = lambda text: "tt" - - lex_attr_getters.update(LEX_ATTRS) - tokenizer_exceptions = update_exc(BASE_EXCEPTIONS, TOKENIZER_EXCEPTIONS) infixes = tuple(TOKENIZER_INFIXES) - stop_words = STOP_WORDS - class Tatar(Language): lang = "tt" Defaults = TatarDefaults + default_config = Config().from_str(DEFAULT_CONFIG) __all__ = ["Tatar"] diff --git a/spacy/lang/uk/__init__.py b/spacy/lang/uk/__init__.py index 51165112a..72b70caa9 100644 --- a/spacy/lang/uk/__init__.py +++ b/spacy/lang/uk/__init__.py @@ -1,36 +1,49 @@ +from typing import Set, Dict, Callable, Any +from thinc.api import Config + from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS from .stop_words import STOP_WORDS from .lex_attrs import LEX_ATTRS - from ..tokenizer_exceptions import BASE_EXCEPTIONS -from ..norm_exceptions import BASE_NORMS -from ...util import update_exc, add_lookups +from ...util import update_exc, registry from ...language import Language -from ...lookups import Lookups -from ...attrs import LANG, NORM from .lemmatizer import UkrainianLemmatizer -class UkrainianDefaults(Language.Defaults): - lex_attr_getters = dict(Language.Defaults.lex_attr_getters) - lex_attr_getters[LANG] = lambda text: "uk" - lex_attr_getters[NORM] = add_lookups( - Language.Defaults.lex_attr_getters[NORM], BASE_NORMS - ) - lex_attr_getters.update(LEX_ATTRS) - tokenizer_exceptions = update_exc(BASE_EXCEPTIONS, TOKENIZER_EXCEPTIONS) - stop_words = STOP_WORDS +DEFAULT_CONFIG = """ +[nlp] +lang = "uk" +stop_words = {"@language_data": "spacy.uk.stop_words"} +lex_attr_getters = {"@language_data": "spacy.uk.lex_attr_getters"} - @classmethod - def create_lemmatizer(cls, nlp=None, lookups=None): - if lookups is None: - lookups = Lookups() - return UkrainianLemmatizer(lookups) +[nlp.lemmatizer] +@lemmatizers = "spacy.UkrainianLemmatizer.v1" +""" + + +@registry.language_data("spacy.uk.stop_words") +def stop_words() -> Set[str]: + return STOP_WORDS + + +@registry.language_data("spacy.uk.lex_attr_getters") +def lex_attr_getters() -> Dict[int, Callable[[str], Any]]: + return LEX_ATTRS + + +@registry.lemmatizers("spacy.UkrainianLemmatizer.v1") +def create_ukrainian_lemmatizer() -> UkrainianLemmatizer: + return UkrainianLemmatizer() + + +class UkrainianDefaults(Language.Defaults): + tokenizer_exceptions = update_exc(BASE_EXCEPTIONS, TOKENIZER_EXCEPTIONS) class Ukrainian(Language): lang = "uk" Defaults = UkrainianDefaults + default_config = Config().from_str(DEFAULT_CONFIG) __all__ = ["Ukrainian"] diff --git a/spacy/lang/uk/lemmatizer.py b/spacy/lang/uk/lemmatizer.py index ff61d711f..de2d0c170 100644 --- a/spacy/lang/uk/lemmatizer.py +++ b/spacy/lang/uk/lemmatizer.py @@ -1,11 +1,17 @@ +from typing import Optional, List, Tuple, Dict + from ...symbols import ADJ, DET, NOUN, NUM, PRON, PROPN, PUNCT, VERB, POS +from ...lookups import Lookups from ...lemmatizer import Lemmatizer +PUNCT_RULES = {"«": '"', "»": '"'} + + class UkrainianLemmatizer(Lemmatizer): _morph = None - def __init__(self, lookups=None): + def __init__(self, lookups: Optional[Lookups] = None) -> None: super(UkrainianLemmatizer, self).__init__(lookups) try: from pymorphy2 import MorphAnalyzer @@ -19,15 +25,15 @@ class UkrainianLemmatizer(Lemmatizer): '"pip install git+https://github.com/kmike/pymorphy2.git pymorphy2-dicts-uk"' ) - def __call__(self, string, univ_pos, morphology=None): + def __call__( + self, string: str, univ_pos: str, morphology: Optional[dict] = None + ) -> List[str]: univ_pos = self.normalize_univ_pos(univ_pos) if univ_pos == "PUNCT": return [PUNCT_RULES.get(string, string)] - if univ_pos not in ("ADJ", "DET", "NOUN", "NUM", "PRON", "PROPN", "VERB"): # Skip unchangeable pos return [string.lower()] - analyses = self._morph.parse(string) filtered_analyses = [] for analysis in analyses: @@ -39,12 +45,10 @@ class UkrainianLemmatizer(Lemmatizer): analysis_pos in ("NOUN", "PROPN") and univ_pos in ("NOUN", "PROPN") ): filtered_analyses.append(analysis) - if not len(filtered_analyses): return [string.lower()] if morphology is None or (len(morphology) == 1 and POS in morphology): return list(set([analysis.normal_form for analysis in filtered_analyses])) - if univ_pos in ("ADJ", "DET", "NOUN", "PROPN"): features_to_compare = ["Case", "Number", "Gender"] elif univ_pos == "NUM": @@ -61,7 +65,6 @@ class UkrainianLemmatizer(Lemmatizer): "VerbForm", "Voice", ] - analyses, filtered_analyses = filtered_analyses, [] for analysis in analyses: _, analysis_morph = oc2ud(str(analysis.tag)) @@ -74,16 +77,14 @@ class UkrainianLemmatizer(Lemmatizer): break else: filtered_analyses.append(analysis) - if not len(filtered_analyses): return [string.lower()] return list(set([analysis.normal_form for analysis in filtered_analyses])) @staticmethod - def normalize_univ_pos(univ_pos): + def normalize_univ_pos(univ_pos: str) -> Optional[str]: if isinstance(univ_pos, str): return univ_pos.upper() - symbols_to_str = { ADJ: "ADJ", DET: "DET", @@ -98,14 +99,14 @@ class UkrainianLemmatizer(Lemmatizer): return symbols_to_str[univ_pos] return None - def lookup(self, string, orth=None): + def lookup(self, string: str, orth: Optional[int] = None) -> str: analyses = self._morph.parse(string) if len(analyses) == 1: return analyses[0].normal_form return string -def oc2ud(oc_tag): +def oc2ud(oc_tag: str) -> Tuple[str, Dict[str, str]]: gram_map = { "_POS": { "ADJF": "ADJ", @@ -160,11 +161,9 @@ def oc2ud(oc_tag): "Voice": {"actv": "Act", "pssv": "Pass"}, "Abbr": {"Abbr": "Yes"}, } - pos = "X" morphology = dict() unmatched = set() - grams = oc_tag.replace(" ", ",").split(",") for gram in grams: match = False @@ -177,7 +176,6 @@ def oc2ud(oc_tag): morphology[categ] = gmap[gram] if not match: unmatched.add(gram) - while len(unmatched) > 0: gram = unmatched.pop() if gram in ("Name", "Patr", "Surn", "Geox", "Orgn"): @@ -186,8 +184,4 @@ def oc2ud(oc_tag): pos = "AUX" elif gram == "Pltm": morphology["Number"] = "Ptan" - return pos, morphology - - -PUNCT_RULES = {"«": '"', "»": '"'} diff --git a/spacy/lang/ur/__init__.py b/spacy/lang/ur/__init__.py index c835166df..c7977d6b8 100644 --- a/spacy/lang/ur/__init__.py +++ b/spacy/lang/ur/__init__.py @@ -1,26 +1,53 @@ +from typing import Set, Dict, Callable, Any +from thinc.api import Config + from .stop_words import STOP_WORDS from .lex_attrs import LEX_ATTRS from .punctuation import TOKENIZER_SUFFIXES - from ..tokenizer_exceptions import BASE_EXCEPTIONS from ...language import Language -from ...attrs import LANG +from ...util import registry + + +DEFAULT_CONFIG = """ +[nlp] +lang = "ur" +stop_words = {"@language_data": "spacy.ur.stop_words"} +lex_attr_getters = {"@language_data": "spacy.ur.lex_attr_getters"} + +[nlp.writing_system] +direction = "rtl" +has_case = false +has_letters = true + +[nlp.lemmatizer] +@lemmatizers = "spacy.Lemmatizer.v1" + +[nlp.lemmatizer.data_paths] +@language_data = "spacy-lookups-data" +lang = ${nlp:lang} +""" + + +@registry.language_data("spacy.ur.stop_words") +def stop_words() -> Set[str]: + return STOP_WORDS + + +@registry.language_data("spacy.ur.lex_attr_getters") +def lex_attr_getters() -> Dict[int, Callable[[str], Any]]: + return LEX_ATTRS class UrduDefaults(Language.Defaults): - lex_attr_getters = dict(Language.Defaults.lex_attr_getters) - lex_attr_getters.update(LEX_ATTRS) - lex_attr_getters[LANG] = lambda text: "ur" - tokenizer_exceptions = BASE_EXCEPTIONS - stop_words = STOP_WORDS suffixes = TOKENIZER_SUFFIXES - writing_system = {"direction": "rtl", "has_case": False, "has_letters": True} class Urdu(Language): lang = "ur" Defaults = UrduDefaults + default_config = Config().from_str(DEFAULT_CONFIG) __all__ = ["Urdu"] diff --git a/spacy/lang/vi/__init__.py b/spacy/lang/vi/__init__.py index 7496763ee..2003e904b 100644 --- a/spacy/lang/vi/__init__.py +++ b/spacy/lang/vi/__init__.py @@ -1,38 +1,62 @@ -from ...attrs import LANG, NORM -from ..norm_exceptions import BASE_NORMS +from typing import Set, Dict, Callable, Any +from thinc.api import Config + from ...language import Language from ...tokens import Doc from .stop_words import STOP_WORDS -from ...util import add_lookups +from ...util import DummyTokenizer, registry from .lex_attrs import LEX_ATTRS -class VietnameseDefaults(Language.Defaults): - lex_attr_getters = dict(Language.Defaults.lex_attr_getters) - lex_attr_getters[LANG] = lambda text: "vi" # for pickling - lex_attr_getters[NORM] = add_lookups( - Language.Defaults.lex_attr_getters[NORM], BASE_NORMS - ) - lex_attr_getters.update(LEX_ATTRS) - stop_words = STOP_WORDS - use_pyvi = True +DEFAULT_CONFIG = """ +[nlp] +lang = "vi" +stop_words = {"@language_data": "spacy.vi.stop_words"} +lex_attr_getters = {"@language_data": "spacy.vi.lex_attr_getters"} + +[nlp.tokenizer] +@tokenizers = "spacy.VietnameseTokenizer.v1" +use_pyvi = true +""" -class Vietnamese(Language): - lang = "vi" - Defaults = VietnameseDefaults # override defaults +@registry.language_data("spacy.vi.stop_words") +def stop_words() -> Set[str]: + return STOP_WORDS - def make_doc(self, text): - if self.Defaults.use_pyvi: + +@registry.language_data("spacy.vi.lex_attr_getters") +def lex_attr_getters() -> Dict[int, Callable[[str], Any]]: + return LEX_ATTRS + + +@registry.tokenizers("spacy.VietnameseTokenizer.v1") +def create_vietnamese_tokenizer(use_pyvi: bool = True,): + def vietnamese_tokenizer_factory(nlp): + return VietnameseTokenizer(nlp, use_pyvi=use_pyvi) + + return vietnamese_tokenizer_factory + + +class VietnameseTokenizer(DummyTokenizer): + def __init__(self, nlp: Language, use_pyvi: bool = False): + self.vocab = nlp.vocab + self.use_pyvi = use_pyvi + if self.use_pyvi: try: from pyvi import ViTokenizer + + self.ViTokenizer = ViTokenizer except ImportError: msg = ( - "Pyvi not installed. Either set Vietnamese.use_pyvi = False, " + "Pyvi not installed. Either set use_pyvi = False, " "or install it https://pypi.python.org/pypi/pyvi" ) raise ImportError(msg) - words, spaces = ViTokenizer.spacy_tokenize(text) + + def __call__(self, text: str) -> Doc: + if self.use_pyvi: + words, spaces = self.ViTokenizer.spacy_tokenize(text) return Doc(self.vocab, words=words, spaces=spaces) else: words = [] @@ -44,4 +68,9 @@ class Vietnamese(Language): return Doc(self.vocab, words=words, spaces=spaces) +class Vietnamese(Language): + lang = "vi" + default_config = Config().from_str(DEFAULT_CONFIG) + + __all__ = ["Vietnamese"] diff --git a/spacy/lang/xx/__init__.py b/spacy/lang/xx/__init__.py index 347c624fd..2167d9a5e 100644 --- a/spacy/lang/xx/__init__.py +++ b/spacy/lang/xx/__init__.py @@ -1,17 +1,17 @@ +from thinc.api import Config + from ..tokenizer_exceptions import BASE_EXCEPTIONS -from ..norm_exceptions import BASE_NORMS from ...language import Language -from ...attrs import LANG, NORM -from ...util import update_exc, add_lookups + + +DEFAULT_CONFIG = """ +[nlp] +lang = "xx" +""" class MultiLanguageDefaults(Language.Defaults): - lex_attr_getters = dict(Language.Defaults.lex_attr_getters) - lex_attr_getters[LANG] = lambda text: "xx" - lex_attr_getters[NORM] = add_lookups( - Language.Defaults.lex_attr_getters[NORM], BASE_NORMS - ) - tokenizer_exceptions = update_exc(BASE_EXCEPTIONS) + tokenizer_exceptions = BASE_EXCEPTIONS class MultiLanguage(Language): @@ -21,6 +21,7 @@ class MultiLanguage(Language): lang = "xx" Defaults = MultiLanguageDefaults + default_config = Config().from_str(DEFAULT_CONFIG) __all__ = ["MultiLanguage"] diff --git a/spacy/lang/yo/__init__.py b/spacy/lang/yo/__init__.py index 08e3166e1..b739ffbd7 100644 --- a/spacy/lang/yo/__init__.py +++ b/spacy/lang/yo/__init__.py @@ -1,21 +1,39 @@ +from typing import Set, Dict, Callable, Any +from thinc.api import Config + from .stop_words import STOP_WORDS from .lex_attrs import LEX_ATTRS from ..tokenizer_exceptions import BASE_EXCEPTIONS from ...language import Language -from ...attrs import LANG +from ...util import registry + + +DEFAULT_CONFIG = """ +[nlp] +lang = "si" +stop_words = {"@language_data": "spacy.yo.stop_words"} +lex_attr_getters = {"@language_data": "spacy.yo.lex_attr_getters"} +""" + + +@registry.language_data("spacy.yo.stop_words") +def stop_words() -> Set[str]: + return STOP_WORDS + + +@registry.language_data("spacy.yo.lex_attr_getters") +def lex_attr_getters() -> Dict[int, Callable[[str], Any]]: + return LEX_ATTRS class YorubaDefaults(Language.Defaults): - lex_attr_getters = dict(Language.Defaults.lex_attr_getters) - lex_attr_getters.update(LEX_ATTRS) - lex_attr_getters[LANG] = lambda text: "yo" - stop_words = STOP_WORDS tokenizer_exceptions = BASE_EXCEPTIONS class Yoruba(Language): lang = "yo" Defaults = YorubaDefaults + default_config = Config().from_str(DEFAULT_CONFIG) __all__ = ["Yoruba"] diff --git a/spacy/lang/zh/__init__.py b/spacy/lang/zh/__init__.py index 2eec7acfe..ba5489dfd 100644 --- a/spacy/lang/zh/__init__.py +++ b/spacy/lang/zh/__init__.py @@ -1,13 +1,15 @@ +from typing import Optional, List, Set, Dict, Callable, Any +from enum import Enum import tempfile import srsly import warnings from pathlib import Path -from collections import OrderedDict -from ...attrs import LANG +from thinc.api import Config + from ...errors import Warnings, Errors from ...language import Language from ...tokens import Doc -from ...util import DummyTokenizer +from ...util import DummyTokenizer, registry from ..tokenizer_exceptions import BASE_EXCEPTIONS from .lex_attrs import LEX_ATTRS from .stop_words import STOP_WORDS @@ -16,88 +18,103 @@ from ... import util _PKUSEG_INSTALL_MSG = "install it with `pip install pkuseg==0.0.25` or from https://github.com/lancopku/pkuseg-python" +DEFAULT_CONFIG = """ +[nlp] +lang = "zh" +stop_words = {"@language_data": "spacy.zh.stop_words"} +lex_attr_getters = {"@language_data": "spacy.zh.lex_attr_getters"} -def try_jieba_import(segmenter): - try: - import jieba +[nlp.tokenizer] +@tokenizers = "spacy.ChineseTokenizer.v1" +segmenter = "char" +pkuseg_model = null +pkuseg_user_dict = "default" - if segmenter == "jieba": - # segment a short text to have jieba initialize its cache in advance - list(jieba.cut("作为", cut_all=False)) - - return jieba - except ImportError: - if segmenter == "jieba": - msg = ( - "Jieba not installed. To use jieba, install it with `pip " - " install jieba` or from https://github.com/fxsjy/jieba" - ) - raise ImportError(msg) +[nlp.writing_system] +direction = "ltr" +has_case = false +has_letters = false +""" -def try_pkuseg_import(segmenter, pkuseg_model, pkuseg_user_dict): - try: - import pkuseg +class Segmenter(str, Enum): + char = "char" + jieba = "jieba" + pkuseg = "pkuseg" - if pkuseg_model: - return pkuseg.pkuseg(pkuseg_model, pkuseg_user_dict) - elif segmenter == "pkuseg": - msg = ( - "The Chinese word segmenter is 'pkuseg' but no pkuseg model " - "was specified. Please provide the name of a pretrained model " - "or the path to a model with " - '`cfg = {"segmenter": "pkuseg", "pkuseg_model": name_or_path}; ' - 'nlp = Chinese(meta={"tokenizer": {"config": cfg}})`' - ) - raise ValueError(msg) - except ImportError: - if segmenter == "pkuseg": - msg = "pkuseg not installed. To use pkuseg, " + _PKUSEG_INSTALL_MSG - raise ImportError(msg) - except FileNotFoundError: - if segmenter == "pkuseg": - msg = "Unable to load pkuseg model from: " + pkuseg_model - raise FileNotFoundError(msg) + @classmethod + def values(cls): + return list(cls.__members__.keys()) + + +@registry.language_data("spacy.zh.stop_words") +def stop_words() -> Set[str]: + return STOP_WORDS + + +@registry.language_data("spacy.zh.lex_attr_getters") +def lex_attr_getters() -> Dict[int, Callable[[str], Any]]: + return LEX_ATTRS + + +@registry.tokenizers("spacy.ChineseTokenizer.v1") +def create_chinese_tokenizer( + segmenter: Segmenter = Segmenter.char, + pkuseg_model: Optional[str] = None, + pkuseg_user_dict: Optional[str] = "default", +): + def chinese_tokenizer_factory(nlp): + return ChineseTokenizer( + nlp, + segmenter=segmenter, + pkuseg_model=pkuseg_model, + pkuseg_user_dict=pkuseg_user_dict, + ) + + return chinese_tokenizer_factory class ChineseTokenizer(DummyTokenizer): - def __init__(self, cls, nlp=None, config={}): - self.supported_segmenters = ("char", "jieba", "pkuseg") - self.configure_segmenter(config) - self.vocab = nlp.vocab if nlp is not None else cls.create_vocab(nlp) - # remove relevant settings from config so they're not also saved in - # Language.meta - for key in ["segmenter", "pkuseg_model", "pkuseg_user_dict"]: - if key in config: - del config[key] - self.tokenizer = Language.Defaults().create_tokenizer(nlp) + def __init__( + self, + nlp: Language, + segmenter: Segmenter = Segmenter.char, + pkuseg_model: Optional[str] = None, + pkuseg_user_dict: Optional[str] = None, + ): + self.vocab = nlp.vocab + if isinstance(segmenter, Segmenter): # we might have the Enum here + segmenter = segmenter.value + self.segmenter = segmenter + self.pkuseg_model = pkuseg_model + self.pkuseg_user_dict = pkuseg_user_dict + self.pkuseg_seg = None + self.jieba_seg = None + self.configure_segmenter(segmenter) - def configure_segmenter(self, config): - self.segmenter = "char" - if "segmenter" in config: - if config["segmenter"] in self.supported_segmenters: - self.segmenter = config["segmenter"] - else: - warn_msg = Warnings.W103.format( - lang="Chinese", - segmenter=config["segmenter"], - supported=", ".join([repr(s) for s in self.supported_segmenters]), - default="'char' (character segmentation)", - ) - warnings.warn(warn_msg) + def configure_segmenter(self, segmenter: str): + if segmenter not in Segmenter.values(): + warn_msg = Warnings.W103.format( + lang="Chinese", + segmenter=segmenter, + supported=", ".join(Segmenter.values()), + default="'char' (character segmentation)", + ) + warnings.warn(warn_msg) + self.segmenter = Segmenter.char self.jieba_seg = try_jieba_import(self.segmenter) self.pkuseg_seg = try_pkuseg_import( self.segmenter, - pkuseg_model=config.get("pkuseg_model", None), - pkuseg_user_dict=config.get("pkuseg_user_dict", "default"), + pkuseg_model=self.pkuseg_model, + pkuseg_user_dict=self.pkuseg_user_dict, ) - def __call__(self, text): - if self.segmenter == "jieba": + def __call__(self, text: str) -> Doc: + if self.segmenter == Segmenter.jieba: words = list([x for x in self.jieba_seg.cut(text, cut_all=False) if x]) (words, spaces) = util.get_words_and_spaces(words, text) return Doc(self.vocab, words=words, spaces=spaces) - elif self.segmenter == "pkuseg": + elif self.segmenter == Segmenter.pkuseg: if self.pkuseg_seg is None: raise ValueError(Errors.E1000) words = self.pkuseg_seg.cut(text) @@ -105,11 +122,11 @@ class ChineseTokenizer(DummyTokenizer): return Doc(self.vocab, words=words, spaces=spaces) # warn if segmenter setting is not the only remaining option "char" - if self.segmenter != "char": + if self.segmenter != Segmenter.char: warn_msg = Warnings.W103.format( lang="Chinese", segmenter=self.segmenter, - supported=", ".join([repr(s) for s in self.supported_segmenters]), + supported=", ".join(Segmenter.values()), default="'char' (character segmentation)", ) warnings.warn(warn_msg) @@ -119,33 +136,25 @@ class ChineseTokenizer(DummyTokenizer): (words, spaces) = util.get_words_and_spaces(words, text) return Doc(self.vocab, words=words, spaces=spaces) - def pkuseg_update_user_dict(self, words, reset=False): - if self.segmenter == "pkuseg": + def pkuseg_update_user_dict(self, words: List[str], reset: bool = False): + if self.segmenter == Segmenter.pkuseg: if reset: try: import pkuseg self.pkuseg_seg.preprocesser = pkuseg.Preprocesser(None) except ImportError: - if self.segmenter == "pkuseg": - msg = ( - "pkuseg not installed: unable to reset pkuseg " - "user dict. Please " + _PKUSEG_INSTALL_MSG - ) - raise ImportError(msg) + msg = ( + "pkuseg not installed: unable to reset pkuseg " + "user dict. Please " + _PKUSEG_INSTALL_MSG + ) + raise ImportError(msg) for word in words: self.pkuseg_seg.preprocesser.insert(word.strip(), "") else: warn_msg = Warnings.W104.format(target="pkuseg", current=self.segmenter) warnings.warn(warn_msg) - def _get_config(self): - config = OrderedDict((("segmenter", self.segmenter),)) - return config - - def _set_config(self, config={}): - self.configure_segmenter(config) - def to_bytes(self, **kwargs): pkuseg_features_b = b"" pkuseg_weights_b = b"" @@ -165,17 +174,11 @@ class ChineseTokenizer(DummyTokenizer): sorted(list(self.pkuseg_seg.postprocesser.common_words)), sorted(list(self.pkuseg_seg.postprocesser.other_words)), ) - serializers = OrderedDict( - ( - ("cfg", lambda: srsly.json_dumps(self._get_config())), - ("pkuseg_features", lambda: pkuseg_features_b), - ("pkuseg_weights", lambda: pkuseg_weights_b), - ( - "pkuseg_processors", - lambda: srsly.msgpack_dumps(pkuseg_processors_data), - ), - ) - ) + serializers = { + "pkuseg_features": lambda: pkuseg_features_b, + "pkuseg_weights": lambda: pkuseg_weights_b, + "pkuseg_processors": lambda: srsly.msgpack_dumps(pkuseg_processors_data), + } return util.to_bytes(serializers, []) def from_bytes(self, data, **kwargs): @@ -190,14 +193,11 @@ class ChineseTokenizer(DummyTokenizer): def deserialize_pkuseg_processors(b): pkuseg_data["processors_data"] = srsly.msgpack_loads(b) - deserializers = OrderedDict( - ( - ("cfg", lambda b: self._set_config(srsly.json_loads(b))), - ("pkuseg_features", deserialize_pkuseg_features), - ("pkuseg_weights", deserialize_pkuseg_weights), - ("pkuseg_processors", deserialize_pkuseg_processors), - ) - ) + deserializers = { + "pkuseg_features": deserialize_pkuseg_features, + "pkuseg_weights": deserialize_pkuseg_weights, + "pkuseg_processors": deserialize_pkuseg_processors, + } util.from_bytes(data, deserializers, []) if pkuseg_data["features_b"] and pkuseg_data["weights_b"]: @@ -245,13 +245,10 @@ class ChineseTokenizer(DummyTokenizer): ) srsly.write_msgpack(path, data) - serializers = OrderedDict( - ( - ("cfg", lambda p: srsly.write_json(p, self._get_config())), - ("pkuseg_model", lambda p: save_pkuseg_model(p)), - ("pkuseg_processors", lambda p: save_pkuseg_processors(p)), - ) - ) + serializers = { + "pkuseg_model": lambda p: save_pkuseg_model(p), + "pkuseg_processors": lambda p: save_pkuseg_processors(p), + } return util.to_disk(path, serializers, []) def from_disk(self, path, **kwargs): @@ -261,7 +258,7 @@ class ChineseTokenizer(DummyTokenizer): try: import pkuseg except ImportError: - if self.segmenter == "pkuseg": + if self.segmenter == Segmenter.pkuseg: raise ImportError( "pkuseg not installed. To use this model, " + _PKUSEG_INSTALL_MSG @@ -273,9 +270,9 @@ class ChineseTokenizer(DummyTokenizer): try: import pkuseg except ImportError: - if self.segmenter == "pkuseg": + if self.segmenter == Segmenter.pkuseg: raise ImportError(self._pkuseg_install_msg) - if self.segmenter == "pkuseg": + if self.segmenter == Segmenter.pkuseg: data = srsly.read_msgpack(path) (user_dict, do_process, common_words, other_words) = data self.pkuseg_seg.preprocesser = pkuseg.Preprocesser(user_dict) @@ -283,35 +280,64 @@ class ChineseTokenizer(DummyTokenizer): self.pkuseg_seg.postprocesser.common_words = set(common_words) self.pkuseg_seg.postprocesser.other_words = set(other_words) - serializers = OrderedDict( - ( - ("cfg", lambda p: self._set_config(srsly.read_json(p))), - ("pkuseg_model", lambda p: load_pkuseg_model(p)), - ("pkuseg_processors", lambda p: load_pkuseg_processors(p)), - ) - ) + serializers = { + "pkuseg_model": lambda p: load_pkuseg_model(p), + "pkuseg_processors": lambda p: load_pkuseg_processors(p), + } util.from_disk(path, serializers, []) class ChineseDefaults(Language.Defaults): - lex_attr_getters = dict(Language.Defaults.lex_attr_getters) - lex_attr_getters.update(LEX_ATTRS) - lex_attr_getters[LANG] = lambda text: "zh" tokenizer_exceptions = BASE_EXCEPTIONS - stop_words = STOP_WORDS - writing_system = {"direction": "ltr", "has_case": False, "has_letters": False} - - @classmethod - def create_tokenizer(cls, nlp=None, config={}): - return ChineseTokenizer(cls, nlp, config=config) class Chinese(Language): lang = "zh" - Defaults = ChineseDefaults # override defaults + Defaults = ChineseDefaults + default_config = Config().from_str(DEFAULT_CONFIG) - def make_doc(self, text): - return self.tokenizer(text) + +def try_jieba_import(segmenter: str) -> None: + try: + import jieba + + if segmenter == Segmenter.jieba: + # segment a short text to have jieba initialize its cache in advance + list(jieba.cut("作为", cut_all=False)) + + return jieba + except ImportError: + if segmenter == Segmenter.jieba: + msg = ( + "Jieba not installed. To use jieba, install it with `pip " + " install jieba` or from https://github.com/fxsjy/jieba" + ) + raise ImportError(msg) + + +def try_pkuseg_import(segmenter: str, pkuseg_model: str, pkuseg_user_dict: str) -> None: + try: + import pkuseg + + if pkuseg_model: + return pkuseg.pkuseg(pkuseg_model, pkuseg_user_dict) + elif segmenter == Segmenter.pkuseg: + msg = ( + "The Chinese word segmenter is 'pkuseg' but no pkuseg model " + "was specified. Please provide the name of a pretrained model " + "or the path to a model with:\n" + 'cfg = {"nlp": {"tokenizer": {"segmenter": "pkuseg", "pkuseg_model": name_or_path }}\n' + "nlp = Chinese.from_config(cfg)" + ) + raise ValueError(msg) + except ImportError: + if segmenter == Segmenter.pkuseg: + msg = "pkuseg not installed. To use pkuseg, " + _PKUSEG_INSTALL_MSG + raise ImportError(msg) + except FileNotFoundError: + if segmenter == Segmenter.pkuseg: + msg = "Unable to load pkuseg model from: " + pkuseg_model + raise FileNotFoundError(msg) def _get_pkuseg_trie_data(node, path=""): diff --git a/spacy/language.py b/spacy/language.py index 7e42b44bb..97c8f31b7 100644 --- a/spacy/language.py +++ b/spacy/language.py @@ -1,129 +1,62 @@ +from typing import Optional, Any, Dict, Callable, Iterable, Union, List, Pattern +from typing import Tuple, Iterator +from dataclasses import dataclass import random import itertools import weakref import functools -from collections import Iterable +from collections import Iterable as IterableInstance from contextlib import contextmanager from copy import copy, deepcopy from pathlib import Path import warnings - -from thinc.api import get_current_ops, Config, require_gpu +from thinc.api import get_current_ops, Config, require_gpu, Optimizer import srsly import multiprocessing as mp from itertools import chain, cycle -from .tokenizer import Tokenizer from .tokens.underscore import Underscore from .vocab import Vocab -from .lemmatizer import Lemmatizer -from .lookups import Lookups from .pipe_analysis import analyze_pipes, analyze_all_pipes, validate_attrs -from .pipe_analysis import count_pipeline_interdependencies from .gold import Example from .scorer import Scorer from .util import link_vectors_to_models, create_default_optimizer, registry -from .attrs import IS_STOP, LANG, NORM +from .util import SimpleFrozenDict from .lang.punctuation import TOKENIZER_PREFIXES, TOKENIZER_SUFFIXES from .lang.punctuation import TOKENIZER_INFIXES from .lang.tokenizer_exceptions import TOKEN_MATCH, URL_MATCH -from .lang.norm_exceptions import BASE_NORMS from .lang.tag_map import TAG_MAP -from .tokens import Doc -from .lang.lex_attrs import LEX_ATTRS, is_stop +from .tokens import Doc, Span from .errors import Errors, Warnings +from .schemas import ConfigSchema from .git_info import GIT_VERSION from . import util from . import about +# We also need to import these to make sure the functions are registered +from .tokenizer import Tokenizer # noqa: F401 +from .lemmatizer import Lemmatizer # noqa: F401 +from .lookups import Lookups # noqa: F401 + ENABLE_PIPELINE_ANALYSIS = False +# This is the base config will all settings (training etc.) +DEFAULT_CONFIG_PATH = Path(__file__).parent / "default_config.cfg" +DEFAULT_CONFIG = Config().from_disk(DEFAULT_CONFIG_PATH) class BaseDefaults: - @classmethod - def create_lemmatizer(cls, nlp=None, lookups=None): - if lookups is None: - lookups = cls.create_lookups(nlp=nlp) - return Lemmatizer(lookups=lookups, is_base_form=cls.is_base_form) - - @classmethod - def create_lookups(cls, nlp=None): - root = util.get_module_path(cls) - filenames = {name: root / filename for name, filename in cls.resources} - if LANG in cls.lex_attr_getters: - lang = cls.lex_attr_getters[LANG](None) - if lang in util.registry.lookups: - filenames.update(util.registry.lookups.get(lang)) - lookups = Lookups() - for name, filename in filenames.items(): - data = util.load_language_data(filename) - lookups.add_table(name, data) - return lookups - - @classmethod - def create_vocab(cls, nlp=None): - lookups = cls.create_lookups(nlp) - lemmatizer = cls.create_lemmatizer(nlp, lookups=lookups) - lex_attr_getters = dict(cls.lex_attr_getters) - # This is messy, but it's the minimal working fix to Issue #639. - lex_attr_getters[IS_STOP] = functools.partial(is_stop, stops=cls.stop_words) - vocab = Vocab( - lex_attr_getters=lex_attr_getters, - tag_map=cls.tag_map, - lemmatizer=lemmatizer, - lookups=lookups, - ) - vocab.lex_attr_getters[NORM] = util.add_lookups( - vocab.lex_attr_getters.get(NORM, LEX_ATTRS[NORM]), - BASE_NORMS, - vocab.lookups.get_table("lexeme_norm"), - ) - vocab.morphology.load_morph_exceptions(cls.morph_rules) - return vocab - - @classmethod - def create_tokenizer(cls, nlp=None): - rules = cls.tokenizer_exceptions - token_match = cls.token_match - url_match = cls.url_match - prefix_search = ( - util.compile_prefix_regex(cls.prefixes).search if cls.prefixes else None - ) - suffix_search = ( - util.compile_suffix_regex(cls.suffixes).search if cls.suffixes else None - ) - infix_finditer = ( - util.compile_infix_regex(cls.infixes).finditer if cls.infixes else None - ) - vocab = nlp.vocab if nlp is not None else cls.create_vocab(nlp) - return Tokenizer( - vocab, - rules=rules, - prefix_search=prefix_search, - suffix_search=suffix_search, - infix_finditer=infix_finditer, - token_match=token_match, - url_match=url_match, - ) - - pipe_names = ["tagger", "parser", "ner"] - token_match = TOKEN_MATCH - url_match = URL_MATCH - prefixes = tuple(TOKENIZER_PREFIXES) - suffixes = tuple(TOKENIZER_SUFFIXES) - infixes = tuple(TOKENIZER_INFIXES) - tag_map = dict(TAG_MAP) - tokenizer_exceptions = {} - stop_words = set() - morph_rules = {} - is_base_form = None - lex_attr_getters = LEX_ATTRS - syntax_iterators = {} - resources = {} - writing_system = {"direction": "ltr", "has_case": True, "has_letters": True} - single_orth_variants = [] - paired_orth_variants = [] + token_match: Optional[Pattern] = TOKEN_MATCH + url_match: Pattern = URL_MATCH + prefixes: Tuple[Pattern, ...] = tuple(TOKENIZER_PREFIXES) + suffixes: Tuple[Pattern, ...] = tuple(TOKENIZER_SUFFIXES) + infixes: Tuple[Pattern, ...] = tuple(TOKENIZER_INFIXES) + tag_map: Dict[str, dict] = dict(TAG_MAP) + tokenizer_exceptions: Dict[str, List[dict]] = {} + morph_rules: Dict[str, Dict[str, dict]] = {} + syntax_iterators: Dict[str, Callable[[Union[Doc, Span]], Iterator]] = {} + single_orth_variants: List[Dict[str, List[str]]] = [] + paired_orth_variants: List[Dict[str, Union[List[str], List[Tuple[str, str]]]]] = [] class Language: @@ -138,30 +71,27 @@ class Language: """ Defaults = BaseDefaults - lang = None + lang: str = None + default_config = DEFAULT_CONFIG + factories = SimpleFrozenDict(error=Errors.E957) - factories = {"tokenizer": lambda nlp: nlp.Defaults.create_tokenizer(nlp)} + _factory_meta: Dict[str, "FactoryMeta"] = {} # meta by factory def __init__( self, - vocab=True, - make_doc=True, - max_length=10 ** 6, - meta={}, - config=None, + vocab: Union[Vocab, bool] = True, + max_length: int = 10 ** 6, + meta: Dict[str, Any] = {}, + create_tokenizer: Optional[Callable[["Language"], Callable[[str], Doc]]] = None, **kwargs, ): """Initialise a Language object. - vocab (Vocab): A `Vocab` object. If `True`, a vocab is created via - `Language.Defaults.create_vocab`. - make_doc (callable): A function that takes text and returns a `Doc` - object. Usually a `Tokenizer`. + vocab (Vocab): A `Vocab` object. If `True`, a vocab is created. meta (dict): Custom meta data for the Language class. Is written to by models to add model meta data. - config (Config): Configuration data for creating the pipeline components. max_length (int) : - Maximum number of characters in a single text. The current v2 models + Maximum number of characters in a single text. The current models may run out memory on extremely long texts, due to large internal allocations. You should segment these texts into meaningful units, e.g. paragraphs, subsections etc, before passing them to spaCy. @@ -171,36 +101,53 @@ class Language: 100,000 characters in one text. RETURNS (Language): The newly constructed object. """ - user_factories = util.registry.factories.get_all() - self.factories.update(user_factories) + # We're only calling this to import all factories provided via entry + # points. The factory decorator applied to these functions takes care + # of the rest. + util.registry._entry_point_factories.get_all() + + self._config = util.deep_merge_configs(self.default_config, DEFAULT_CONFIG) self._meta = dict(meta) - self._config = config - if not self._config: - self._config = Config() self._path = None + self._optimizer = None + # Component meta and configs are only needed on the instance + self._pipe_meta: Dict[str, "FactoryMeta"] = {} # meta by component + self._pipe_configs: Dict[str, Config] = {} # config by component + if vocab is True: - factory = self.Defaults.create_vocab - vocab = factory(self, **meta.get("vocab", {})) - if vocab.vectors.name is None: - vocab.vectors.name = meta.get("vectors", {}).get("name") + vectors_name = meta.get("vectors", {}).get("name") + vocab = Vocab.from_config( + self._config, + vectors_name=vectors_name, + # TODO: what should we do with these? + tag_map=self.Defaults.tag_map, + morph_rules=self.Defaults.morph_rules, + ) else: if (self.lang and vocab.lang) and (self.lang != vocab.lang): raise ValueError(Errors.E150.format(nlp=self.lang, vocab=vocab.lang)) self.vocab = vocab - if make_doc is True: - factory = self.Defaults.create_tokenizer - make_doc = factory(self, **meta.get("tokenizer", {})) - self.tokenizer = make_doc + if self.lang is None: + self.lang = self.vocab.lang self.pipeline = [] self.max_length = max_length - self._optimizer = None + self.resolved = {} + # Create the default tokenizer from the default config + if not create_tokenizer: + tokenizer_cfg = {"tokenizer": self._config["nlp"]["tokenizer"]} + create_tokenizer = registry.make_from_config(tokenizer_cfg)["tokenizer"] + self.tokenizer = create_tokenizer(self) + + def __init_subclass__(cls, **kwargs): + super().__init_subclass__(**kwargs) + cls.default_config = util.deep_merge_configs(cls.default_config, DEFAULT_CONFIG) @property def path(self): return self._path @property - def meta(self): + def meta(self) -> Dict[str, Any]: spacy_version = util.get_model_version_range(about.__version__) if self.vocab.lang: self._meta.setdefault("lang", self.vocab.lang) @@ -221,44 +168,67 @@ class Language: "keys": self.vocab.vectors.n_keys, "name": self.vocab.vectors.name, } - self._meta["pipeline"] = self.pipe_names - self._meta["factories"] = self.pipe_factories self._meta["labels"] = self.pipe_labels return self._meta @meta.setter - def meta(self, value): + def meta(self, value: Dict[str, Any]) -> None: self._meta = value @property - def config(self): + def config(self) -> Config: + self._config.setdefault("nlp", {}) + self._config["nlp"]["lang"] = self.lang + # We're storing the filled config for each pipeline component and so + # we can populate the config again later + pipeline = {} + for pipe_name in self.pipe_names: + pipe_meta = self.get_pipe_meta(pipe_name) + pipe_config = self.get_pipe_config(pipe_name) + pipeline[pipe_name] = {"@factories": pipe_meta.factory, **pipe_config} + self._config["nlp"]["pipeline"] = self.pipe_names + self._config["components"] = pipeline + if not srsly.is_json_serializable(self._config): + raise ValueError(Errors.E961.format(config=self._config)) return self._config + @config.setter + def config(self, value: Config) -> None: + self._config = value + @property - def pipe_names(self): + def factory_names(self) -> List[str]: + """Get names of all available factories. + + RETURNS (List[str]): The factory names. + """ + return list(self.factories.keys()) + + @property + def pipe_names(self) -> List[str]: """Get names of available pipeline components. - RETURNS (list): List of component name strings, in order. + RETURNS (List[str]): List of component name strings, in order. """ return [pipe_name for pipe_name, _ in self.pipeline] @property - def pipe_factories(self): + def pipe_factories(self) -> Dict[str, str]: """Get the component factories for the available pipeline components. - RETURNS (dict): Factory names, keyed by component names. + RETURNS (Dict[str, str]): Factory names, keyed by component names. """ factories = {} for pipe_name, pipe in self.pipeline: - factories[pipe_name] = getattr(pipe, "factory", pipe_name) + factories[pipe_name] = self.get_pipe_meta(pipe_name).factory return factories @property - def pipe_labels(self): + def pipe_labels(self) -> Dict[str, List[str]]: """Get the labels set by the pipeline components, if available (if the component exposes a labels property). - RETURNS (dict): Labels keyed by component name. + RETURNS (Dict[str, List[str]]): Labels keyed by component name. """ labels = {} for name, pipe in self.pipeline: @@ -266,7 +236,189 @@ class Language: labels[name] = list(pipe.labels) return labels - def get_pipe(self, name): + @classmethod + def has_factory(cls, name: str) -> bool: + """RETURNS (bool): Whether a factory of that name is registered.""" + internal_name = cls.get_factory_name(name) + return name in registry.factories or internal_name in registry.factories + + @classmethod + def get_factory_name(cls, name: str) -> str: + """Get the internal factory name based on the language subclass. + + name (str): The factory name. + RETURNS (str): The internal factory name. + """ + if cls.lang is None: + return name + return f"{cls.lang}.{name}" + + @classmethod + def get_factory_meta(cls, name: str) -> "FactoryMeta": + """Get the meta information for a given factory name. + + name (str): The component factory name. + RETURNS (FactoryMeta): The meta for the given factory name. + """ + internal_name = cls.get_factory_name(name) + if internal_name in cls._factory_meta: + return cls._factory_meta[internal_name] + if name in cls._factory_meta: + return cls._factory_meta[name] + raise ValueError(Errors.E967.format(meta="factory", name=name)) + + @classmethod + def set_factory_meta(cls, name: str, value: "FactoryMeta") -> None: + """Set the meta information for a given factory name. + + name (str): The component factory name. + value (FactoryMeta): The meta to set. + """ + cls._factory_meta[cls.get_factory_name(name)] = value + + def get_pipe_meta(self, name: str) -> "FactoryMeta": + """Get the meta information for a given component name. + + name (str): The component name. + RETURNS (FactoryMeta): The meta for the given component name. + """ + if name not in self._pipe_meta: + raise ValueError(Errors.E967.format(meta="component", name=name)) + return self._pipe_meta[name] + + def get_pipe_config(self, name: str) -> Config: + """Get the config used to create a pipeline component. + + name (str): The component name. + RETURNS (Config): The config used to create the pipeline component. + """ + if name not in self._pipe_configs: + raise ValueError(Errors.E960.format(name=name)) + pipe_config = self._pipe_configs[name] + pipe_config.pop("nlp", None) + pipe_config.pop("name", None) + return pipe_config + + @classmethod + def factory( + cls, + name: str, + *, + default_config: Dict[str, Any] = SimpleFrozenDict(), + assigns: Iterable[str] = tuple(), + requires: Iterable[str] = tuple(), + retokenizes: bool = False, + func: Optional[Callable] = None, + ) -> Callable: + """Register a new pipeline component factory. Can be used as a decorator + on a function or classmethod, or called as a function with the factory + provided as the func keyword argument. To create a component and add + it to the pipeline, you can use nlp.add_pipe(name). + + name (str): The name of the component factory. + default_config (Dict[str, Any]): Default configuration, describing the + default values of the factory arguments. + assigns (Iterable[str]): Doc/Token attributes assigned by this component, + e.g. "token.ent_id". Used for pipeline analyis. + requires (Iterable[str]): Doc/Token attributes required by this component, + e.g. "token.ent_id". Used for pipeline analyis. + retokenizes (bool): Whether the component changes the tokenization. + Used for pipeline analysis. + func (Optional[Callable]): Factory function if not used as a decorator. + """ + if not isinstance(name, str): + raise ValueError(Errors.E963.format(decorator="factory")) + if not isinstance(default_config, dict): + err = Errors.E962.format( + style="default config", name=name, cfg_type=type(default_config) + ) + raise ValueError(err) + internal_name = cls.get_factory_name(name) + if internal_name in registry.factories: + # We only check for the internal name here – it's okay if it's a + # subclass and the base class has a factory of the same name + raise ValueError(Errors.E004.format(name=name)) + + def add_factory(factory_func: Callable) -> Callable: + arg_names = util.get_arg_names(factory_func) + if "nlp" not in arg_names or "name" not in arg_names: + raise ValueError(Errors.E964.format(name=name)) + # Officially register the factory so we can later call + # registry.make_from_config and refer to it in the config as + # @factories = "spacy.Language.xyz". We use the class name here so + # different classes can have different factories. + registry.factories.register(internal_name, func=factory_func) + factory_meta = FactoryMeta( + factory=name, + default_config=default_config, + assigns=validate_attrs(assigns), + requires=validate_attrs(requires), + retokenizes=retokenizes, + ) + cls.set_factory_meta(name, factory_meta) + # We're overwriting the class attr with a frozen dict to handle + # backwards-compat (writing to Language.factories directly). This + # wouldn't work with an instance property and just produce a + # confusing error – here we can show a custom error + cls.factories = SimpleFrozenDict( + registry.factories.get_all(), error=Errors.E957 + ) + return factory_func + + if func is not None: # Support non-decorator use cases + return add_factory(func) + return add_factory + + @classmethod + def component( + cls, + name: Optional[str] = None, + *, + assigns: Iterable[str] = tuple(), + requires: Iterable[str] = tuple(), + retokenizes: bool = False, + func: Optional[Callable[[Doc], Doc]] = None, + ) -> Callable: + """Register a new pipeline component. Can be used for stateless function + components that don't require a separate factory. Can be used as a + decorator on a function or classmethod, or called as a function with the + factory provided as the func keyword argument. To create a component and + add it to the pipeline, you can use nlp.add_pipe(name). + + name (str): The name of the component factory. + assigns (Iterable[str]): Doc/Token attributes assigned by this component, + e.g. "token.ent_id". Used for pipeline analyis. + requires (Iterable[str]): Doc/Token attributes required by this component, + e.g. "token.ent_id". Used for pipeline analyis. + retokenizes (bool): Whether the component changes the tokenization. + Used for pipeline analysis. + func (Optional[Callable]): Factory function if not used as a decorator. + """ + if name is not None and not isinstance(name, str): + raise ValueError(Errors.E963.format(decorator="component")) + component_name = name if name is not None else util.get_object_name(func) + + def add_component(component_func: Callable[[Doc], Doc]) -> Callable: + if isinstance(func, type): # function is a class + raise ValueError(Errors.E965.format(name=component_name)) + + def factory_func(nlp: cls, name: str) -> Callable[[Doc], Doc]: + return component_func + + cls.factory( + component_name, + assigns=assigns, + requires=requires, + retokenizes=retokenizes, + func=factory_func, + ) + return component_func + + if func is not None: # Support non-decorator use cases + return add_component(func) + return add_component + + def get_pipe(self, name: str) -> Callable[[Doc], Doc]: """Get a pipeline component for a given component name. name (str): Name of pipeline component to get. @@ -279,92 +431,183 @@ class Language: return component raise KeyError(Errors.E001.format(name=name, opts=self.pipe_names)) - def create_pipe(self, name, config=dict()): - """Create a pipeline component from a factory. + def create_pipe( + self, + factory_name: str, + name: Optional[str] = None, + config: Optional[Dict[str, Any]] = SimpleFrozenDict(), + overrides: Optional[Dict[str, Any]] = SimpleFrozenDict(), + validate: bool = True, + ) -> Callable[[Doc], Doc]: + """Create a pipeline component. Mostly used internally. To create and + add a component to the pipeline, you can use nlp.add_pipe. - name (str): Factory name to look up in `Language.factories`. - config (dict): Configuration parameters to initialise component. - RETURNS (callable): Pipeline component. - - DOCS: https://spacy.io/api/language#create_pipe + factory_name (str): Name of component factory. + name (Optional[str]): Optional name to assign to component instance. + Defaults to factory name if not set. + config (Optional[Dict[str, Any]]): Config parameters to use for this + component. Will be merged with default config, if available. + overrides (Optional[Dict[str, Any]]): Config overrides, typically + passed in via the CLI. + validate (bool): Whether to validate the component config against the + arguments and types expected by the factory. + RETURNS (Callable[[Doc], Doc]): The pipeline component. """ - if name not in self.factories: - raise KeyError(Errors.E002.format(name=name)) - factory = self.factories[name] - - # transform the model's config to an actual Model - factory_cfg = dict(config) - - # check whether we have a proper model config, ignore if the type is wrong - if "model" in factory_cfg and not isinstance(factory_cfg["model"], dict): - warnings.warn( - Warnings.W099.format(type=type(factory_cfg["model"]), pipe=name) + name = name if name is not None else factory_name + if not isinstance(config, dict): + err = Errors.E962.format(style="config", name=name, cfg_type=type(config)) + raise ValueError(err) + if not srsly.is_json_serializable(config): + raise ValueError(Errors.E961.format(config=config)) + if not self.has_factory(factory_name): + err = Errors.E002.format( + name=factory_name, + opts=", ".join(self.factory_names), + method="create_pipe", + lang=util.get_object_name(self), + lang_code=self.lang, ) - - # refer to the model configuration in the cfg settings for this component - elif "model" in factory_cfg: - self.config[name] = {"model": factory_cfg["model"]} - - # create all objects in the config - factory_cfg = registry.make_from_config({"config": factory_cfg}, validate=True)[ - "config" - ] - model = factory_cfg.get("model", None) - if model is not None: - del factory_cfg["model"] - return factory(self, model, **factory_cfg) + raise ValueError(err) + pipe_meta = self.get_factory_meta(factory_name) + config = config or {} + # This is unideal, but the alternative would mean you always need to + # specify the full config settings, which is not really viable. + if pipe_meta.default_config: + config = util.deep_merge_configs(config, pipe_meta.default_config) + # We need to create a top-level key because Thinc doesn't allow resolving + # top-level references to registered functions. Also gives nicer errors. + # The name allows components to know their pipe name and use it in the + # losses etc. (even if multiple instances of the same factory are used) + internal_name = self.get_factory_name(factory_name) + # If the language-specific factory doesn't exist, try again with the + # not-specific name + if internal_name not in registry.factories: + internal_name = factory_name + config = {"nlp": self, "name": name, **config, "@factories": internal_name} + cfg = {factory_name: config} + # We're calling the internal _fill here to avoid constructing the + # registered functions twice + # TODO: customize validation to make it more readable / relate it to + # pipeline component and why it failed, explain default config + resolved, filled = registry.resolve(cfg, validate=validate, overrides=overrides) + filled = filled[factory_name] + filled["@factories"] = factory_name + self._pipe_configs[name] = filled + return resolved[factory_name] def add_pipe( - self, component, name=None, before=None, after=None, first=None, last=None - ): + self, + factory_name: str, + name: Optional[str] = None, + *, + before: Optional[Union[str, int]] = None, + after: Optional[Union[str, int]] = None, + first: Optional[bool] = None, + last: Optional[bool] = None, + config: Optional[Dict[str, Any]] = SimpleFrozenDict(), + overrides: Optional[Dict[str, Any]] = SimpleFrozenDict(), + validate: bool = True, + ) -> Callable[[Doc], Doc]: """Add a component to the processing pipeline. Valid components are callables that take a `Doc` object, modify it and return it. Only one of before/after/first/last can be set. Default behaviour is "last". - component (callable): The pipeline component. + factory_name (str): Name of the component factory. name (str): Name of pipeline component. Overwrites existing component.name attribute if available. If no name is set and the component exposes no name attribute, component.__name__ is used. An error is raised if a name already exists in the pipeline. - before (str): Component name to insert component directly before. - after (str): Component name to insert component directly after. - first (bool): Insert component first / not first in the pipeline. - last (bool): Insert component last / not last in the pipeline. + before (Union[str, int]): Name or index of the component to insert new + component directly before. + after (Union[str, int]): Name or index of the component to insert new + component directly after. + first (bool): If True, insert component first in the pipeline. + last (bool): If True, insert component last in the pipeline. + config (Optional[Dict[str, Any]]): Config parameters to use for this + component. Will be merged with default config, if available. + overrides (Optional[Dict[str, Any]]): Config overrides, typically + passed in via the CLI. + validate (bool): Whether to validate the component config against the + arguments and types expected by the factory. + RETURNS (Callable[[Doc], Doc]): The pipeline component. DOCS: https://spacy.io/api/language#add_pipe """ - if not hasattr(component, "__call__"): - msg = Errors.E003.format(component=repr(component), name=name) - if isinstance(component, str) and component in self.factories: - msg += Errors.E004.format(component=component) - raise ValueError(msg) - if name is None: - name = util.get_component_name(component) + if not isinstance(factory_name, str): + bad_val = repr(factory_name) + err = Errors.E966.format(component=bad_val, name=name) + raise ValueError(err) + if not self.has_factory(factory_name): + err = Errors.E002.format( + name=factory_name, + opts=", ".join(self.factory_names), + method="add_pipe", + lang=util.get_object_name(self), + lang_code=self.lang, + ) + name = name if name is not None else factory_name if name in self.pipe_names: raise ValueError(Errors.E007.format(name=name, opts=self.pipe_names)) - if sum([bool(before), bool(after), bool(first), bool(last)]) >= 2: - raise ValueError(Errors.E006) - pipe_index = 0 - pipe = (name, component) - if last or not any([first, before, after]): - pipe_index = len(self.pipeline) - self.pipeline.append(pipe) - elif first: - self.pipeline.insert(0, pipe) - elif before and before in self.pipe_names: - pipe_index = self.pipe_names.index(before) - self.pipeline.insert(self.pipe_names.index(before), pipe) - elif after and after in self.pipe_names: - pipe_index = self.pipe_names.index(after) + 1 - self.pipeline.insert(self.pipe_names.index(after) + 1, pipe) - else: - raise ValueError( - Errors.E001.format(name=before or after, opts=self.pipe_names) - ) + pipe_component = self.create_pipe( + factory_name, + name=name, + config=config, + overrides=overrides, + validate=validate, + ) + pipe_index = self._get_pipe_index(before, after, first, last) + self._pipe_meta[name] = self.get_factory_meta(factory_name) + self.pipeline.insert(pipe_index, (name, pipe_component)) if ENABLE_PIPELINE_ANALYSIS: - analyze_pipes(self.pipeline, name, component, pipe_index) + analyze_pipes(self, name, pipe_index) + return pipe_component - def has_pipe(self, name): + def _get_pipe_index( + self, + before: Optional[Union[str, int]] = None, + after: Optional[Union[str, int]] = None, + first: Optional[bool] = None, + last: Optional[bool] = None, + ) -> int: + """Determine where to insert a pipeline component based on the before/ + after/first/last values. + + before (str): Name or index of the component to insert directly before. + after (str): Name or index of component to insert directly after. + first (bool): If True, insert component first in the pipeline. + last (bool): If True, insert component last in the pipeline. + RETURNS (int): The index of the new pipeline component. + """ + all_args = {"before": before, "after": after, "first": first, "last": last} + if sum(arg is not None for arg in [before, after, first, last]) >= 2: + raise ValueError(Errors.E006.format(args=all_args, opts=self.pipe_names)) + if last or not any(value is not None for value in [first, before, after]): + return len(self.pipeline) + elif first: + return 0 + elif isinstance(before, str): + if before not in self.pipe_names: + raise ValueError(Errors.E001.format(name=before, opts=self.pipe_names)) + return self.pipe_names.index(before) + elif isinstance(after, str): + if after not in self.pipe_names: + raise ValueError(Errors.E001.format(name=after, opts=self.pipe_names)) + return self.pipe_names.index(after) + 1 + # We're only accepting indices referring to components that exist + # (can't just do isinstance here because bools are instance of int, too) + elif type(before) == int: + if before >= len(self.pipeline) or before < 0: + err = Errors.E959.format(dir="before", idx=before, opts=self.pipe_names) + raise ValueError(err) + return before + elif type(after) == int: + if after >= len(self.pipeline) or after < 0: + err = Errors.E959.format(dir="after", idx=after, opts=self.pipe_names) + raise ValueError(err) + return after + 1 + raise ValueError(Errors.E006.format(args=all_args, opts=self.pipe_names)) + + def has_pipe(self, name: str) -> bool: """Check if a component name is present in the pipeline. Equivalent to `name in nlp.pipe_names`. @@ -375,26 +618,41 @@ class Language: """ return name in self.pipe_names - def replace_pipe(self, name, component): + def replace_pipe( + self, + name: str, + factory_name: str, + config: Dict[str, Any] = SimpleFrozenDict(), + validate: bool = True, + ) -> None: """Replace a component in the pipeline. name (str): Name of the component to replace. - component (callable): Pipeline component. + factory_name (str): Factory name of replacement component. + config (Optional[Dict[str, Any]]): Config parameters to use for this + component. Will be merged with default config, if available. + validate (bool): Whether to validate the component config against the + arguments and types expected by the factory. DOCS: https://spacy.io/api/language#replace_pipe """ if name not in self.pipe_names: raise ValueError(Errors.E001.format(name=name, opts=self.pipe_names)) - if not hasattr(component, "__call__"): - msg = Errors.E003.format(component=repr(component), name=name) - if isinstance(component, str) and component in self.factories: - msg += Errors.E135.format(name=name) - raise ValueError(msg) - self.pipeline[self.pipe_names.index(name)] = (name, component) + if hasattr(factory_name, "__call__"): + err = Errors.E968.format(component=repr(factory_name), name=name) + raise ValueError(err) + # We need to delegate to Language.add_pipe here instead of just writing + # to Language.pipeline to make sure the configs are handled correctly + pipe_index = self.pipe_names.index(name) + self.remove_pipe(name) + if not len(self.pipeline): # we have no components to insert before/after + self.add_pipe(factory_name, name=name) + else: + self.add_pipe(factory_name, name=name, before=pipe_index) if ENABLE_PIPELINE_ANALYSIS: - analyze_all_pipes(self.pipeline) + analyze_all_pipes(self) - def rename_pipe(self, old_name, new_name): + def rename_pipe(self, old_name: str, new_name: str) -> None: """Rename a pipeline component. old_name (str): Name of the component to rename. @@ -408,8 +666,10 @@ class Language: raise ValueError(Errors.E007.format(name=new_name, opts=self.pipe_names)) i = self.pipe_names.index(old_name) self.pipeline[i] = (new_name, self.pipeline[i][1]) + self._pipe_meta[new_name] = self._pipe_meta.pop(old_name) + self._pipe_configs[new_name] = self._pipe_configs.pop(old_name) - def remove_pipe(self, name): + def remove_pipe(self, name: str) -> Tuple[str, Callable[[Doc], Doc]]: """Remove a component from the pipeline. name (str): Name of the component to remove. @@ -420,11 +680,20 @@ class Language: if name not in self.pipe_names: raise ValueError(Errors.E001.format(name=name, opts=self.pipe_names)) removed = self.pipeline.pop(self.pipe_names.index(name)) + # We're only removing the component itself from the metas/configs here + # because factory may be used for something else + self._pipe_meta.pop(name) + self._pipe_configs.pop(name) if ENABLE_PIPELINE_ANALYSIS: - analyze_all_pipes(self.pipeline) + analyze_all_pipes(self) return removed - def __call__(self, text, disable=[], component_cfg=None): + def __call__( + self, + text: str, + disable: Iterable[str] = tuple(), + component_cfg: Optional[Dict[str, Dict[str, Any]]] = None, + ) -> Doc: """Apply the pipeline to some text. The text can span multiple sentences, and can contain arbitrary whitespace. Alignment into the original string is preserved. @@ -457,7 +726,7 @@ class Language: raise ValueError(Errors.E005.format(name=name)) return doc - def disable_pipes(self, *names): + def disable_pipes(self, *names) -> "DisabledPipes": """Disable one or more pipeline components. If used as a context manager, the pipeline will be restored to the initial state at the end of the block. Otherwise, a DisabledPipes object is returned, that has @@ -470,7 +739,11 @@ class Language: names = names[0] # support list of names instead of spread return DisabledPipes(self, names) - def select_pipes(self, disable=None, enable=None): + def select_pipes( + self, + disable: Optional[Union[str, Iterable[str]]] = None, + enable: Optional[Union[str, Iterable[str]]] = None, + ) -> "DisabledPipes": """Disable one or more pipeline components. If used as a context manager, the pipeline will be restored to the initial state at the end of the block. Otherwise, a DisabledPipes object is returned, that has @@ -499,18 +772,23 @@ class Language: disable = to_disable return DisabledPipes(self, disable) - def make_doc(self, text): + def make_doc(self, text: str) -> Doc: + """Turn a text into a Doc object. + + text (str): The text to process. + RETURNS (Doc): The processed doc. + """ return self.tokenizer(text) def update( self, - examples, - dummy=None, + examples: Iterable[Example], + dummy: Optional[Any] = None, *, - drop=0.0, - sgd=None, - losses=None, - component_cfg=None, + drop: float = 0.0, + sgd: Optional[Optimizer] = None, + losses: Optional[Dict[str, float]] = None, + component_cfg: Optional[Dict[str, Dict[str, Any]]] = None, ): """Update the models in the pipeline. @@ -531,7 +809,7 @@ class Language: losses = {} if len(examples) == 0: return losses - if not isinstance(examples, Iterable): + if not isinstance(examples, IterableInstance): raise TypeError( Errors.E978.format( name="language", method="update", types=type(examples) @@ -564,16 +842,23 @@ class Language: proc.model.finish_update(sgd) return losses - def rehearse(self, examples, sgd=None, losses=None, config=None): + def rehearse( + self, + examples: Iterable[Example], + sgd: Optional[Optimizer] = None, + losses: Optional[Dict[str, float]] = None, + component_cfg: Optional[Dict[str, Dict[str, Any]]] = None, + ) -> Dict[str, float]: """Make a "rehearsal" update to the models in the pipeline, to prevent forgetting. Rehearsal updates run an initial copy of the model over some data, and update the model so its current predictions are more like the initial ones. This is useful for keeping a pretrained model on-track, even if you're updating it with a smaller set of examples. - examples (iterable): A batch of `Example` objects. - drop (float): The dropout rate. - sgd (callable): An optimizer. + examples (Iterable[Example]): A batch of `Example` objects. + sgd (Optional[Optimizer]): An optimizer. + component_cfg (Dict[str, Dict]): Config parameters for specific pipeline + components, keyed by component name. RETURNS (dict): Results from the update. EXAMPLE: @@ -586,7 +871,7 @@ class Language: # TODO: document if len(examples) == 0: return - if not isinstance(examples, Iterable): + if not isinstance(examples, IterableInstance): raise TypeError( Errors.E978.format( name="language", method="rehearse", types=type(examples) @@ -605,8 +890,8 @@ class Language: sgd = self._optimizer pipes = list(self.pipeline) random.shuffle(pipes) - if config is None: - config = {} + if component_cfg is None: + component_cfg = {} grads = {} def get_grads(W, dW, key=None): @@ -620,19 +905,24 @@ class Language: continue grads = {} proc.rehearse( - examples, sgd=get_grads, losses=losses, **config.get(name, {}) + examples, sgd=get_grads, losses=losses, **component_cfg.get(name, {}) ) for key, (W, dW) in grads.items(): sgd(W, dW, key=key) return losses - def begin_training(self, get_examples=None, sgd=None, component_cfg=None, **cfg): + def begin_training( + self, + get_examples: Optional[Callable] = None, + sgd: Optional[Optimizer] = None, + device: int = -1, + ) -> Optimizer: """Allocate models, pre-process training data and acquire a trainer and optimizer. Used as a contextmanager. - get_examples (function): Function returning example training data (TODO: document format change since 3.0) - component_cfg (dict): Config parameters for specific components. - **cfg: Config parameters. + get_examples (function): Function returning example training data. + TODO: document format change since 3.0. + sgd (Optional[Optimizer]): An optimizer. RETURNS: An optimizer. DOCS: https://spacy.io/api/language#begin_training @@ -640,14 +930,12 @@ class Language: # TODO: throw warning when get_gold_tuples is provided instead of get_examples if get_examples is None: get_examples = lambda: [] - # Populate vocab - else: + else: # Populate vocab for example in get_examples(): for word in [t.text for t in example.reference]: _ = self.vocab[word] # noqa: F841 - - if cfg.get("device", -1) >= 0: - require_gpu(cfg["device"]) + if device >= 0: # TODO: do we need this here? + require_gpu(device) if self.vocab.vectors.data.shape[1] >= 1: ops = get_current_ops() self.vocab.vectors.data = ops.asarray(self.vocab.vectors.data) @@ -655,19 +943,17 @@ class Language: if sgd is None: sgd = create_default_optimizer() self._optimizer = sgd - if component_cfg is None: - component_cfg = {} for name, proc in self.pipeline: if hasattr(proc, "begin_training"): - kwargs = component_cfg.get(name, {}) - kwargs.update(cfg) proc.begin_training( - get_examples, pipeline=self.pipeline, sgd=self._optimizer, **kwargs + get_examples, pipeline=self.pipeline, sgd=self._optimizer ) self._link_components() return self._optimizer - def resume_training(self, sgd=None, **cfg): + def resume_training( + self, sgd: Optional[Optimizer] = None, device: int = -1 + ) -> Optimizer: """Continue training a pretrained model. Create and return an optimizer, and initialize "rehearsal" for any pipeline @@ -675,9 +961,12 @@ class Language: models from "forgetting" their initialised "knowledge". To perform rehearsal, collect samples of text you want the models to retain performance on, and call nlp.rehearse() with a batch of Example objects. + + sgd (Optional[Optimizer]): An optimizer. + RETURNS (Optimizer): The optimizer. """ - if cfg.get("device", -1) >= 0: - require_gpu(cfg["device"]) + if device >= 0: # TODO: do we need this here? + require_gpu(device) ops = get_current_ops() if self.vocab.vectors.data.shape[1] >= 1: self.vocab.vectors.data = ops.asarray(self.vocab.vectors.data) @@ -691,14 +980,19 @@ class Language: return self._optimizer def evaluate( - self, examples, verbose=False, batch_size=256, scorer=None, component_cfg=None - ): + self, + examples: Iterable[Example], + verbose: bool = False, + batch_size: int = 256, + scorer: Optional[Scorer] = None, + component_cfg: Optional[Dict[str, Dict[str, Any]]] = None, + ) -> Scorer: """Evaluate a model's pipeline components. - examples (iterable): `Example` objects. + examples (Iterable[Example]): `Example` objects. verbose (bool): Print debugging information. batch_size (int): Batch size to use. - scorer (Scorer): Optional `Scorer` to use. If not passed in, a new one + scorer (Optional[Scorer]): Scorer to use. If not passed in, a new one will be created. component_cfg (dict): An optional dictionary with extra keyword arguments for specific components. @@ -706,19 +1000,17 @@ class Language: DOCS: https://spacy.io/api/language#evaluate """ - if not isinstance(examples, Iterable): - raise TypeError( - Errors.E978.format( - name="language", method="evaluate", types=type(examples) - ) + if not isinstance(examples, IterableInstance): + err = Errors.E978.format( + name="language", method="evaluate", types=type(examples) ) + raise TypeError(err) wrong_types = set([type(eg) for eg in examples if not isinstance(eg, Example)]) if wrong_types: - raise TypeError( - Errors.E978.format( - name="language", method="evaluate", types=wrong_types - ) + err = Errors.E978.format( + name="language", method="evaluate", types=wrong_types ) + raise TypeError(err) if scorer is None: scorer = Scorer(pipeline=self.pipeline) if component_cfg is None: @@ -741,7 +1033,7 @@ class Language: return scorer @contextmanager - def use_params(self, params, **cfg): + def use_params(self, params: dict, **cfg): """Replace weights of models in the pipeline with those provided in the params dictionary. Can be used as a contextmanager, in which case, models go back to their original weights after the block. @@ -774,13 +1066,13 @@ class Language: def pipe( self, - texts, - as_tuples=False, - batch_size=1000, - disable=[], - cleanup=False, - component_cfg=None, - n_process=1, + texts: Iterable[str], + as_tuples: bool = False, + batch_size: int = 1000, + disable: Iterable[str] = tuple(), + cleanup: bool = False, + component_cfg: Optional[Dict[str, Dict[str, Any]]] = None, + n_process: int = 1, ): """Process texts as a stream, and yield `Doc` objects in order. @@ -872,7 +1164,13 @@ class Language: self.tokenizer._reset_cache(keys) nr_seen = 0 - def _multiprocessing_pipe(self, texts, pipes, n_process, batch_size): + def _multiprocessing_pipe( + self, + texts: Iterable[str], + pipes: Iterable[Callable[[Doc], Doc]], + n_process: int, + batch_size: int, + ) -> None: # raw_texts is used later to stop iteration. texts, raw_texts = itertools.tee(texts) # for sending texts to worker @@ -915,7 +1213,7 @@ class Language: for proc in procs: proc.terminate() - def _link_components(self): + def _link_components(self) -> None: """Register 'listeners' within pipeline components, to allow them to effectively share weights. """ @@ -925,7 +1223,86 @@ class Language: if hasattr(proc2, "model"): proc1.find_listeners(proc2.model) - def to_disk(self, path, exclude=tuple()): + @classmethod + def from_config( + cls, + config: Union[Dict[str, Any], Config] = {}, + disable: Iterable[str] = tuple(), + overrides: Dict[str, Any] = {}, + auto_fill: bool = True, + validate: bool = True, + ) -> "Language": + """Create the nlp object from a loaded config. Will set up the tokenizer + and language data, add pipeline components etc. If no config is provided, + the default config of the given language is used. + """ + if auto_fill: + config = util.deep_merge_configs(config, cls.default_config) + if "nlp" not in config: + raise ValueError(Errors.E985.format(config=config)) + nlp_config = config["nlp"] + config_lang = nlp_config["lang"] + if cls.lang is not None and config_lang is not None and config_lang != cls.lang: + raise ValueError( + Errors.E958.format( + bad_lang_code=nlp_config["lang"], + lang_code=cls.lang, + lang=util.get_object_name(cls), + ) + ) + nlp_config["lang"] = cls.lang + # This isn't very elegant, but we remove the [components] block here to prevent + # it from getting resolved (causes problems because we expect to pass in + # the nlp and name args for each component). If we're auto-filling, we're + # using the nlp.config with all defaults. + config = util.copy_config(config) + orig_pipeline = config.pop("components", {}) + config["components"] = {} + non_pipe_overrides, pipe_overrides = _get_config_overrides(overrides) + resolved, filled = registry.resolve( + config, validate=validate, schema=ConfigSchema, overrides=non_pipe_overrides + ) + filled["components"] = orig_pipeline + config["components"] = orig_pipeline + create_tokenizer = resolved["nlp"]["tokenizer"] + lemmatizer = resolved["nlp"]["lemmatizer"] + lex_attr_getters = resolved["nlp"]["lex_attr_getters"] + stop_words = resolved["nlp"]["stop_words"] + vocab = Vocab.from_config( + filled, + lemmatizer=lemmatizer, + lex_attr_getters=lex_attr_getters, + stop_words=stop_words, + # TODO: what should we do with these? + tag_map=cls.Defaults.tag_map, + morph_rules=cls.Defaults.morph_rules, + ) + nlp = cls(vocab, create_tokenizer=create_tokenizer) + pipeline = config.get("components", {}) + for pipe_name in nlp_config["pipeline"]: + if pipe_name not in pipeline: + opts = ", ".join(pipeline.keys()) + raise ValueError(Errors.E956.format(name=pipe_name, opts=opts)) + pipe_cfg = pipeline[pipe_name] + if pipe_name not in disable: + if "@factories" not in pipe_cfg: + err = Errors.E984.format(name=pipe_name, config=pipe_cfg) + raise ValueError(err) + factory = pipe_cfg["@factories"] + # The pipe name (key in the config) here is the unique name of the + # component, not necessarily the factory + nlp.add_pipe( + factory, + name=pipe_name, + config=pipe_cfg, + overrides=pipe_overrides, + validate=validate, + ) + nlp.config = filled if auto_fill else config + nlp.resolved = resolved + return nlp + + def to_disk(self, path: Union[str, Path], exclude: Iterable[str] = tuple()) -> None: """Save the current state to a directory. If a model is loaded, this will include the model. @@ -953,7 +1330,9 @@ class Language: serializers["vocab"] = lambda p: self.vocab.to_disk(p) util.to_disk(path, serializers, exclude) - def from_disk(self, path, exclude=tuple()): + def from_disk( + self, path: Union[str, Path], exclude: Iterable[str] = tuple() + ) -> "Language": """Loads state from a directory. Modifies the object in place and returns it. If the saved `Language` object contains a model, the model will be loaded. @@ -965,7 +1344,7 @@ class Language: DOCS: https://spacy.io/api/language#from_disk """ - def deserialize_meta(path): + def deserialize_meta(path: Path) -> None: if path.exists(): data = srsly.read_json(path) self.meta.update(data) @@ -973,7 +1352,7 @@ class Language: # from self.vocab.vectors, so set the name directly self.vocab.vectors.name = data.get("vectors", {}).get("name") - def deserialize_vocab(path): + def deserialize_vocab(path: Path) -> None: if path.exists(): self.vocab.from_disk(path) _fix_pretrained_vectors_name(self) @@ -1004,7 +1383,7 @@ class Language: self._link_components() return self - def to_bytes(self, exclude=tuple()): + def to_bytes(self, exclude: Iterable[str] = tuple()) -> bytes: """Serialize the current state to a binary string. exclude (list): Names of components or serialization fields to exclude. @@ -1025,7 +1404,9 @@ class Language: serializers[name] = lambda proc=proc: proc.to_bytes(exclude=["vocab"]) return util.to_bytes(serializers, exclude) - def from_bytes(self, bytes_data, exclude=tuple()): + def from_bytes( + self, bytes_data: bytes, exclude: Iterable[str] = tuple() + ) -> "Language": """Load state from a binary string. bytes_data (bytes): The data to load from. @@ -1066,69 +1447,25 @@ class Language: return self -class component: - """Decorator for pipeline components. Can decorate both function components - and class components and will automatically register components in the - Language.factories. If the component is a class and needs access to the - nlp object or config parameters, it can expose a from_nlp classmethod - that takes the nlp & model objects and **cfg arguments, and returns the - initialized component. - """ - - # NB: This decorator needs to live here, because it needs to write to - # Language.factories. All other solutions would cause circular import. - - def __init__( - self, - name=None, - assigns=tuple(), - requires=tuple(), - retokenizes=False, - default_model=lambda: None, - default_config=None, - ): - """Decorate a pipeline component. - - name (str): Default component and factory name. - assigns (list): Attributes assigned by component, e.g. `["token.pos"]`. - requires (list): Attributes required by component, e.g. `["token.dep"]`. - retokenizes (bool): Whether the component changes the tokenization. - """ - self.name = name - self.assigns = validate_attrs(assigns) - self.requires = validate_attrs(requires) - self.retokenizes = retokenizes - self.default_model = default_model - self.default_config = default_config - - def __call__(self, *args, **kwargs): - obj = args[0] - args = args[1:] - factory_name = self.name or util.get_component_name(obj) - obj.name = factory_name - obj.factory = factory_name - obj.assigns = self.assigns - obj.requires = self.requires - obj.retokenizes = self.retokenizes - - def factory(nlp, model, **cfg): - if model is None: - model = self.default_model() - if self.default_config: - for key, value in self.default_config.items(): - if key not in cfg: - cfg[key] = value - if hasattr(obj, "from_nlp"): - return obj.from_nlp(nlp, model, **cfg) - elif isinstance(obj, type): - return obj() - return obj - - Language.factories[obj.factory] = factory - return obj +@dataclass +class FactoryMeta: + factory: str + default_config: Optional[Dict[str, Any]] = None # noqa: E704 + assigns: Iterable[str] = tuple() + requires: Iterable[str] = tuple() + retokenizes: bool = False -def _fix_pretrained_vectors_name(nlp): +def _get_config_overrides( + items: Dict[str, Any], prefix: str = "components" +) -> Tuple[Dict[str, Any], Dict[str, Any]]: + prefix = f"{prefix}." + non_pipe = {k: v for k, v in items.items() if not k.startswith(prefix)} + pipe = {k.replace(prefix, ""): v for k, v in items.items() if k.startswith(prefix)} + return non_pipe, pipe + + +def _fix_pretrained_vectors_name(nlp: Language) -> None: # TODO: Replace this once we handle vectors consistently as static # data if "vectors" in nlp.meta and "name" in nlp.meta["vectors"]: @@ -1152,13 +1489,15 @@ def _fix_pretrained_vectors_name(nlp): class DisabledPipes(list): """Manager for temporary pipeline disabling.""" - def __init__(self, nlp, names): + def __init__(self, nlp: Language, names: List[str]): self.nlp = nlp self.names = names # Important! Not deep copy -- we just want the container (but we also # want to support people providing arbitrarily typed nlp.pipeline # objects.) self.original_pipeline = copy(nlp.pipeline) + self.metas = {name: nlp.get_pipe_meta(name) for name in names} + self.configs = {name: nlp.get_pipe_config(name) for name in names} list.__init__(self) self.extend(nlp.remove_pipe(name) for name in names) @@ -1168,7 +1507,7 @@ class DisabledPipes(list): def __exit__(self, *args): self.restore() - def restore(self): + def restore(self) -> None: """Restore the pipeline to its state when DisabledPipes was created.""" current, self.nlp.pipeline = self.nlp.pipeline, self.original_pipeline unexpected = [name for name, pipe in current if not self.nlp.has_pipe(name)] @@ -1176,10 +1515,14 @@ class DisabledPipes(list): # Don't change the pipeline if we're raising an error. self.nlp.pipeline = current raise ValueError(Errors.E008.format(names=unexpected)) + self.nlp._pipe_meta.update(self.metas) + self.nlp._pipe_configs.update(self.configs) self[:] = [] -def _pipe(examples, proc, kwargs): +def _pipe( + examples: Iterable[Example], proc: Callable[[Doc], Doc], kwargs: Dict[str, Any] +) -> Iterator[Example]: # We added some args for pipe that __call__ doesn't expect. kwargs = dict(kwargs) for arg in ["batch_size"]: @@ -1190,14 +1533,23 @@ def _pipe(examples, proc, kwargs): yield eg -def _apply_pipes(make_doc, pipes, receiver, sender, underscore_state): +def _apply_pipes( + make_doc: Callable[[str], Doc], + pipes: Iterable[Callable[[Doc], Doc]], + receiver, + sender, + underscore_state: Tuple[dict, dict, dict], +) -> None: """Worker for Language.pipe + make_doc (Callable[[str,] Doc]): Function to create Doc from text. + pipes (Iterable[Callable[[Doc], Doc]]): The components to apply. receiver (multiprocessing.Connection): Pipe to receive text. Usually created by `multiprocessing.Pipe()` sender (multiprocessing.Connection): Pipe to send doc. Usually created by `multiprocessing.Pipe()` - underscore_state (tuple): The data in the Underscore class of the parent + underscore_state (Tuple[dict, dict, dict]): The data in the Underscore class + of the parent. """ Underscore.load_state(underscore_state) while True: @@ -1212,13 +1564,15 @@ def _apply_pipes(make_doc, pipes, receiver, sender, underscore_state): class _Sender: """Util for sending data to multiprocessing workers in Language.pipe""" - def __init__(self, data, queues, chunk_size): + def __init__( + self, data: Iterable[Any], queues: List[mp.Queue], chunk_size: int + ) -> None: self.data = iter(data) self.queues = iter(cycle(queues)) self.chunk_size = chunk_size self.count = 0 - def send(self): + def send(self) -> None: """Send chunk_size items from self.data to channels.""" for item, q in itertools.islice( zip(self.data, cycle(self.queues)), self.chunk_size @@ -1226,10 +1580,10 @@ class _Sender: # cycle channels so that distribute the texts evenly q.put(item) - def step(self): - """Tell sender that comsumed one item. - - Data is sent to the workers after every chunk_size calls.""" + def step(self) -> None: + """Tell sender that comsumed one item. Data is sent to the workers after + every chunk_size calls. + """ self.count += 1 if self.count >= self.chunk_size: self.count = 0 diff --git a/spacy/lemmatizer.py b/spacy/lemmatizer.py index 50425ea35..81dbf4ea3 100644 --- a/spacy/lemmatizer.py +++ b/spacy/lemmatizer.py @@ -1,5 +1,14 @@ +from typing import Optional, Callable, List, Dict + +from .lookups import Lookups from .errors import Errors from .parts_of_speech import NAMES as UPOS_NAMES +from .util import registry, load_language_data, SimpleFrozenDict + + +@registry.lemmatizers("spacy.Lemmatizer.v1") +def create_lemmatizer(data_paths: dict = {}) -> "Lemmatizer": + return Lemmatizer(data_paths=data_paths) class Lemmatizer: @@ -14,17 +23,27 @@ class Lemmatizer: def load(cls, *args, **kwargs): raise NotImplementedError(Errors.E172) - def __init__(self, lookups, is_base_form=None): + def __init__( + self, + lookups: Optional[Lookups] = None, + data_paths: dict = SimpleFrozenDict(), + is_base_form: Optional[Callable] = None, + ) -> None: """Initialize a Lemmatizer. lookups (Lookups): The lookups object containing the (optional) tables "lemma_rules", "lemma_index", "lemma_exc" and "lemma_lookup". RETURNS (Lemmatizer): The newly constructed object. """ - self.lookups = lookups + self.lookups = lookups if lookups is not None else Lookups() + for name, filename in data_paths.items(): + data = load_language_data(filename) + self.lookups.add_table(name, data) self.is_base_form = is_base_form - def __call__(self, string, univ_pos, morphology=None): + def __call__( + self, string: str, univ_pos: str, morphology: Optional[dict] = None + ) -> List[str]: """Lemmatize a string. string (str): The string to lemmatize, e.g. the token text. @@ -39,7 +58,6 @@ class Lemmatizer: if isinstance(univ_pos, int): univ_pos = UPOS_NAMES.get(univ_pos, "X") univ_pos = univ_pos.lower() - if univ_pos in ("", "eol", "space"): return [string.lower()] # See Issue #435 for example of where this logic is requied. @@ -67,65 +85,31 @@ class Lemmatizer: ) return lemmas - def is_base_form(self, univ_pos, morphology=None): - """ - Check whether we're dealing with an uninflected paradigm, so we can - avoid lemmatization entirely. - - univ_pos (str / int): The token's universal part-of-speech tag. - morphology (dict): The token's morphological features following the - Universal Dependencies scheme. - """ - if morphology is None: - morphology = {} - if univ_pos == "noun" and morphology.get("Number") == "sing": - return True - elif univ_pos == "verb" and morphology.get("VerbForm") == "inf": - return True - # This maps 'VBP' to base form -- probably just need 'IS_BASE' - # morphology - elif univ_pos == "verb" and ( - morphology.get("VerbForm") == "fin" - and morphology.get("Tense") == "pres" - and morphology.get("Number") is None - ): - return True - elif univ_pos == "adj" and morphology.get("Degree") == "pos": - return True - elif morphology.get("VerbForm") == "inf": - return True - elif morphology.get("VerbForm") == "none": - return True - elif morphology.get("Degree") == "pos": - return True - else: - return False - - def noun(self, string, morphology=None): + def noun(self, string: str, morphology: Optional[dict] = None) -> List[str]: return self(string, "noun", morphology) - def verb(self, string, morphology=None): + def verb(self, string: str, morphology: Optional[dict] = None) -> List[str]: return self(string, "verb", morphology) - def adj(self, string, morphology=None): + def adj(self, string: str, morphology: Optional[dict] = None) -> List[str]: return self(string, "adj", morphology) - def det(self, string, morphology=None): + def det(self, string: str, morphology: Optional[dict] = None) -> List[str]: return self(string, "det", morphology) - def pron(self, string, morphology=None): + def pron(self, string: str, morphology: Optional[dict] = None) -> List[str]: return self(string, "pron", morphology) - def adp(self, string, morphology=None): + def adp(self, string: str, morphology: Optional[dict] = None) -> List[str]: return self(string, "adp", morphology) - def num(self, string, morphology=None): + def num(self, string: str, morphology: Optional[dict] = None) -> List[str]: return self(string, "num", morphology) - def punct(self, string, morphology=None): + def punct(self, string: str, morphology: Optional[dict] = None) -> List[str]: return self(string, "punct", morphology) - def lookup(self, string, orth=None): + def lookup(self, string: str, orth: Optional[int] = None) -> str: """Look up a lemma in the table, if available. If no lemma is found, the original string is returned. @@ -141,7 +125,13 @@ class Lemmatizer: return lookup_table[key] return string - def lemmatize(self, string, index, exceptions, rules): + def lemmatize( + self, + string: str, + index: Dict[str, List[str]], + exceptions: Dict[str, Dict[str, List[str]]], + rules: Dict[str, List[List[str]]], + ) -> List[str]: orig = string string = string.lower() forms = [] diff --git a/spacy/lookups.py b/spacy/lookups.py index ddd18a850..b03a326b6 100644 --- a/spacy/lookups.py +++ b/spacy/lookups.py @@ -1,15 +1,32 @@ +from typing import Dict, Any, List, Union, Optional +from pathlib import Path import srsly from preshed.bloom import BloomFilter from collections import OrderedDict from .errors import Errors -from .util import SimpleFrozenDict, ensure_path +from .util import SimpleFrozenDict, ensure_path, registry from .strings import get_string_id UNSET = object() +@registry.language_data("spacy-lookups-data") +def get_lookups(lang: str) -> Dict[str, Any]: + """Load the data from the spacy-lookups-data package for a given language, + if available. Returns an empty dict if there's no data or if the package + is not installed. + + lang (str): The language code (corresponds to entry point exposed by + the spacy-lookups-data package). + RETURNS (Dict[str, Any]): The lookups, keyed by table name. + """ + if lang in registry.lookups: + return registry.lookups.get(lang) + return {} + + class Lookups: """Container for large lookup tables and dictionaries, e.g. lemmatization data or tokenizer exception lists. Lookups are available via vocab.lookups, @@ -18,7 +35,7 @@ class Lookups: via doc.vocab.lookups. """ - def __init__(self): + def __init__(self) -> None: """Initialize the Lookups object. RETURNS (Lookups): The newly created object. @@ -27,7 +44,7 @@ class Lookups: """ self._tables = {} - def __contains__(self, name): + def __contains__(self, name: str) -> bool: """Check if the lookups contain a table of a given name. Delegates to Lookups.has_table. @@ -36,16 +53,16 @@ class Lookups: """ return self.has_table(name) - def __len__(self): + def __len__(self) -> int: """RETURNS (int): The number of tables in the lookups.""" return len(self._tables) @property - def tables(self): - """RETURNS (list): Names of all tables in the lookups.""" + def tables(self) -> List[str]: + """RETURNS (List[str]): Names of all tables in the lookups.""" return list(self._tables.keys()) - def add_table(self, name, data=SimpleFrozenDict()): + def add_table(self, name: str, data: dict = SimpleFrozenDict()) -> "Table": """Add a new table to the lookups. Raises an error if the table exists. name (str): Unique name of table. @@ -60,12 +77,12 @@ class Lookups: self._tables[name] = table return table - def get_table(self, name, default=UNSET): + def get_table(self, name: str, default: Any = UNSET) -> "Table": """Get a table. Raises an error if the table doesn't exist and no default value is provided. name (str): Name of the table. - default: Optional default value to return if table doesn't exist. + default (Any): Optional default value to return if table doesn't exist. RETURNS (Table): The table. DOCS: https://spacy.io/api/lookups#get_table @@ -76,7 +93,7 @@ class Lookups: return default return self._tables[name] - def remove_table(self, name): + def remove_table(self, name: str) -> "Table": """Remove a table. Raises an error if the table doesn't exist. name (str): Name of the table to remove. @@ -88,7 +105,7 @@ class Lookups: raise KeyError(Errors.E159.format(name=name, tables=self.tables)) return self._tables.pop(name) - def has_table(self, name): + def has_table(self, name: str) -> bool: """Check if the lookups contain a table of a given name. name (str): Name of the table. @@ -98,7 +115,7 @@ class Lookups: """ return name in self._tables - def to_bytes(self, **kwargs): + def to_bytes(self, **kwargs) -> bytes: """Serialize the lookups to a bytestring. RETURNS (bytes): The serialized Lookups. @@ -107,7 +124,7 @@ class Lookups: """ return srsly.msgpack_dumps(self._tables) - def from_bytes(self, bytes_data, **kwargs): + def from_bytes(self, bytes_data: bytes, **kwargs) -> "Lookups": """Load the lookups from a bytestring. bytes_data (bytes): The data to load. @@ -120,7 +137,9 @@ class Lookups: self._tables[key] = Table(key, value) return self - def to_disk(self, path, filename="lookups.bin", **kwargs): + def to_disk( + self, path: Union[str, Path], filename: str = "lookups.bin", **kwargs + ) -> None: """Save the lookups to a directory as lookups.bin. Expects a path to a directory, which will be created if it doesn't exist. @@ -136,7 +155,9 @@ class Lookups: with filepath.open("wb") as file_: file_.write(self.to_bytes()) - def from_disk(self, path, filename="lookups.bin", **kwargs): + def from_disk( + self, path: Union[str, Path], filename: str = "lookups.bin", **kwargs + ) -> "Lookups": """Load lookups from a directory containing a lookups.bin. Will skip loading if the file doesn't exist. @@ -162,7 +183,7 @@ class Table(OrderedDict): """ @classmethod - def from_dict(cls, data, name=None): + def from_dict(cls, data: dict, name: Optional[str] = None) -> "Table": """Initialize a new table from a dict. data (dict): The dictionary. @@ -175,7 +196,7 @@ class Table(OrderedDict): self.update(data) return self - def __init__(self, name=None, data=None): + def __init__(self, name: Optional[str] = None, data: Optional[dict] = None) -> None: """Initialize a new table. name (str): Optional table name for reference. @@ -193,7 +214,7 @@ class Table(OrderedDict): if data: self.update(data) - def __setitem__(self, key, value): + def __setitem__(self, key: Union[str, int], value: Any) -> None: """Set new key/value pair. String keys will be hashed. key (str / int): The key to set. @@ -203,7 +224,7 @@ class Table(OrderedDict): OrderedDict.__setitem__(self, key, value) self.bloom.add(key) - def set(self, key, value): + def set(self, key: Union[str, int], value: Any) -> None: """Set new key/value pair. String keys will be hashed. Same as table[key] = value. @@ -212,7 +233,7 @@ class Table(OrderedDict): """ self[key] = value - def __getitem__(self, key): + def __getitem__(self, key: Union[str, int]) -> Any: """Get the value for a given key. String keys will be hashed. key (str / int): The key to get. @@ -221,7 +242,7 @@ class Table(OrderedDict): key = get_string_id(key) return OrderedDict.__getitem__(self, key) - def get(self, key, default=None): + def get(self, key: Union[str, int], default: Optional[Any] = None) -> Any: """Get the value for a given key. String keys will be hashed. key (str / int): The key to get. @@ -231,7 +252,7 @@ class Table(OrderedDict): key = get_string_id(key) return OrderedDict.get(self, key, default) - def __contains__(self, key): + def __contains__(self, key: Union[str, int]) -> bool: """Check whether a key is in the table. String keys will be hashed. key (str / int): The key to check. @@ -243,7 +264,7 @@ class Table(OrderedDict): return False return OrderedDict.__contains__(self, key) - def to_bytes(self): + def to_bytes(self) -> bytes: """Serialize table to a bytestring. RETURNS (bytes): The serialized table. @@ -257,7 +278,7 @@ class Table(OrderedDict): } return srsly.msgpack_dumps(data) - def from_bytes(self, bytes_data): + def from_bytes(self, bytes_data: bytes) -> "Table": """Load a table from a bytestring. bytes_data (bytes): The data to load. diff --git a/spacy/ml/models/entity_linker.py b/spacy/ml/models/entity_linker.py index 00689e85b..ffd6c3c1c 100644 --- a/spacy/ml/models/entity_linker.py +++ b/spacy/ml/models/entity_linker.py @@ -26,8 +26,8 @@ def build_nel_encoder(tok2vec, nO=None): @registry.assets.register("spacy.KBFromFile.v1") -def load_kb(nlp_path, kb_path) -> KnowledgeBase: - vocab = Vocab().from_disk(Path(nlp_path) / "vocab") +def load_kb(vocab_path, kb_path) -> KnowledgeBase: + vocab = Vocab().from_disk(vocab_path) kb = KnowledgeBase(vocab=vocab) kb.load_bulk(kb_path) return kb diff --git a/spacy/ml/models/textcat.py b/spacy/ml/models/textcat.py index 879cac2ec..94e4f72b5 100644 --- a/spacy/ml/models/textcat.py +++ b/spacy/ml/models/textcat.py @@ -1,30 +1,9 @@ -from thinc.api import ( - Model, - reduce_mean, - Linear, - list2ragged, - Logistic, - ParametricAttention, -) -from thinc.api import chain, concatenate, clone, Dropout -from thinc.api import ( - SparseLinear, - Softmax, - softmax_activation, - Maxout, - reduce_sum, - Relu, - residual, - expand_window, -) -from thinc.api import ( - HashEmbed, - with_ragged, - with_array, - with_cpu, - uniqued, - FeatureExtractor, -) +from typing import Optional +from thinc.api import Model, reduce_mean, Linear, list2ragged, Logistic +from thinc.api import chain, concatenate, clone, Dropout, ParametricAttention +from thinc.api import SparseLinear, Softmax, softmax_activation, Maxout, reduce_sum +from thinc.api import HashEmbed, with_ragged, with_array, with_cpu, uniqued +from thinc.api import Relu, residual, expand_window, FeatureExtractor from ..spacy_vectors import SpacyVectors from ... import util @@ -34,7 +13,9 @@ from ..extract_ngrams import extract_ngrams @registry.architectures.register("spacy.TextCatCNN.v1") -def build_simple_cnn_text_classifier(tok2vec, exclusive_classes, nO=None): +def build_simple_cnn_text_classifier( + tok2vec: Model, exclusive_classes: bool, nO: Optional[int] = None +) -> Model: """ Build a simple CNN text classifier, given a token-to-vector model as inputs. If exclusive_classes=True, a softmax non-linearity is applied, so that the @@ -90,13 +71,25 @@ def build_text_classifier( nO=width, nV=embed_size, column=cols.index(LOWER), dropout=dropout, seed=10 ) prefix = HashEmbed( - nO=width // 2, nV=embed_size, column=cols.index(PREFIX), dropout=dropout, seed=11 + nO=width // 2, + nV=embed_size, + column=cols.index(PREFIX), + dropout=dropout, + seed=11, ) suffix = HashEmbed( - nO=width // 2, nV=embed_size, column=cols.index(SUFFIX), dropout=dropout, seed=12 + nO=width // 2, + nV=embed_size, + column=cols.index(SUFFIX), + dropout=dropout, + seed=12, ) shape = HashEmbed( - nO=width // 2, nV=embed_size, column=cols.index(SHAPE), dropout=dropout, seed=13 + nO=width // 2, + nV=embed_size, + column=cols.index(SHAPE), + dropout=dropout, + seed=13, ) width_nI = sum(layer.get_dim("nO") for layer in [lower, prefix, suffix, shape]) diff --git a/spacy/ml/models/tok2vec.py b/spacy/ml/models/tok2vec.py index 2e03d4620..f5249ae24 100644 --- a/spacy/ml/models/tok2vec.py +++ b/spacy/ml/models/tok2vec.py @@ -11,8 +11,8 @@ from ...attrs import ID, ORTH, NORM, PREFIX, SUFFIX, SHAPE @registry.architectures.register("spacy.Tok2VecTensors.v1") -def tok2vec_tensors_v1(width): - tok2vec = Tok2VecListener("tok2vec", width=width) +def tok2vec_tensors_v1(width, upstream="*"): + tok2vec = Tok2VecListener(upstream_name=upstream, width=width) return tok2vec diff --git a/spacy/pipe_analysis.py b/spacy/pipe_analysis.py index 971ebe518..b57f1524b 100644 --- a/spacy/pipe_analysis.py +++ b/spacy/pipe_analysis.py @@ -1,30 +1,37 @@ +from typing import List, Dict, Iterable, Optional, Union, TYPE_CHECKING from wasabi import Printer import warnings from .tokens import Doc, Token, Span from .errors import Errors, Warnings +from .util import dot_to_dict + +if TYPE_CHECKING: + # This lets us add type hints for mypy etc. without causing circular imports + from .language import Language # noqa: F401 -def analyze_pipes(pipeline, name, pipe, index, warn=True): +def analyze_pipes( + nlp: "Language", name: str, index: int, warn: bool = True +) -> List[str]: """Analyze a pipeline component with respect to its position in the current pipeline and the other components. Will check whether requirements are fulfilled (e.g. if previous components assign the attributes). - pipeline (list): A list of (name, pipe) tuples e.g. nlp.pipeline. + nlp (Language): The current nlp object. name (str): The name of the pipeline component to analyze. - pipe (callable): The pipeline component function to analyze. index (int): The index of the component in the pipeline. warn (bool): Show user warning if problem is found. - RETURNS (list): The problems found for the given pipeline component. + RETURNS (List[str]): The problems found for the given pipeline component. """ - assert pipeline[index][0] == name - prev_pipes = pipeline[:index] - pipe_requires = getattr(pipe, "requires", []) - requires = {annot: False for annot in pipe_requires} + assert nlp.pipeline[index][0] == name + prev_pipes = nlp.pipeline[:index] + meta = nlp.get_pipe_meta(name) + requires = {annot: False for annot in meta.requires} if requires: for prev_name, prev_pipe in prev_pipes: - prev_assigns = getattr(prev_pipe, "assigns", []) - for annot in prev_assigns: + prev_meta = nlp.get_pipe_meta(prev_name) + for annot in prev_meta.assigns: requires[annot] = True problems = [] for annot, fulfilled in requires.items(): @@ -35,46 +42,29 @@ def analyze_pipes(pipeline, name, pipe, index, warn=True): return problems -def analyze_all_pipes(pipeline, warn=True): +def analyze_all_pipes(nlp: "Language", warn: bool = True) -> Dict[str, List[str]]: """Analyze all pipes in the pipeline in order. - pipeline (list): A list of (name, pipe) tuples e.g. nlp.pipeline. + nlp (Language): The current nlp object. warn (bool): Show user warning if problem is found. - RETURNS (dict): The problems found, keyed by component name. + RETURNS (Dict[str, List[str]]): The problems found, keyed by component name. """ problems = {} - for i, (name, pipe) in enumerate(pipeline): - problems[name] = analyze_pipes(pipeline, name, pipe, i, warn=warn) + for i, name in enumerate(nlp.pipe_names): + problems[name] = analyze_pipes(nlp, name, i, warn=warn) return problems -def dot_to_dict(values): - """Convert dot notation to a dict. For example: ["token.pos", "token._.xyz"] - become {"token": {"pos": True, "_": {"xyz": True }}}. - - values (iterable): The values to convert. - RETURNS (dict): The converted values. - """ - result = {} - for value in values: - path = result - parts = value.lower().split(".") - for i, item in enumerate(parts): - is_last = i == len(parts) - 1 - path = path.setdefault(item, True if is_last else {}) - return result - - -def validate_attrs(values): +def validate_attrs(values: Iterable[str]) -> Iterable[str]: """Validate component attributes provided to "assigns", "requires" etc. Raises error for invalid attributes and formatting. Doesn't check if custom extension attributes are registered, since this is something the user might want to do themselves later in the component. - values (iterable): The string attributes to check, e.g. `["token.pos"]`. - RETURNS (iterable): The checked attributes. + values (Iterable[str]): The string attributes to check, e.g. `["token.pos"]`. + RETURNS (Iterable[str]): The checked attributes. """ - data = dot_to_dict(values) + data = dot_to_dict({value: True for value in values}) objs = {"doc": Doc, "token": Token, "span": Span} for obj_key, attrs in data.items(): if obj_key == "span": @@ -111,37 +101,40 @@ def validate_attrs(values): return values -def _get_feature_for_attr(pipeline, attr, feature): +def _get_feature_for_attr(nlp: "Language", attr: str, feature: str) -> List[str]: assert feature in ["assigns", "requires"] result = [] - for pipe_name, pipe in pipeline: - pipe_assigns = getattr(pipe, feature, []) + for pipe_name in nlp.pipe_names: + meta = nlp.get_pipe_meta(pipe_name) + pipe_assigns = getattr(meta, feature, []) if attr in pipe_assigns: - result.append((pipe_name, pipe)) + result.append(pipe_name) return result -def get_assigns_for_attr(pipeline, attr): +def get_assigns_for_attr(nlp: "Language", attr: str) -> List[str]: """Get all pipeline components that assign an attr, e.g. "doc.tensor". - pipeline (list): A list of (name, pipe) tuples e.g. nlp.pipeline. + pipeline (Language): The current nlp object. attr (str): The attribute to check. - RETURNS (list): (name, pipeline) tuples of components that assign the attr. + RETURNS (List[str]): Names of components that require the attr. """ - return _get_feature_for_attr(pipeline, attr, "assigns") + return _get_feature_for_attr(nlp, attr, "assigns") -def get_requires_for_attr(pipeline, attr): +def get_requires_for_attr(nlp: "Language", attr: str) -> List[str]: """Get all pipeline components that require an attr, e.g. "doc.tensor". - pipeline (list): A list of (name, pipe) tuples e.g. nlp.pipeline. + pipeline (Language): The current nlp object. attr (str): The attribute to check. - RETURNS (list): (name, pipeline) tuples of components that require the attr. + RETURNS (List[str]): Names of components that require the attr. """ - return _get_feature_for_attr(pipeline, attr, "requires") + return _get_feature_for_attr(nlp, attr, "requires") -def print_summary(nlp, pretty=True, no_print=False): +def print_summary( + nlp: "Language", pretty: bool = True, no_print: bool = False +) -> Optional[Dict[str, Union[List[str], Dict[str, List[str]]]]]: """Print a formatted summary for the current nlp object's pipeline. Shows a table with the pipeline components and why they assign and require, as well as any problems if available. @@ -154,12 +147,10 @@ def print_summary(nlp, pretty=True, no_print=False): msg = Printer(pretty=pretty, no_print=no_print) overview = [] problems = {} - for i, (name, pipe) in enumerate(nlp.pipeline): - requires = getattr(pipe, "requires", []) - assigns = getattr(pipe, "assigns", []) - retok = getattr(pipe, "retokenizes", False) - overview.append((i, name, requires, assigns, retok)) - problems[name] = analyze_pipes(nlp.pipeline, name, pipe, i, warn=False) + for i, name in enumerate(nlp.pipe_names): + meta = nlp.get_pipe_meta(name) + overview.append((i, name, meta.requires, meta.assigns, meta.retokenizes)) + problems[name] = analyze_pipes(nlp, name, i, warn=False) msg.divider("Pipeline Overview") header = ("#", "Component", "Requires", "Assigns", "Retokenizes") msg.table(overview, header=header, divider=True, multiline=True) @@ -175,15 +166,19 @@ def print_summary(nlp, pretty=True, no_print=False): return {"overview": overview, "problems": problems} -def count_pipeline_interdependencies(pipeline): +def count_pipeline_interdependencies(nlp: "Language") -> List[int]: """Count how many subsequent components require an annotation set by each component in the pipeline. + + nlp (Language): The current nlp object. + RETURNS (List[int]): The interdependency counts. """ pipe_assigns = [] pipe_requires = [] - for name, pipe in pipeline: - pipe_assigns.append(set(getattr(pipe, "assigns", []))) - pipe_requires.append(set(getattr(pipe, "requires", []))) + for name in nlp.pipe_names: + meta = nlp.get_pipe_meta(name) + pipe_assigns.append(set(meta.assigns)) + pipe_requires.append(set(meta.requires)) counts = [] for i, assigns in enumerate(pipe_assigns): count = 0 diff --git a/spacy/pipeline/__init__.py b/spacy/pipeline/__init__.py index 116a08e92..5075121bc 100644 --- a/spacy/pipeline/__init__.py +++ b/spacy/pipeline/__init__.py @@ -1,28 +1,33 @@ -from .pipes import Tagger, DependencyParser, EntityRecognizer, EntityLinker -from .pipes import TextCategorizer, Pipe, Sentencizer -from .pipes import SentenceRecognizer -from .simple_ner import SimpleNER -from .morphologizer import Morphologizer +from .dep_parser import DependencyParser +from .entity_linker import EntityLinker +from .ner import EntityRecognizer from .entityruler import EntityRuler +from .morphologizer import Morphologizer +from .pipe import Pipe +from spacy.pipeline.senter import SentenceRecognizer +from .sentencizer import Sentencizer +from .simple_ner import SimpleNER +from .tagger import Tagger +from .textcat import TextCategorizer from .tok2vec import Tok2Vec from .hooks import SentenceSegmenter, SimilarityHook from .functions import merge_entities, merge_noun_chunks, merge_subtokens __all__ = [ - "Tagger", "DependencyParser", - "EntityRecognizer", "EntityLinker", - "TextCategorizer", - "Tok2Vec", - "Pipe", - "Morphologizer", + "EntityRecognizer", "EntityRuler", - "Sentencizer", - "SentenceSegmenter", + "Morphologizer", + "Pipe", "SentenceRecognizer", + "SentenceSegmenter", + "Sentencizer", "SimilarityHook", "SimpleNER", + "Tagger", + "TextCategorizer", + "Tok2Vec", "merge_entities", "merge_noun_chunks", "merge_subtokens", diff --git a/spacy/pipeline/defaults/__init__.py b/spacy/pipeline/defaults/__init__.py deleted file mode 100644 index 483c6bbd6..000000000 --- a/spacy/pipeline/defaults/__init__.py +++ /dev/null @@ -1,93 +0,0 @@ -from pathlib import Path - -from ... import util - - -def default_nel_config(): - loc = Path(__file__).parent / "entity_linker_defaults.cfg" - return util.load_config(loc, create_objects=False) - - -def default_nel(): - loc = Path(__file__).parent / "entity_linker_defaults.cfg" - return util.load_config(loc, create_objects=True)["model"] - - -def default_morphologizer_config(): - loc = Path(__file__).parent / "morphologizer_defaults.cfg" - return util.load_config(loc, create_objects=False) - - -def default_morphologizer(): - loc = Path(__file__).parent / "morphologizer_defaults.cfg" - return util.load_config(loc, create_objects=True)["model"] - - -def default_parser_config(): - loc = Path(__file__).parent / "parser_defaults.cfg" - return util.load_config(loc, create_objects=False) - - -def default_parser(): - loc = Path(__file__).parent / "parser_defaults.cfg" - return util.load_config(loc, create_objects=True)["model"] - - -def default_ner_config(): - loc = Path(__file__).parent / "ner_defaults.cfg" - return util.load_config(loc, create_objects=False) - - -def default_ner(): - loc = Path(__file__).parent / "ner_defaults.cfg" - return util.load_config(loc, create_objects=True)["model"] - - -def default_senter_config(): - loc = Path(__file__).parent / "senter_defaults.cfg" - return util.load_config(loc, create_objects=False) - - -def default_senter(): - loc = Path(__file__).parent / "senter_defaults.cfg" - return util.load_config(loc, create_objects=True)["model"] - - -def default_tagger_config(): - loc = Path(__file__).parent / "tagger_defaults.cfg" - return util.load_config(loc, create_objects=False) - - -def default_tagger(): - loc = Path(__file__).parent / "tagger_defaults.cfg" - return util.load_config(loc, create_objects=True)["model"] - - -def default_textcat_config(): - loc = Path(__file__).parent / "textcat_defaults.cfg" - return util.load_config(loc, create_objects=False) - - -def default_textcat(): - loc = Path(__file__).parent / "textcat_defaults.cfg" - return util.load_config(loc, create_objects=True)["model"] - - -def default_tok2vec_config(): - loc = Path(__file__).parent / "tok2vec_defaults.cfg" - return util.load_config(loc, create_objects=False) - - -def default_tok2vec(): - loc = Path(__file__).parent / "tok2vec_defaults.cfg" - return util.load_config(loc, create_objects=True)["model"] - - -def default_simple_ner_config(): - loc = Path(__file__).parent / "simple_ner_defaults.cfg" - return util.load_config(loc, create_objects=False) - - -def default_simple_ner(): - loc = Path(__file__).parent / "simple_ner_defaults.cfg" - return util.load_config(loc, create_objects=True)["model"] diff --git a/spacy/pipeline/defaults/entity_linker_defaults.cfg b/spacy/pipeline/defaults/entity_linker_defaults.cfg deleted file mode 100644 index 8dddf9e7b..000000000 --- a/spacy/pipeline/defaults/entity_linker_defaults.cfg +++ /dev/null @@ -1,13 +0,0 @@ -[model] -@architectures = "spacy.EntityLinker.v1" - -[model.tok2vec] -@architectures = "spacy.HashEmbedCNN.v1" -pretrained_vectors = null -width = 96 -depth = 2 -embed_size = 300 -window_size = 1 -maxout_pieces = 3 -subword_features = true -dropout = null diff --git a/spacy/pipeline/defaults/morphologizer_defaults.cfg b/spacy/pipeline/defaults/morphologizer_defaults.cfg deleted file mode 100644 index 6ee053a08..000000000 --- a/spacy/pipeline/defaults/morphologizer_defaults.cfg +++ /dev/null @@ -1,14 +0,0 @@ -[model] -@architectures = "spacy.Tagger.v1" - -[model.tok2vec] -@architectures = "spacy.HashCharEmbedCNN.v1" -pretrained_vectors = null -width = 128 -depth = 4 -embed_size = 7000 -window_size = 1 -maxout_pieces = 3 -nM = 64 -nC = 8 -dropout = null diff --git a/spacy/pipeline/defaults/multitask_defaults.cfg b/spacy/pipeline/defaults/multitask_defaults.cfg deleted file mode 100644 index d3dbe9b53..000000000 --- a/spacy/pipeline/defaults/multitask_defaults.cfg +++ /dev/null @@ -1,15 +0,0 @@ -[model] -@architectures = "spacy.MultiTask.v1" -maxout_pieces = 3 -token_vector_width = 96 - -[model.tok2vec] -@architectures = "spacy.HashEmbedCNN.v1" -pretrained_vectors = null -width = 96 -depth = 4 -embed_size = 2000 -window_size = 1 -maxout_pieces = 2 -subword_features = true -dropout = null diff --git a/spacy/pipeline/defaults/ner_defaults.cfg b/spacy/pipeline/defaults/ner_defaults.cfg deleted file mode 100644 index eb926c43b..000000000 --- a/spacy/pipeline/defaults/ner_defaults.cfg +++ /dev/null @@ -1,16 +0,0 @@ -[model] -@architectures = "spacy.TransitionBasedParser.v1" -nr_feature_tokens = 6 -hidden_width = 64 -maxout_pieces = 2 - -[model.tok2vec] -@architectures = "spacy.HashEmbedCNN.v1" -pretrained_vectors = null -width = 96 -depth = 4 -embed_size = 2000 -window_size = 1 -maxout_pieces = 3 -subword_features = true -dropout = null diff --git a/spacy/pipeline/defaults/parser_defaults.cfg b/spacy/pipeline/defaults/parser_defaults.cfg deleted file mode 100644 index 6fe0fd7cb..000000000 --- a/spacy/pipeline/defaults/parser_defaults.cfg +++ /dev/null @@ -1,16 +0,0 @@ -[model] -@architectures = "spacy.TransitionBasedParser.v1" -nr_feature_tokens = 8 -hidden_width = 64 -maxout_pieces = 2 - -[model.tok2vec] -@architectures = "spacy.HashEmbedCNN.v1" -pretrained_vectors = null -width = 96 -depth = 4 -embed_size = 2000 -window_size = 1 -maxout_pieces = 3 -subword_features = true -dropout = null diff --git a/spacy/pipeline/defaults/senter_defaults.cfg b/spacy/pipeline/defaults/senter_defaults.cfg deleted file mode 100644 index 304e42b01..000000000 --- a/spacy/pipeline/defaults/senter_defaults.cfg +++ /dev/null @@ -1,13 +0,0 @@ -[model] -@architectures = "spacy.Tagger.v1" - -[model.tok2vec] -@architectures = "spacy.HashEmbedCNN.v1" -pretrained_vectors = null -width = 12 -depth = 1 -embed_size = 2000 -window_size = 1 -maxout_pieces = 2 -subword_features = true -dropout = null diff --git a/spacy/pipeline/defaults/simple_ner_defaults.cfg b/spacy/pipeline/defaults/simple_ner_defaults.cfg deleted file mode 100644 index 7f206a636..000000000 --- a/spacy/pipeline/defaults/simple_ner_defaults.cfg +++ /dev/null @@ -1,13 +0,0 @@ -[model] -@architectures = "spacy.BiluoTagger.v1" - -[model.tok2vec] -@architectures = "spacy.HashEmbedCNN.v1" -pretrained_vectors = null -width = 128 -depth = 4 -embed_size = 7000 -window_size = 1 -maxout_pieces = 3 -subword_features = true -dropout = null diff --git a/spacy/pipeline/defaults/tagger_defaults.cfg b/spacy/pipeline/defaults/tagger_defaults.cfg deleted file mode 100644 index f26c5f099..000000000 --- a/spacy/pipeline/defaults/tagger_defaults.cfg +++ /dev/null @@ -1,13 +0,0 @@ -[model] -@architectures = "spacy.Tagger.v1" - -[model.tok2vec] -@architectures = "spacy.HashEmbedCNN.v1" -pretrained_vectors = null -width = 96 -depth = 4 -embed_size = 2000 -window_size = 1 -maxout_pieces = 3 -subword_features = true -dropout = null diff --git a/spacy/pipeline/defaults/textcat_bow_defaults.cfg b/spacy/pipeline/defaults/textcat_bow_defaults.cfg deleted file mode 100644 index 84472ea10..000000000 --- a/spacy/pipeline/defaults/textcat_bow_defaults.cfg +++ /dev/null @@ -1,5 +0,0 @@ -[model] -@architectures = "spacy.TextCatBOW.v1" -exclusive_classes = false -ngram_size: 1 -no_output_layer: false diff --git a/spacy/pipeline/defaults/textcat_cnn_defaults.cfg b/spacy/pipeline/defaults/textcat_cnn_defaults.cfg deleted file mode 100644 index 91f3a1742..000000000 --- a/spacy/pipeline/defaults/textcat_cnn_defaults.cfg +++ /dev/null @@ -1,14 +0,0 @@ -[model] -@architectures = "spacy.TextCatCNN.v1" -exclusive_classes = false - -[model.tok2vec] -@architectures = "spacy.HashEmbedCNN.v1" -pretrained_vectors = null -width = 96 -depth = 4 -embed_size = 2000 -window_size = 1 -maxout_pieces = 3 -subword_features = true -dropout = null diff --git a/spacy/pipeline/defaults/textcat_defaults.cfg b/spacy/pipeline/defaults/textcat_defaults.cfg deleted file mode 100644 index 0981cf77c..000000000 --- a/spacy/pipeline/defaults/textcat_defaults.cfg +++ /dev/null @@ -1,10 +0,0 @@ -[model] -@architectures = "spacy.TextCat.v1" -exclusive_classes = false -pretrained_vectors = null -width = 64 -conv_depth = 2 -embed_size = 2000 -window_size = 1 -ngram_size = 1 -dropout = null diff --git a/spacy/pipeline/defaults/tok2vec_defaults.cfg b/spacy/pipeline/defaults/tok2vec_defaults.cfg deleted file mode 100644 index d2718eed1..000000000 --- a/spacy/pipeline/defaults/tok2vec_defaults.cfg +++ /dev/null @@ -1,10 +0,0 @@ -[model] -@architectures = "spacy.HashEmbedCNN.v1" -pretrained_vectors = null -width = 96 -depth = 4 -embed_size = 2000 -window_size = 1 -maxout_pieces = 3 -subword_features = true -dropout = null diff --git a/spacy/pipeline/dep_parser.pyx b/spacy/pipeline/dep_parser.pyx new file mode 100644 index 000000000..1651119f8 --- /dev/null +++ b/spacy/pipeline/dep_parser.pyx @@ -0,0 +1,104 @@ +# cython: infer_types=True, profile=True, binding=True +from typing import Optional, Iterable +from thinc.api import CosineDistance, to_categorical, get_array_module, Model, Config + +from ..syntax.nn_parser cimport Parser +from ..syntax.arc_eager cimport ArcEager + +from .functions import merge_subtokens +from ..language import Language +from ..syntax import nonproj + + +default_model_config = """ +[model] +@architectures = "spacy.TransitionBasedParser.v1" +nr_feature_tokens = 8 +hidden_width = 64 +maxout_pieces = 2 + +[model.tok2vec] +@architectures = "spacy.HashEmbedCNN.v1" +pretrained_vectors = null +width = 96 +depth = 4 +embed_size = 2000 +window_size = 1 +maxout_pieces = 3 +subword_features = true +dropout = null +""" +DEFAULT_PARSER_MODEL = Config().from_str(default_model_config)["model"] + + +@Language.factory( + "parser", + assigns=["token.dep", "token.is_sent_start", "doc.sents"], + default_config={ + "moves": None, + "update_with_oracle_cut_size": 100, + "multitasks": [], + "learn_tokens": False, + "min_action_freq": 30, + "model": DEFAULT_PARSER_MODEL, + } +) +def make_parser( + nlp: Language, + name: str, + model: Model, + moves: Optional[list], + update_with_oracle_cut_size: int, + multitasks: Iterable, + learn_tokens: bool, + min_action_freq: int +): + return DependencyParser( + nlp.vocab, + model, + name, + moves=moves, + update_with_oracle_cut_size=update_with_oracle_cut_size, + multitasks=multitasks, + learn_tokens=learn_tokens, + min_action_freq=min_action_freq + ) + + +cdef class DependencyParser(Parser): + """Pipeline component for dependency parsing. + + DOCS: https://spacy.io/api/dependencyparser + """ + # cdef classes can't have decorators, so we're defining this here + TransitionSystem = ArcEager + + @property + def postprocesses(self): + output = [nonproj.deprojectivize] + if self.cfg.get("learn_tokens") is True: + output.append(merge_subtokens) + return tuple(output) + + def add_multitask_objective(self, mt_component): + self._multitasks.append(mt_component) + + def init_multitask_objectives(self, get_examples, pipeline, sgd=None, **cfg): + # TODO: transfer self.model.get_ref("tok2vec") to the multitask's model ? + for labeller in self._multitasks: + labeller.model.set_dim("nO", len(self.labels)) + if labeller.model.has_ref("output_layer"): + labeller.model.get_ref("output_layer").set_dim("nO", len(self.labels)) + labeller.begin_training(get_examples, pipeline=pipeline, sgd=sgd) + + @property + def labels(self): + labels = set() + # Get the labels from the model by looking at the available moves + for move in self.move_names: + if "-" in move: + label = move.split("-")[1] + if "||" in label: + label = label.split("||")[1] + labels.add(label) + return tuple(sorted(labels)) diff --git a/spacy/pipeline/entity_linker.py b/spacy/pipeline/entity_linker.py new file mode 100644 index 000000000..85a4b6174 --- /dev/null +++ b/spacy/pipeline/entity_linker.py @@ -0,0 +1,366 @@ +from typing import Optional, Iterable, Callable, Dict, Iterator, Union, List, Tuple +from pathlib import Path +import srsly +import random +from thinc.api import CosineDistance, get_array_module, Model, Optimizer, Config +from thinc.api import set_dropout_rate +import warnings + +from ..kb import KnowledgeBase +from ..tokens import Doc +from .pipe import Pipe, deserialize_config +from ..language import Language +from ..vocab import Vocab +from ..gold import Example +from ..errors import Errors, Warnings +from .. import util + + +default_model_config = """ +[model] +@architectures = "spacy.EntityLinker.v1" + +[model.tok2vec] +@architectures = "spacy.HashEmbedCNN.v1" +pretrained_vectors = null +width = 96 +depth = 2 +embed_size = 300 +window_size = 1 +maxout_pieces = 3 +subword_features = true +dropout = null +""" +DEFAULT_NEL_MODEL = Config().from_str(default_model_config)["model"] + + +@Language.factory( + "entity_linker", + requires=["doc.ents", "doc.sents", "token.ent_iob", "token.ent_type"], + assigns=["token.ent_kb_id"], + default_config={ + "kb": None, # TODO - what kind of default makes sense here? + "labels_discard": [], + "incl_prior": True, + "incl_context": True, + "model": DEFAULT_NEL_MODEL, + }, +) +def make_entity_linker( + nlp: Language, + name: str, + model: Model, + kb: Optional[KnowledgeBase], + *, + labels_discard: Iterable[str], + incl_prior: bool, + incl_context: bool, +): + return EntityLinker( + nlp.vocab, + model, + name, + kb=kb, + labels_discard=labels_discard, + incl_prior=incl_prior, + incl_context=incl_context, + ) + + +class EntityLinker(Pipe): + """Pipeline component for named entity linking. + + DOCS: https://spacy.io/api/entitylinker + """ + + NIL = "NIL" # string used to refer to a non-existing link + + def __init__( + self, + vocab: Vocab, + model: Model, + name: str = "entity_linker", + *, + kb: KnowledgeBase, + labels_discard: Iterable[str], + incl_prior: bool, + incl_context: bool, + ) -> None: + self.vocab = vocab + self.model = model + self.name = name + cfg = { + "kb": kb, + "labels_discard": list(labels_discard), + "incl_prior": incl_prior, + "incl_context": incl_context, + } + self.kb = kb + if self.kb is None: + # create an empty KB that should be filled by calling from_disk + self.kb = KnowledgeBase(vocab=vocab) + else: + del cfg["kb"] # we don't want to duplicate its serialization + if not isinstance(self.kb, KnowledgeBase): + raise ValueError(Errors.E990.format(type=type(self.kb))) + self.cfg = dict(cfg) + self.distance = CosineDistance(normalize=False) + # how many neightbour sentences to take into account + self.n_sents = cfg.get("n_sents", 0) + + def require_kb(self) -> None: + # Raise an error if the knowledge base is not initialized. + if len(self.kb) == 0: + raise ValueError(Errors.E139.format(name=self.name)) + + def begin_training( + self, + get_examples: Callable = lambda: [], + pipeline: Optional[List[Tuple[str, Callable[[Doc], Doc]]]] = None, + sgd: Optional[Optimizer] = None, + ) -> Optimizer: + self.require_kb() + nO = self.kb.entity_vector_length + self.set_output(nO) + self.model.initialize() + if sgd is None: + sgd = self.create_optimizer() + return sgd + + def update( + self, + examples: Iterable[Example], + *, + set_annotations: bool = False, + drop: float = 0.0, + sgd: Optional[Optimizer] = None, + losses: Optional[Dict[str, float]] = None, + ) -> Dict[str, float]: + self.require_kb() + if losses is None: + losses = {} + losses.setdefault(self.name, 0.0) + if not examples: + return losses + sentence_docs = [] + try: + docs = [eg.predicted for eg in examples] + except AttributeError: + types = set([type(eg) for eg in examples]) + raise TypeError( + Errors.E978.format(name="EntityLinker", method="update", types=types) + ) + if set_annotations: + # This seems simpler than other ways to get that exact output -- but + # it does run the model twice :( + predictions = self.model.predict(docs) + for eg in examples: + sentences = [s for s in eg.predicted.sents] + kb_ids = eg.get_aligned("ENT_KB_ID", as_string=True) + for ent in eg.predicted.ents: + kb_id = kb_ids[ + ent.start + ] # KB ID of the first token is the same as the whole span + if kb_id: + try: + # find the sentence in the list of sentences. + sent_index = sentences.index(ent.sent) + except AttributeError: + # Catch the exception when ent.sent is None and provide a user-friendly warning + raise RuntimeError(Errors.E030) + # get n previous sentences, if there are any + start_sentence = max(0, sent_index - self.n_sents) + # get n posterior sentences, or as many < n as there are + end_sentence = min(len(sentences) - 1, sent_index + self.n_sents) + # get token positions + start_token = sentences[start_sentence].start + end_token = sentences[end_sentence].end + # append that span as a doc to training + sent_doc = eg.predicted[start_token:end_token].as_doc() + sentence_docs.append(sent_doc) + set_dropout_rate(self.model, drop) + if not sentence_docs: + warnings.warn(Warnings.W093.format(name="Entity Linker")) + return 0.0 + sentence_encodings, bp_context = self.model.begin_update(sentence_docs) + loss, d_scores = self.get_similarity_loss( + sentence_encodings=sentence_encodings, examples=examples + ) + bp_context(d_scores) + if sgd is not None: + self.model.finish_update(sgd) + losses[self.name] += loss + if set_annotations: + self.set_annotations(docs, predictions) + return losses + + def get_similarity_loss(self, examples: Iterable[Example], sentence_encodings): + entity_encodings = [] + for eg in examples: + kb_ids = eg.get_aligned("ENT_KB_ID", as_string=True) + for ent in eg.predicted.ents: + kb_id = kb_ids[ent.start] + if kb_id: + entity_encoding = self.kb.get_vector(kb_id) + entity_encodings.append(entity_encoding) + entity_encodings = self.model.ops.asarray(entity_encodings, dtype="float32") + if sentence_encodings.shape != entity_encodings.shape: + err = Errors.E147.format( + method="get_similarity_loss", msg="gold entities do not match up" + ) + raise RuntimeError(err) + gradients = self.distance.get_grad(sentence_encodings, entity_encodings) + loss = self.distance.get_loss(sentence_encodings, entity_encodings) + loss = loss / len(entity_encodings) + return loss, gradients + + def __call__(self, doc: Doc) -> Doc: + kb_ids = self.predict([doc]) + self.set_annotations([doc], kb_ids) + return doc + + def pipe(self, stream: Iterable[Doc], batch_size: int = 128) -> Iterator[Doc]: + for docs in util.minibatch(stream, size=batch_size): + kb_ids = self.predict(docs) + self.set_annotations(docs, kb_ids) + yield from docs + + def predict(self, docs): + """ Return the KB IDs for each entity in each doc, including NIL if there is no prediction """ + self.require_kb() + entity_count = 0 + final_kb_ids = [] + if not docs: + return final_kb_ids + if isinstance(docs, Doc): + docs = [docs] + for i, doc in enumerate(docs): + sentences = [s for s in doc.sents] + if len(doc) > 0: + # Looping through each sentence and each entity + # This may go wrong if there are entities across sentences - which shouldn't happen normally. + for sent_index, sent in enumerate(sentences): + if sent.ents: + # get n_neightbour sentences, clipped to the length of the document + start_sentence = max(0, sent_index - self.n_sents) + end_sentence = min( + len(sentences) - 1, sent_index + self.n_sents + ) + start_token = sentences[start_sentence].start + end_token = sentences[end_sentence].end + sent_doc = doc[start_token:end_token].as_doc() + # currently, the context is the same for each entity in a sentence (should be refined) + sentence_encoding = self.model.predict([sent_doc])[0] + xp = get_array_module(sentence_encoding) + sentence_encoding_t = sentence_encoding.T + sentence_norm = xp.linalg.norm(sentence_encoding_t) + for ent in sent.ents: + entity_count += 1 + to_discard = self.cfg.get("labels_discard", []) + if to_discard and ent.label_ in to_discard: + # ignoring this entity - setting to NIL + final_kb_ids.append(self.NIL) + else: + candidates = self.kb.get_candidates(ent.text) + if not candidates: + # no prediction possible for this entity - setting to NIL + final_kb_ids.append(self.NIL) + elif len(candidates) == 1: + # shortcut for efficiency reasons: take the 1 candidate + # TODO: thresholding + final_kb_ids.append(candidates[0].entity_) + else: + random.shuffle(candidates) + # this will set all prior probabilities to 0 if they should be excluded from the model + prior_probs = xp.asarray( + [c.prior_prob for c in candidates] + ) + if not self.cfg.get("incl_prior"): + prior_probs = xp.asarray( + [0.0 for c in candidates] + ) + scores = prior_probs + # add in similarity from the context + if self.cfg.get("incl_context"): + entity_encodings = xp.asarray( + [c.entity_vector for c in candidates] + ) + entity_norm = xp.linalg.norm( + entity_encodings, axis=1 + ) + if len(entity_encodings) != len(prior_probs): + raise RuntimeError( + Errors.E147.format( + method="predict", + msg="vectors not of equal length", + ) + ) + # cosine similarity + sims = xp.dot( + entity_encodings, sentence_encoding_t + ) / (sentence_norm * entity_norm) + if sims.shape != prior_probs.shape: + raise ValueError(Errors.E161) + scores = ( + prior_probs + sims - (prior_probs * sims) + ) + # TODO: thresholding + best_index = scores.argmax().item() + best_candidate = candidates[best_index] + final_kb_ids.append(best_candidate.entity_) + if not (len(final_kb_ids) == entity_count): + err = Errors.E147.format( + method="predict", msg="result variables not of equal length" + ) + raise RuntimeError(err) + return final_kb_ids + + def set_annotations(self, docs: Iterable[Doc], kb_ids: List[int]) -> None: + count_ents = len([ent for doc in docs for ent in doc.ents]) + if count_ents != len(kb_ids): + raise ValueError(Errors.E148.format(ents=count_ents, ids=len(kb_ids))) + i = 0 + for doc in docs: + for ent in doc.ents: + kb_id = kb_ids[i] + i += 1 + for token in ent: + token.ent_kb_id_ = kb_id + + def to_disk(self, path: Union[str, Path], exclude: Iterable[str] = tuple()) -> None: + serialize = {} + self.cfg["entity_width"] = self.kb.entity_vector_length + serialize["cfg"] = lambda p: srsly.write_json(p, self.cfg) + serialize["vocab"] = lambda p: self.vocab.to_disk(p) + serialize["kb"] = lambda p: self.kb.dump(p) + serialize["model"] = lambda p: self.model.to_disk(p) + util.to_disk(path, serialize, exclude) + + def from_disk( + self, path: Union[str, Path], exclude: Iterable[str] = tuple() + ) -> "EntityLinker": + def load_model(p): + try: + self.model.from_bytes(p.open("rb").read()) + except AttributeError: + raise ValueError(Errors.E149) + + def load_kb(p): + self.kb = KnowledgeBase( + vocab=self.vocab, entity_vector_length=self.cfg["entity_width"] + ) + self.kb.load_bulk(p) + + deserialize = {} + deserialize["vocab"] = lambda p: self.vocab.from_disk(p) + deserialize["cfg"] = lambda p: self.cfg.update(deserialize_config(p)) + deserialize["kb"] = load_kb + deserialize["model"] = load_model + util.from_disk(path, deserialize, exclude) + return self + + def rehearse(self, examples, sgd=None, losses=None, **config): + raise NotImplementedError + + def add_label(self, label): + raise NotImplementedError diff --git a/spacy/pipeline/entityruler.py b/spacy/pipeline/entityruler.py index d9c950ad0..07863b8e9 100644 --- a/spacy/pipeline/entityruler.py +++ b/spacy/pipeline/entityruler.py @@ -1,16 +1,47 @@ +from typing import Optional, Union, List, Dict, Tuple, Iterable, Any from collections import defaultdict +from pathlib import Path import srsly -from ..language import component +from ..language import Language from ..errors import Errors from ..util import ensure_path, to_disk, from_disk from ..tokens import Doc, Span from ..matcher import Matcher, PhraseMatcher + DEFAULT_ENT_ID_SEP = "||" +PatternType = Dict[str, Union[str, List[Dict[str, Any]]]] + + +@Language.factory( + "entity_ruler", + assigns=["doc.ents", "token.ent_type", "token.ent_iob"], + default_config={ + "phrase_matcher_attr": None, + "validation": False, + "overwrite_ents": False, + "ent_id_sep": DEFAULT_ENT_ID_SEP, + }, +) +def make_entity_ruler( + nlp: Language, + name: str, + phrase_matcher_attr: Optional[Union[int, str]], + validation: bool, + overwrite_ents: bool, + ent_id_sep: str, +): + return EntityRuler( + nlp, + name, + phrase_matcher_attr=phrase_matcher_attr, + validate=validation, + overwrite_ents=overwrite_ents, + ent_id_sep=ent_id_sep, + ) -@component("entity_ruler", assigns=["doc.ents", "token.ent_type", "token.ent_iob"]) class EntityRuler: """The EntityRuler lets you add spans to the `Doc.ents` using token-based rules or exact phrase matches. It can be combined with the statistical @@ -22,7 +53,17 @@ class EntityRuler: USAGE: https://spacy.io/usage/rule-based-matching#entityruler """ - def __init__(self, nlp, phrase_matcher_attr=None, validate=False, **cfg): + def __init__( + self, + nlp: Language, + name: str = "entity_ruler", + *, + phrase_matcher_attr: Optional[Union[int, str]] = None, + validate: bool = False, + overwrite_ents: bool = False, + ent_id_sep: str = DEFAULT_ENT_ID_SEP, + patterns: Optional[List[PatternType]] = None, + ): """Initialize the entitiy ruler. If patterns are supplied here, they need to be a list of dictionaries with a `"label"` and `"pattern"` key. A pattern can either be a token pattern (list) or a phrase pattern @@ -37,15 +78,14 @@ class EntityRuler: patterns (iterable): Optional patterns to load in. overwrite_ents (bool): If existing entities are present, e.g. entities added by the model, overwrite them by matches if necessary. - **cfg: Other config parameters. If pipeline component is loaded as part - of a model pipeline, this will include all keyword arguments passed - to `spacy.load`. + ent_id_sep (str): Separator used internally for entity IDs. RETURNS (EntityRuler): The newly constructed object. DOCS: https://spacy.io/api/entityruler#init """ self.nlp = nlp - self.overwrite = cfg.get("overwrite_ents", False) + self.name = name + self.overwrite = overwrite_ents self.token_patterns = defaultdict(list) self.phrase_patterns = defaultdict(list) self.matcher = Matcher(nlp.vocab, validate=validate) @@ -59,27 +99,22 @@ class EntityRuler: else: self.phrase_matcher_attr = None self.phrase_matcher = PhraseMatcher(nlp.vocab, validate=validate) - self.ent_id_sep = cfg.get("ent_id_sep", DEFAULT_ENT_ID_SEP) + self.ent_id_sep = ent_id_sep self._ent_ids = defaultdict(dict) - patterns = cfg.get("patterns") if patterns is not None: self.add_patterns(patterns) - @classmethod - def from_nlp(cls, nlp, model=None, **cfg): - return cls(nlp, **cfg) - - def __len__(self): + def __len__(self) -> int: """The number of all patterns added to the entity ruler.""" n_token_patterns = sum(len(p) for p in self.token_patterns.values()) n_phrase_patterns = sum(len(p) for p in self.phrase_patterns.values()) return n_token_patterns + n_phrase_patterns - def __contains__(self, label): + def __contains__(self, label: str) -> bool: """Whether a label is present in the patterns.""" return label in self.token_patterns or label in self.phrase_patterns - def __call__(self, doc): + def __call__(self, doc: Doc) -> Doc: """Find matches in document and add them as entities. doc (Doc): The Doc object in the pipeline. @@ -118,7 +153,7 @@ class EntityRuler: return doc @property - def labels(self): + def labels(self) -> Tuple[str, ...]: """All labels present in the match patterns. RETURNS (set): The string labels. @@ -138,7 +173,7 @@ class EntityRuler: return tuple(all_labels) @property - def ent_ids(self): + def ent_ids(self) -> Tuple[str, ...]: """All entity ids present in the match patterns `id` properties RETURNS (set): The string entity ids. @@ -156,7 +191,7 @@ class EntityRuler: return tuple(all_ent_ids) @property - def patterns(self): + def patterns(self) -> List[PatternType]: """Get all patterns that were added to the entity ruler. RETURNS (list): The original patterns, one dictionary per pattern. @@ -178,10 +213,9 @@ class EntityRuler: if ent_id: p["id"] = ent_id all_patterns.append(p) - return all_patterns - def add_patterns(self, patterns): + def add_patterns(self, patterns: List[PatternType]) -> None: """Add patterns to the entitiy ruler. A pattern can either be a token pattern (list of dicts) or a phrase pattern (string). For example: {'label': 'ORG', 'pattern': 'Apple'} @@ -245,11 +279,16 @@ class EntityRuler: for label, patterns in self.phrase_patterns.items(): self.phrase_matcher.add(label, patterns) - def _split_label(self, label): + def clear(self) -> None: + """Reset all patterns.""" + self.token_patterns = defaultdict(list) + self.phrase_patterns = defaultdict(list) + self._ent_ids = defaultdict(dict) + + def _split_label(self, label: str) -> Tuple[str, str]: """Split Entity label into ent_label and ent_id if it contains self.ent_id_sep label (str): The value of label in a pattern entry - RETURNS (tuple): ent_label, ent_id """ if self.ent_id_sep in label: @@ -257,46 +296,45 @@ class EntityRuler: else: ent_label = label ent_id = None - return ent_label, ent_id - def _create_label(self, label, ent_id): + def _create_label(self, label: str, ent_id: str) -> str: """Join Entity label with ent_id if the pattern has an `id` attribute label (str): The label to set for ent.label_ ent_id (str): The label - RETURNS (str): The ent_label joined with configured `ent_id_sep` """ if isinstance(ent_id, str): label = f"{label}{self.ent_id_sep}{ent_id}" return label - def from_bytes(self, patterns_bytes, **kwargs): + def from_bytes( + self, patterns_bytes: bytes, exclude: Iterable[str] = tuple() + ) -> "EntityRuler": """Load the entity ruler from a bytestring. patterns_bytes (bytes): The bytestring to load. - **kwargs: Other config paramters, mostly for consistency. - RETURNS (EntityRuler): The loaded entity ruler. DOCS: https://spacy.io/api/entityruler#from_bytes """ cfg = srsly.msgpack_loads(patterns_bytes) + self.clear() if isinstance(cfg, dict): self.add_patterns(cfg.get("patterns", cfg)) - self.overwrite = cfg.get("overwrite", False) + self.overwrite = cfg.get("overwrite") self.phrase_matcher_attr = cfg.get("phrase_matcher_attr", None) if self.phrase_matcher_attr is not None: self.phrase_matcher = PhraseMatcher( self.nlp.vocab, attr=self.phrase_matcher_attr ) - self.ent_id_sep = cfg.get("ent_id_sep", DEFAULT_ENT_ID_SEP) + self.ent_id_sep = cfg.get("ent_id_sep") else: self.add_patterns(cfg) return self - def to_bytes(self, **kwargs): + def to_bytes(self, exclude: Iterable[str] = tuple()) -> bytes: """Serialize the entity ruler patterns to a bytestring. RETURNS (bytes): The serialized patterns. @@ -311,18 +349,19 @@ class EntityRuler: } return srsly.msgpack_dumps(serial) - def from_disk(self, path, **kwargs): + def from_disk( + self, path: Union[str, Path], exclude: Iterable[str] = tuple() + ) -> "EntityRuler": """Load the entity ruler from a file. Expects a file containing newline-delimited JSON (JSONL) with one entry per line. path (str / Path): The JSONL file to load. - **kwargs: Other config paramters, mostly for consistency. - RETURNS (EntityRuler): The loaded entity ruler. DOCS: https://spacy.io/api/entityruler#from_disk """ path = ensure_path(path) + self.clear() depr_patterns_path = path.with_suffix(".jsonl") if depr_patterns_path.is_file(): patterns = srsly.read_jsonl(depr_patterns_path) @@ -336,9 +375,9 @@ class EntityRuler: } deserializers_cfg = {"cfg": lambda p: cfg.update(srsly.read_json(p))} from_disk(path, deserializers_cfg, {}) - self.overwrite = cfg.get("overwrite", False) + self.overwrite = cfg.get("overwrite") self.phrase_matcher_attr = cfg.get("phrase_matcher_attr") - self.ent_id_sep = cfg.get("ent_id_sep", DEFAULT_ENT_ID_SEP) + self.ent_id_sep = cfg.get("ent_id_sep") if self.phrase_matcher_attr is not None: self.phrase_matcher = PhraseMatcher( @@ -347,12 +386,11 @@ class EntityRuler: from_disk(path, deserializers_patterns, {}) return self - def to_disk(self, path, **kwargs): + def to_disk(self, path: Union[str, Path], exclude: Iterable[str] = tuple()) -> None: """Save the entity ruler patterns to a directory. The patterns will be saved as newline-delimited JSON (JSONL). path (str / Path): The JSONL file to save. - **kwargs: Other config paramters, mostly for consistency. DOCS: https://spacy.io/api/entityruler#to_disk """ diff --git a/spacy/pipeline/functions.py b/spacy/pipeline/functions.py index 622791512..8a6a5188f 100644 --- a/spacy/pipeline/functions.py +++ b/spacy/pipeline/functions.py @@ -1,14 +1,15 @@ -from ..language import component +from ..language import Language from ..matcher import Matcher +from ..tokens import Doc from ..util import filter_spans -@component( +@Language.component( "merge_noun_chunks", requires=["token.dep", "token.tag", "token.pos"], retokenizes=True, ) -def merge_noun_chunks(doc): +def merge_noun_chunks(doc: Doc) -> Doc: """Merge noun chunks into a single token. doc (Doc): The Doc object. @@ -25,12 +26,12 @@ def merge_noun_chunks(doc): return doc -@component( +@Language.component( "merge_entities", requires=["doc.ents", "token.ent_iob", "token.ent_type"], retokenizes=True, ) -def merge_entities(doc): +def merge_entities(doc: Doc): """Merge entities into a single token. doc (Doc): The Doc object. @@ -45,8 +46,8 @@ def merge_entities(doc): return doc -@component("merge_subtokens", requires=["token.dep"], retokenizes=True) -def merge_subtokens(doc, label="subtok"): +@Language.component("merge_subtokens", requires=["token.dep"], retokenizes=True) +def merge_subtokens(doc: Doc, label: str = "subtok") -> Doc: """Merge subtokens into a single token. doc (Doc): The Doc object. @@ -55,6 +56,7 @@ def merge_subtokens(doc, label="subtok"): DOCS: https://spacy.io/api/pipeline-functions#merge_subtokens """ + # TODO: make stateful component with "label" config merger = Matcher(doc.vocab) merger.add("SUBTOK", None, [{"DEP": label, "op": "+"}]) matches = merger(doc) diff --git a/spacy/pipeline/hooks.py b/spacy/pipeline/hooks.py index 368e120ab..60654f6b7 100644 --- a/spacy/pipeline/hooks.py +++ b/spacy/pipeline/hooks.py @@ -1,11 +1,12 @@ from thinc.api import concatenate, reduce_max, reduce_mean, siamese, CauchySimilarity -from .pipes import Pipe -from ..language import component +from .pipe import Pipe from ..util import link_vectors_to_models -@component("sentencizer_hook", assigns=["doc.user_hooks"]) +# TODO: do we want to keep these? + + class SentenceSegmenter: """A simple spaCy hook, to allow custom sentence boundary detection logic (that doesn't require the dependency parse). To change the sentence @@ -40,7 +41,6 @@ class SentenceSegmenter: yield doc[start : len(doc)] -@component("similarity", assigns=["doc.user_hooks"]) class SimilarityHook(Pipe): """ Experimental: A pipeline component to install a hook for supervised diff --git a/spacy/pipeline/morphologizer.pyx b/spacy/pipeline/morphologizer.pyx index bc77dda47..a5a54f139 100644 --- a/spacy/pipeline/morphologizer.pyx +++ b/spacy/pipeline/morphologizer.pyx @@ -1,9 +1,7 @@ -# cython: infer_types=True, profile=True -cimport numpy as np - -import numpy +# cython: infer_types=True, profile=True, binding=True +from typing import Optional import srsly -from thinc.api import SequenceCategoricalCrossentropy +from thinc.api import SequenceCategoricalCrossentropy, Model, Config from ..tokens.doc cimport Doc from ..vocab cimport Vocab @@ -11,31 +9,67 @@ from ..morphology cimport Morphology from ..parts_of_speech import IDS as POS_IDS from ..symbols import POS +from ..language import Language +from ..errors import Errors +from .pipe import deserialize_config +from .tagger import Tagger from .. import util -from ..language import component -from ..util import link_vectors_to_models, create_default_optimizer -from ..errors import Errors, TempErrors -from .pipes import Tagger, _load_cfg -from .. import util -from .defaults import default_morphologizer -@component("morphologizer", assigns=["token.morph", "token.pos"], default_model=default_morphologizer) +default_model_config = """ +[model] +@architectures = "spacy.Tagger.v1" + +[model.tok2vec] +@architectures = "spacy.HashCharEmbedCNN.v1" +pretrained_vectors = null +width = 128 +depth = 4 +embed_size = 7000 +window_size = 1 +maxout_pieces = 3 +nM = 64 +nC = 8 +dropout = null +""" +DEFAULT_MORPH_MODEL = Config().from_str(default_model_config)["model"] + + +@Language.factory( + "morphologizer", + assigns=["token.morph", "token.pos"], + default_config={"model": DEFAULT_MORPH_MODEL} +) +def make_morphologizer( + nlp: Language, + model: Model, + name: str, +): + return Morphologizer(nlp.vocab, model, name) + + class Morphologizer(Tagger): - POS_FEAT = "POS" - def __init__(self, vocab, model, **cfg): + def __init__( + self, + vocab: Vocab, + model: Model, + name: str = "morphologizer", + *, + labels_morph: Optional[dict] = None, + labels_pos: Optional[dict] = None, + ): self.vocab = vocab self.model = model + self.name = name self._rehearsal_model = None - self.cfg = dict(sorted(cfg.items())) # to be able to set annotations without string operations on labels, # store mappings from morph+POS labels to token-level annotations: # 1) labels_morph stores a mapping from morph+POS->morph - self.cfg.setdefault("labels_morph", {}) # 2) labels_pos stores a mapping from morph+POS->POS - self.cfg.setdefault("labels_pos", {}) + cfg = {"labels_morph": labels_morph or {}, "labels_pos": labels_pos or {}} + self.cfg = dict(sorted(cfg.items())) # add mappings for empty morph self.cfg["labels_morph"][Morphology.EMPTY_MORPH] = Morphology.EMPTY_MORPH self.cfg["labels_pos"][Morphology.EMPTY_MORPH] = POS_IDS[""] @@ -64,8 +98,7 @@ class Morphologizer(Tagger): self.cfg["labels_pos"][norm_label] = POS_IDS[pos] return 1 - def begin_training(self, get_examples=lambda: [], pipeline=None, sgd=None, - **kwargs): + def begin_training(self, get_examples=lambda: [], pipeline=None, sgd=None): for example in get_examples(): for i, token in enumerate(example.reference): pos = token.pos_ @@ -81,7 +114,7 @@ class Morphologizer(Tagger): self.cfg["labels_pos"][norm_label] = POS_IDS[pos] self.set_output(len(self.labels)) self.model.initialize() - link_vectors_to_models(self.vocab) + util.link_vectors_to_models(self.vocab) if sgd is None: sgd = self.create_optimizer() return sgd @@ -169,7 +202,7 @@ class Morphologizer(Tagger): deserialize = { "vocab": lambda p: self.vocab.from_disk(p), - "cfg": lambda p: self.cfg.update(_load_cfg(p)), + "cfg": lambda p: self.cfg.update(deserialize_config(p)), "model": load_model, } util.from_disk(path, deserialize, exclude) diff --git a/spacy/pipeline/multitask.pyx b/spacy/pipeline/multitask.pyx new file mode 100644 index 000000000..4945afe4f --- /dev/null +++ b/spacy/pipeline/multitask.pyx @@ -0,0 +1,224 @@ +# cython: infer_types=True, profile=True, binding=True +from typing import Optional +import numpy +from thinc.api import CosineDistance, to_categorical, to_categorical, Model, Config +from thinc.api import set_dropout_rate + +from ..tokens.doc cimport Doc + +from .pipe import Pipe +from .tagger import Tagger +from ..language import Language +from ..syntax import nonproj +from ..attrs import POS, ID +from ..util import link_vectors_to_models +from ..errors import Errors + + +default_model_config = """ +[model] +@architectures = "spacy.MultiTask.v1" +maxout_pieces = 3 +token_vector_width = 96 + +[model.tok2vec] +@architectures = "spacy.HashEmbedCNN.v1" +pretrained_vectors = null +width = 96 +depth = 4 +embed_size = 2000 +window_size = 1 +maxout_pieces = 2 +subword_features = true +dropout = null +""" +DEFAULT_MT_MODEL = Config().from_str(default_model_config)["model"] + + +@Language.factory( + "nn_labeller", + default_config={"labels": None, "target": "dep_tag_offset", "model": DEFAULT_MT_MODEL} +) +def make_nn_labeller(nlp: Language, name: str, model: Model, labels: Optional[dict], target: str): + return MultitaskObjective(nlp.vocab, model, name) + + +class MultitaskObjective(Tagger): + """Experimental: Assist training of a parser or tagger, by training a + side-objective. + """ + + def __init__(self, vocab, model, name="nn_labeller", *, labels, target): + self.vocab = vocab + self.model = model + self.name = name + if target == "dep": + self.make_label = self.make_dep + elif target == "tag": + self.make_label = self.make_tag + elif target == "ent": + self.make_label = self.make_ent + elif target == "dep_tag_offset": + self.make_label = self.make_dep_tag_offset + elif target == "ent_tag": + self.make_label = self.make_ent_tag + elif target == "sent_start": + self.make_label = self.make_sent_start + elif hasattr(target, "__call__"): + self.make_label = target + else: + raise ValueError(Errors.E016) + cfg = {"labels": labels or {}, "target": target} + self.cfg = dict(cfg) + + @property + def labels(self): + return self.cfg.setdefault("labels", {}) + + @labels.setter + def labels(self, value): + self.cfg["labels"] = value + + def set_annotations(self, docs, dep_ids): + pass + + def begin_training(self, get_examples=lambda: [], pipeline=None, sgd=None): + gold_examples = nonproj.preprocess_training_data(get_examples()) + # for raw_text, doc_annot in gold_tuples: + for example in gold_examples: + for token in example.y: + label = self.make_label(token) + if label is not None and label not in self.labels: + self.labels[label] = len(self.labels) + self.model.initialize() + link_vectors_to_models(self.vocab) + if sgd is None: + sgd = self.create_optimizer() + return sgd + + def predict(self, docs): + tokvecs = self.model.get_ref("tok2vec")(docs) + scores = self.model.get_ref("softmax")(tokvecs) + return tokvecs, scores + + def get_loss(self, examples, scores): + cdef int idx = 0 + correct = numpy.zeros((scores.shape[0],), dtype="i") + guesses = scores.argmax(axis=1) + docs = [eg.predicted for eg in examples] + for i, eg in enumerate(examples): + # Handles alignment for tokenization differences + doc_annots = eg.get_aligned() # TODO + for j in range(len(eg.predicted)): + tok_annots = {key: values[j] for key, values in tok_annots.items()} + label = self.make_label(j, tok_annots) + if label is None or label not in self.labels: + correct[idx] = guesses[idx] + else: + correct[idx] = self.labels[label] + idx += 1 + correct = self.model.ops.xp.array(correct, dtype="i") + d_scores = scores - to_categorical(correct, n_classes=scores.shape[1]) + loss = (d_scores**2).sum() + return float(loss), d_scores + + @staticmethod + def make_dep(token): + return token.dep_ + + @staticmethod + def make_tag(token): + return token.tag_ + + @staticmethod + def make_ent(token): + if token.ent_iob_ == "O": + return "O" + else: + return token.ent_iob_ + "-" + token.ent_type_ + + @staticmethod + def make_dep_tag_offset(token): + dep = token.dep_ + tag = token.tag_ + offset = token.head.i - token.i + offset = min(offset, 2) + offset = max(offset, -2) + return f"{dep}-{tag}:{offset}" + + @staticmethod + def make_ent_tag(token): + if token.ent_iob_ == "O": + ent = "O" + else: + ent = token.ent_iob_ + "-" + token.ent_type_ + tag = token.tag_ + return f"{tag}-{ent}" + + @staticmethod + def make_sent_start(token): + """A multi-task objective for representing sentence boundaries, + using BILU scheme. (O is impossible) + """ + if token.is_sent_start and token.is_sent_end: + return "U-SENT" + elif token.is_sent_start: + return "B-SENT" + else: + return "I-SENT" + + +class ClozeMultitask(Pipe): + def __init__(self, vocab, model, **cfg): + self.vocab = vocab + self.model = model + self.cfg = cfg + self.distance = CosineDistance(ignore_zeros=True, normalize=False) # TODO: in config + + def set_annotations(self, docs, dep_ids): + pass + + def begin_training(self, get_examples=lambda: [], pipeline=None, sgd=None): + link_vectors_to_models(self.vocab) + self.model.initialize() + X = self.model.ops.alloc((5, self.model.get_ref("tok2vec").get_dim("nO"))) + self.model.output_layer.begin_training(X) + if sgd is None: + sgd = self.create_optimizer() + return sgd + + def predict(self, docs): + tokvecs = self.model.get_ref("tok2vec")(docs) + vectors = self.model.get_ref("output_layer")(tokvecs) + return tokvecs, vectors + + def get_loss(self, examples, vectors, prediction): + # The simplest way to implement this would be to vstack the + # token.vector values, but that's a bit inefficient, especially on GPU. + # Instead we fetch the index into the vectors table for each of our tokens, + # and look them up all at once. This prevents data copying. + ids = self.model.ops.flatten([eg.predicted.to_array(ID).ravel() for eg in examples]) + target = vectors[ids] + gradient = self.distance.get_grad(prediction, target) + loss = self.distance.get_loss(prediction, target) + return loss, gradient + + def update(self, examples, *, drop=0., set_annotations=False, sgd=None, losses=None): + pass + + def rehearse(self, examples, drop=0., sgd=None, losses=None): + if losses is not None and self.name not in losses: + losses[self.name] = 0. + set_dropout_rate(self.model, drop) + try: + predictions, bp_predictions = self.model.begin_update([eg.predicted for eg in examples]) + except AttributeError: + types = set([type(eg) for eg in examples]) + raise TypeError(Errors.E978.format(name="ClozeMultitask", method="rehearse", types=types)) + loss, d_predictions = self.get_loss(examples, self.vocab.vectors.data, predictions) + bp_predictions(d_predictions) + if sgd is not None: + self.model.finish_update(sgd) + + if losses is not None: + losses[self.name] += loss diff --git a/spacy/pipeline/ner.pyx b/spacy/pipeline/ner.pyx new file mode 100644 index 000000000..ea904f69e --- /dev/null +++ b/spacy/pipeline/ner.pyx @@ -0,0 +1,90 @@ +# cython: infer_types=True, profile=True, binding=True +from typing import Optional, Iterable +from thinc.api import CosineDistance, to_categorical, get_array_module, Model, Config + +from ..syntax.nn_parser cimport Parser +from ..syntax.ner cimport BiluoPushDown + +from ..language import Language + + +default_model_config = """ +[model] +@architectures = "spacy.TransitionBasedParser.v1" +nr_feature_tokens = 6 +hidden_width = 64 +maxout_pieces = 2 + +[model.tok2vec] +@architectures = "spacy.HashEmbedCNN.v1" +pretrained_vectors = null +width = 96 +depth = 4 +embed_size = 2000 +window_size = 1 +maxout_pieces = 3 +subword_features = true +dropout = null +""" +DEFAULT_NER_MODEL = Config().from_str(default_model_config)["model"] + + +@Language.factory( + "ner", + assigns=["doc.ents", "token.ent_iob", "token.ent_type"], + default_config={ + "moves": None, + "update_with_oracle_cut_size": 100, + "multitasks": [], + "learn_tokens": False, + "min_action_freq": 30, + "model": DEFAULT_NER_MODEL, + } +) +def make_ner( + nlp: Language, + name: str, + model: Model, + moves: Optional[list], + update_with_oracle_cut_size: int, + multitasks: Iterable, + learn_tokens: bool, + min_action_freq: int +): + return EntityRecognizer( + nlp.vocab, + model, + name, + moves=moves, + update_with_oracle_cut_size=update_with_oracle_cut_size, + multitasks=multitasks, + learn_tokens=learn_tokens, + min_action_freq=min_action_freq + ) + + +cdef class EntityRecognizer(Parser): + """Pipeline component for named entity recognition. + + DOCS: https://spacy.io/api/entityrecognizer + """ + TransitionSystem = BiluoPushDown + + def add_multitask_objective(self, mt_component): + self._multitasks.append(mt_component) + + def init_multitask_objectives(self, get_examples, pipeline, sgd=None, **cfg): + # TODO: transfer self.model.get_ref("tok2vec") to the multitask's model ? + for labeller in self._multitasks: + labeller.model.set_dim("nO", len(self.labels)) + if labeller.model.has_ref("output_layer"): + labeller.model.get_ref("output_layer").set_dim("nO", len(self.labels)) + labeller.begin_training(get_examples, pipeline=pipeline) + + @property + def labels(self): + # Get the labels from the model by looking at the available moves, e.g. + # B-PERSON, I-PERSON, L-PERSON, U-PERSON + labels = set(move.split("-")[1] for move in self.move_names + if move[0] in ("B", "I", "L", "U")) + return tuple(sorted(labels)) diff --git a/spacy/pipeline/pipe.pyx b/spacy/pipeline/pipe.pyx new file mode 100644 index 000000000..5fa7d82db --- /dev/null +++ b/spacy/pipeline/pipe.pyx @@ -0,0 +1,172 @@ +# cython: infer_types=True, profile=True, binding=True +import srsly + +from ..tokens.doc cimport Doc + +from ..util import link_vectors_to_models, create_default_optimizer +from ..errors import Errors +from .. import util + + +def deserialize_config(path): + if path.exists(): + return srsly.read_json(path) + else: + return {} + + +class Pipe: + """This class is not instantiated directly. Components inherit from it, and + it defines the interface that components should follow to function as + components in a spaCy analysis pipeline. + """ + + name = None + + def __init__(self, vocab, model, **cfg): + """Create a new pipe instance.""" + raise NotImplementedError + + def __call__(self, Doc doc): + """Apply the pipe to one document. The document is + modified in-place, and returned. + + Both __call__ and pipe should delegate to the `predict()` + and `set_annotations()` methods. + """ + scores = self.predict([doc]) + self.set_annotations([doc], scores) + return doc + + def pipe(self, stream, batch_size=128): + """Apply the pipe to a stream of documents. + + Both __call__ and pipe should delegate to the `predict()` + and `set_annotations()` methods. + """ + for docs in util.minibatch(stream, size=batch_size): + scores = self.predict(docs) + self.set_annotations(docs, scores) + yield from docs + + def predict(self, docs): + """Apply the pipeline's model to a batch of docs, without + modifying them. + """ + raise NotImplementedError + + def set_annotations(self, docs, scores): + """Modify a batch of documents, using pre-computed scores.""" + raise NotImplementedError + + def rehearse(self, examples, sgd=None, losses=None, **config): + pass + + def get_loss(self, examples, scores): + """Find the loss and gradient of loss for the batch of + examples (with embedded docs) and their predicted scores.""" + raise NotImplementedError + + def add_label(self, label): + """Add an output label, to be predicted by the model. + + It's possible to extend pretrained models with new labels, + but care should be taken to avoid the "catastrophic forgetting" + problem. + """ + raise NotImplementedError + + def create_optimizer(self): + return create_default_optimizer() + + def begin_training(self, get_examples=lambda: [], pipeline=None, sgd=None): + """Initialize the pipe for training, using data exampes if available. + If no model has been initialized yet, the model is added.""" + self.model.initialize() + if hasattr(self, "vocab"): + link_vectors_to_models(self.vocab) + if sgd is None: + sgd = self.create_optimizer() + return sgd + + def set_output(self, nO): + if self.model.has_dim("nO") is not False: + self.model.set_dim("nO", nO) + if self.model.has_ref("output_layer"): + self.model.get_ref("output_layer").set_dim("nO", nO) + + def get_gradients(self): + """Get non-zero gradients of the model's parameters, as a dictionary + keyed by the parameter ID. The values are (weights, gradients) tuples. + """ + gradients = {} + queue = [self.model] + seen = set() + for node in queue: + if node.id in seen: + continue + seen.add(node.id) + if hasattr(node, "_mem") and node._mem.gradient.any(): + gradients[node.id] = [node._mem.weights, node._mem.gradient] + if hasattr(node, "_layers"): + queue.extend(node._layers) + return gradients + + def use_params(self, params): + """Modify the pipe's model, to use the given parameter values.""" + with self.model.use_params(params): + yield + + def to_bytes(self, exclude=tuple()): + """Serialize the pipe to a bytestring. + + exclude (list): String names of serialization fields to exclude. + RETURNS (bytes): The serialized object. + """ + serialize = {} + serialize["cfg"] = lambda: srsly.json_dumps(self.cfg) + serialize["model"] = self.model.to_bytes + if hasattr(self, "vocab"): + serialize["vocab"] = self.vocab.to_bytes + return util.to_bytes(serialize, exclude) + + def from_bytes(self, bytes_data, exclude=tuple()): + """Load the pipe from a bytestring.""" + + def load_model(b): + try: + self.model.from_bytes(b) + except AttributeError: + raise ValueError(Errors.E149) + + deserialize = {} + if hasattr(self, "vocab"): + deserialize["vocab"] = lambda b: self.vocab.from_bytes(b) + deserialize["cfg"] = lambda b: self.cfg.update(srsly.json_loads(b)) + deserialize["model"] = load_model + util.from_bytes(bytes_data, deserialize, exclude) + return self + + def to_disk(self, path, exclude=tuple()): + """Serialize the pipe to disk.""" + serialize = {} + serialize["cfg"] = lambda p: srsly.write_json(p, self.cfg) + serialize["vocab"] = lambda p: self.vocab.to_disk(p) + serialize["model"] = lambda p: self.model.to_disk(p) + util.to_disk(path, serialize, exclude) + + def from_disk(self, path, exclude=tuple()): + """Load the pipe from disk.""" + + def load_model(p): + try: + self.model.from_bytes(p.open("rb").read()) + except AttributeError: + raise ValueError(Errors.E149) + + deserialize = {} + deserialize["vocab"] = lambda p: self.vocab.from_disk(p) + deserialize["cfg"] = lambda p: self.cfg.update(deserialize_config(p)) + deserialize["model"] = load_model + util.from_disk(path, deserialize, exclude) + return self diff --git a/spacy/pipeline/pipes.pyx b/spacy/pipeline/pipes.pyx deleted file mode 100644 index e3ed0fcb7..000000000 --- a/spacy/pipeline/pipes.pyx +++ /dev/null @@ -1,1504 +0,0 @@ -# cython: infer_types=True, profile=True -import numpy -import srsly -import random - -from thinc.api import CosineDistance, to_categorical, get_array_module -from thinc.api import set_dropout_rate, SequenceCategoricalCrossentropy -import warnings - -from ..tokens.doc cimport Doc -from ..syntax.nn_parser cimport Parser -from ..syntax.ner cimport BiluoPushDown -from ..syntax.arc_eager cimport ArcEager -from ..morphology cimport Morphology -from ..vocab cimport Vocab - -from .defaults import default_tagger, default_parser, default_ner, default_textcat -from .defaults import default_nel, default_senter -from .functions import merge_subtokens -from ..language import Language, component -from ..syntax import nonproj -from ..gold.example import Example -from ..attrs import POS, ID -from ..util import link_vectors_to_models, create_default_optimizer -from ..parts_of_speech import X -from ..kb import KnowledgeBase -from ..errors import Errors, TempErrors, Warnings -from .. import util - - -def _load_cfg(path): - if path.exists(): - return srsly.read_json(path) - else: - return {} - - -class Pipe: - """This class is not instantiated directly. Components inherit from it, and - it defines the interface that components should follow to function as - components in a spaCy analysis pipeline. - """ - - name = None - - @classmethod - def from_nlp(cls, nlp, model, **cfg): - return cls(nlp.vocab, model, **cfg) - - def __init__(self, vocab, model, **cfg): - """Create a new pipe instance.""" - raise NotImplementedError - - def __call__(self, Doc doc): - """Apply the pipe to one document. The document is - modified in-place, and returned. - - Both __call__ and pipe should delegate to the `predict()` - and `set_annotations()` methods. - """ - scores = self.predict([doc]) - self.set_annotations([doc], scores) - return doc - - def pipe(self, stream, batch_size=128): - """Apply the pipe to a stream of documents. - - Both __call__ and pipe should delegate to the `predict()` - and `set_annotations()` methods. - """ - for docs in util.minibatch(stream, size=batch_size): - scores = self.predict(docs) - self.set_annotations(docs, scores) - yield from docs - - def predict(self, docs): - """Apply the pipeline's model to a batch of docs, without - modifying them. - """ - raise NotImplementedError - - def set_annotations(self, docs, scores): - """Modify a batch of documents, using pre-computed scores.""" - raise NotImplementedError - - def rehearse(self, examples, sgd=None, losses=None, **config): - pass - - def get_loss(self, examples, scores): - """Find the loss and gradient of loss for the batch of - examples (with embedded docs) and their predicted scores.""" - raise NotImplementedError - - def add_label(self, label): - """Add an output label, to be predicted by the model. - - It's possible to extend pretrained models with new labels, - but care should be taken to avoid the "catastrophic forgetting" - problem. - """ - raise NotImplementedError - - def create_optimizer(self): - return create_default_optimizer() - - def begin_training( - self, get_examples=lambda: [], pipeline=None, sgd=None, **kwargs - ): - """Initialize the pipe for training, using data exampes if available. - If no model has been initialized yet, the model is added.""" - self.model.initialize() - if hasattr(self, "vocab"): - link_vectors_to_models(self.vocab) - if sgd is None: - sgd = self.create_optimizer() - return sgd - - def set_output(self, nO): - if self.model.has_dim("nO") is not False: - self.model.set_dim("nO", nO) - if self.model.has_ref("output_layer"): - self.model.get_ref("output_layer").set_dim("nO", nO) - - def get_gradients(self): - """Get non-zero gradients of the model's parameters, as a dictionary - keyed by the parameter ID. The values are (weights, gradients) tuples. - """ - gradients = {} - queue = [self.model] - seen = set() - for node in queue: - if node.id in seen: - continue - seen.add(node.id) - if hasattr(node, "_mem") and node._mem.gradient.any(): - gradients[node.id] = [node._mem.weights, node._mem.gradient] - if hasattr(node, "_layers"): - queue.extend(node._layers) - return gradients - - def use_params(self, params): - """Modify the pipe's model, to use the given parameter values.""" - with self.model.use_params(params): - yield - - def to_bytes(self, exclude=tuple()): - """Serialize the pipe to a bytestring. - - exclude (list): String names of serialization fields to exclude. - RETURNS (bytes): The serialized object. - """ - serialize = {} - serialize["cfg"] = lambda: srsly.json_dumps(self.cfg) - serialize["model"] = self.model.to_bytes - if hasattr(self, "vocab"): - serialize["vocab"] = self.vocab.to_bytes - return util.to_bytes(serialize, exclude) - - def from_bytes(self, bytes_data, exclude=tuple()): - """Load the pipe from a bytestring.""" - - def load_model(b): - try: - self.model.from_bytes(b) - except AttributeError: - raise ValueError(Errors.E149) - - deserialize = {} - if hasattr(self, "vocab"): - deserialize["vocab"] = lambda b: self.vocab.from_bytes(b) - deserialize["cfg"] = lambda b: self.cfg.update(srsly.json_loads(b)) - deserialize["model"] = load_model - util.from_bytes(bytes_data, deserialize, exclude) - return self - - def to_disk(self, path, exclude=tuple()): - """Serialize the pipe to disk.""" - serialize = {} - serialize["cfg"] = lambda p: srsly.write_json(p, self.cfg) - serialize["vocab"] = lambda p: self.vocab.to_disk(p) - serialize["model"] = lambda p: self.model.to_disk(p) - util.to_disk(path, serialize, exclude) - - def from_disk(self, path, exclude=tuple()): - """Load the pipe from disk.""" - - def load_model(p): - try: - self.model.from_bytes(p.open("rb").read()) - except AttributeError: - raise ValueError(Errors.E149) - - deserialize = {} - deserialize["vocab"] = lambda p: self.vocab.from_disk(p) - deserialize["cfg"] = lambda p: self.cfg.update(_load_cfg(p)) - deserialize["model"] = load_model - util.from_disk(path, deserialize, exclude) - return self - - -@component("tagger", assigns=["token.tag", "token.pos", "token.lemma"], default_model=default_tagger) -class Tagger(Pipe): - """Pipeline component for part-of-speech tagging. - - DOCS: https://spacy.io/api/tagger - """ - - def __init__(self, vocab, model, **cfg): - self.vocab = vocab - self.model = model - self._rehearsal_model = None - self.cfg = dict(sorted(cfg.items())) - - @property - def labels(self): - return tuple(self.vocab.morphology.tag_names) - - def __call__(self, doc): - tags = self.predict([doc]) - self.set_annotations([doc], tags) - return doc - - def pipe(self, stream, batch_size=128): - for docs in util.minibatch(stream, size=batch_size): - tag_ids = self.predict(docs) - self.set_annotations(docs, tag_ids) - yield from docs - - def predict(self, docs): - if not any(len(doc) for doc in docs): - # Handle cases where there are no tokens in any docs. - n_labels = len(self.labels) - guesses = [self.model.ops.alloc((0, n_labels)) for doc in docs] - assert len(guesses) == len(docs) - return guesses - scores = self.model.predict(docs) - assert len(scores) == len(docs), (len(scores), len(docs)) - guesses = self._scores2guesses(scores) - assert len(guesses) == len(docs) - return guesses - - def _scores2guesses(self, scores): - guesses = [] - for doc_scores in scores: - doc_guesses = doc_scores.argmax(axis=1) - if not isinstance(doc_guesses, numpy.ndarray): - doc_guesses = doc_guesses.get() - guesses.append(doc_guesses) - return guesses - - def set_annotations(self, docs, batch_tag_ids): - if isinstance(docs, Doc): - docs = [docs] - cdef Doc doc - cdef int idx = 0 - cdef Vocab vocab = self.vocab - assign_morphology = self.cfg.get("set_morphology", True) - for i, doc in enumerate(docs): - doc_tag_ids = batch_tag_ids[i] - if hasattr(doc_tag_ids, "get"): - doc_tag_ids = doc_tag_ids.get() - for j, tag_id in enumerate(doc_tag_ids): - # Don't clobber preset POS tags - if doc.c[j].tag == 0: - if doc.c[j].pos == 0 and assign_morphology: - # Don't clobber preset lemmas - lemma = doc.c[j].lemma - vocab.morphology.assign_tag_id(&doc.c[j], tag_id) - if lemma != 0 and lemma != doc.c[j].lex.orth: - doc.c[j].lemma = lemma - else: - doc.c[j].tag = self.vocab.strings[self.labels[tag_id]] - idx += 1 - doc.is_tagged = True - - def update(self, examples, *, drop=0., sgd=None, losses=None, set_annotations=False): - if losses is None: - losses = {} - losses.setdefault(self.name, 0.0) - - try: - if not any(len(eg.predicted) if eg.predicted else 0 for eg in examples): - # Handle cases where there are no tokens in any docs. - return - except AttributeError: - types = set([type(eg) for eg in examples]) - raise TypeError(Errors.E978.format(name="Tagger", method="update", types=types)) - set_dropout_rate(self.model, drop) - tag_scores, bp_tag_scores = self.model.begin_update( - [eg.predicted for eg in examples]) - for sc in tag_scores: - if self.model.ops.xp.isnan(sc.sum()): - raise ValueError("nan value in scores") - loss, d_tag_scores = self.get_loss(examples, tag_scores) - bp_tag_scores(d_tag_scores) - if sgd not in (None, False): - self.model.finish_update(sgd) - - losses[self.name] += loss - if set_annotations: - docs = [eg.predicted for eg in examples] - self.set_annotations(docs, self._scores2guesses(tag_scores)) - return losses - - def rehearse(self, examples, drop=0., sgd=None, losses=None): - """Perform a 'rehearsal' update, where we try to match the output of - an initial model. - """ - try: - docs = [eg.predicted for eg in examples] - except AttributeError: - types = set([type(eg) for eg in examples]) - raise TypeError(Errors.E978.format(name="Tagger", method="rehearse", types=types)) - if self._rehearsal_model is None: - return - if not any(len(doc) for doc in docs): - # Handle cases where there are no tokens in any docs. - return - set_dropout_rate(self.model, drop) - guesses, backprop = self.model.begin_update(docs) - target = self._rehearsal_model(examples) - gradient = guesses - target - backprop(gradient) - self.model.finish_update(sgd) - if losses is not None: - losses.setdefault(self.name, 0.0) - losses[self.name] += (gradient**2).sum() - - def get_loss(self, examples, scores): - loss_func = SequenceCategoricalCrossentropy(names=self.labels, normalize=False) - truths = [eg.get_aligned("tag", as_string=True) for eg in examples] - d_scores, loss = loss_func(scores, truths) - if self.model.ops.xp.isnan(loss): - raise ValueError("nan value when computing loss") - return float(loss), d_scores - - def begin_training(self, get_examples=lambda: [], pipeline=None, sgd=None, - **kwargs): - lemma_tables = ["lemma_rules", "lemma_index", "lemma_exc", "lemma_lookup"] - if not any(table in self.vocab.lookups for table in lemma_tables): - warnings.warn(Warnings.W022) - if len(self.vocab.lookups.get_table("lexeme_norm", {})) == 0: - warnings.warn(Warnings.W033.format(model="part-of-speech tagger")) - orig_tag_map = dict(self.vocab.morphology.tag_map) - new_tag_map = {} - for example in get_examples(): - try: - y = example.y - except AttributeError: - raise TypeError(Errors.E978.format(name="Tagger", method="begin_training", types=type(example))) - for token in y: - tag = token.tag_ - if tag in orig_tag_map: - new_tag_map[tag] = orig_tag_map[tag] - else: - new_tag_map[tag] = {POS: X} - - cdef Vocab vocab = self.vocab - if new_tag_map: - if "_SP" in orig_tag_map: - new_tag_map["_SP"] = orig_tag_map["_SP"] - vocab.morphology.load_tag_map(new_tag_map) - self.set_output(len(self.labels)) - doc_sample = [Doc(self.vocab, words=["hello", "world"])] - if pipeline is not None: - for name, component in pipeline: - if component is self: - break - if hasattr(component, "pipe"): - doc_sample = list(component.pipe(doc_sample)) - else: - doc_sample = [component(doc) for doc in doc_sample] - self.model.initialize(X=doc_sample) - # Get batch of example docs, example outputs to call begin_training(). - # This lets the model infer shapes. - link_vectors_to_models(self.vocab) - if sgd is None: - sgd = self.create_optimizer() - return sgd - - def add_label(self, label, values=None): - if not isinstance(label, str): - raise ValueError(Errors.E187) - if label in self.labels: - return 0 - if self.model.has_dim("nO"): - # Here's how the model resizing will work, once the - # neuron-to-tag mapping is no longer controlled by - # the Morphology class, which sorts the tag names. - # The sorting makes adding labels difficult. - # smaller = self.model._layers[-1] - # larger = Softmax(len(self.labels)+1, smaller.nI) - # copy_array(larger.W[:smaller.nO], smaller.W) - # copy_array(larger.b[:smaller.nO], smaller.b) - # self.model._layers[-1] = larger - raise ValueError(TempErrors.T003) - tag_map = dict(self.vocab.morphology.tag_map) - if values is None: - values = {POS: "X"} - tag_map[label] = values - self.vocab.morphology.load_tag_map(tag_map) - return 1 - - def use_params(self, params): - with self.model.use_params(params): - yield - - def to_bytes(self, exclude=tuple()): - serialize = {} - serialize["model"] = self.model.to_bytes - serialize["vocab"] = self.vocab.to_bytes - serialize["cfg"] = lambda: srsly.json_dumps(self.cfg) - tag_map = dict(sorted(self.vocab.morphology.tag_map.items())) - serialize["tag_map"] = lambda: srsly.msgpack_dumps(tag_map) - morph_rules = dict(self.vocab.morphology.exc) - serialize["morph_rules"] = lambda: srsly.msgpack_dumps(morph_rules) - return util.to_bytes(serialize, exclude) - - def from_bytes(self, bytes_data, exclude=tuple()): - def load_model(b): - try: - self.model.from_bytes(b) - except AttributeError: - raise ValueError(Errors.E149) - - def load_tag_map(b): - tag_map = srsly.msgpack_loads(b) - self.vocab.morphology.load_tag_map(tag_map) - - def load_morph_rules(b): - morph_rules = srsly.msgpack_loads(b) - self.vocab.morphology.load_morph_exceptions(morph_rules) - - self.vocab.morphology = Morphology(self.vocab.strings, dict(), - lemmatizer=self.vocab.morphology.lemmatizer) - deserialize = { - "vocab": lambda b: self.vocab.from_bytes(b), - "tag_map": load_tag_map, - "morph_rules": load_morph_rules, - "cfg": lambda b: self.cfg.update(srsly.json_loads(b)), - "model": lambda b: load_model(b), - } - util.from_bytes(bytes_data, deserialize, exclude) - return self - - def to_disk(self, path, exclude=tuple()): - tag_map = dict(sorted(self.vocab.morphology.tag_map.items())) - morph_rules = dict(self.vocab.morphology.exc) - serialize = { - "vocab": lambda p: self.vocab.to_disk(p), - "tag_map": lambda p: srsly.write_msgpack(p, tag_map), - "morph_rules": lambda p: srsly.write_msgpack(p, morph_rules), - "model": lambda p: self.model.to_disk(p), - "cfg": lambda p: srsly.write_json(p, self.cfg), - } - util.to_disk(path, serialize, exclude) - - def from_disk(self, path, exclude=tuple()): - def load_model(p): - with p.open("rb") as file_: - try: - self.model.from_bytes(file_.read()) - except AttributeError: - raise ValueError(Errors.E149) - - def load_tag_map(p): - tag_map = srsly.read_msgpack(p) - self.vocab.morphology.load_tag_map(tag_map) - - def load_morph_rules(p): - morph_rules = srsly.read_msgpack(p) - self.vocab.morphology.load_morph_exceptions(morph_rules) - - self.vocab.morphology = Morphology(self.vocab.strings, dict(), - lemmatizer=self.vocab.morphology.lemmatizer) - deserialize = { - "vocab": lambda p: self.vocab.from_disk(p), - "cfg": lambda p: self.cfg.update(_load_cfg(p)), - "tag_map": load_tag_map, - "morph_rules": load_morph_rules, - "model": load_model, - } - util.from_disk(path, deserialize, exclude) - return self - - -@component("senter", assigns=["token.is_sent_start"], default_model=default_senter) -class SentenceRecognizer(Tagger): - """Pipeline component for sentence segmentation. - - DOCS: https://spacy.io/api/sentencerecognizer - """ - - def __init__(self, vocab, model, **cfg): - self.vocab = vocab - self.model = model - self._rehearsal_model = None - self.cfg = dict(sorted(cfg.items())) - - @property - def labels(self): - # labels are numbered by index internally, so this matches GoldParse - # and Example where the sentence-initial tag is 1 and other positions - # are 0 - return tuple(["I", "S"]) - - def set_annotations(self, docs, batch_tag_ids): - if isinstance(docs, Doc): - docs = [docs] - cdef Doc doc - for i, doc in enumerate(docs): - doc_tag_ids = batch_tag_ids[i] - if hasattr(doc_tag_ids, "get"): - doc_tag_ids = doc_tag_ids.get() - for j, tag_id in enumerate(doc_tag_ids): - # Don't clobber existing sentence boundaries - if doc.c[j].sent_start == 0: - if tag_id == 1: - doc.c[j].sent_start = 1 - else: - doc.c[j].sent_start = -1 - - def get_loss(self, examples, scores): - labels = self.labels - loss_func = SequenceCategoricalCrossentropy(names=labels, normalize=False) - truths = [] - for eg in examples: - eg_truth = [] - for x in eg.get_aligned("sent_start"): - if x == None: - eg_truth.append(None) - elif x == 1: - eg_truth.append(labels[1]) - else: - # anything other than 1: 0, -1, -1 as uint64 - eg_truth.append(labels[0]) - truths.append(eg_truth) - d_scores, loss = loss_func(scores, truths) - if self.model.ops.xp.isnan(loss): - raise ValueError("nan value when computing loss") - return float(loss), d_scores - - def begin_training(self, get_examples=lambda: [], pipeline=None, sgd=None, - **kwargs): - self.set_output(len(self.labels)) - self.model.initialize() - link_vectors_to_models(self.vocab) - if sgd is None: - sgd = self.create_optimizer() - return sgd - - def add_label(self, label, values=None): - raise NotImplementedError - - def to_bytes(self, exclude=tuple()): - serialize = {} - serialize["model"] = self.model.to_bytes - serialize["vocab"] = self.vocab.to_bytes - serialize["cfg"] = lambda: srsly.json_dumps(self.cfg) - return util.to_bytes(serialize, exclude) - - def from_bytes(self, bytes_data, exclude=tuple()): - def load_model(b): - try: - self.model.from_bytes(b) - except AttributeError: - raise ValueError(Errors.E149) - - deserialize = { - "vocab": lambda b: self.vocab.from_bytes(b), - "cfg": lambda b: self.cfg.update(srsly.json_loads(b)), - "model": lambda b: load_model(b), - } - util.from_bytes(bytes_data, deserialize, exclude) - return self - - def to_disk(self, path, exclude=tuple()): - serialize = { - "vocab": lambda p: self.vocab.to_disk(p), - "model": lambda p: p.open("wb").write(self.model.to_bytes()), - "cfg": lambda p: srsly.write_json(p, self.cfg), - } - util.to_disk(path, serialize, exclude) - - def from_disk(self, path, exclude=tuple()): - def load_model(p): - with p.open("rb") as file_: - try: - self.model.from_bytes(file_.read()) - except AttributeError: - raise ValueError(Errors.E149) - - deserialize = { - "vocab": lambda p: self.vocab.from_disk(p), - "cfg": lambda p: self.cfg.update(_load_cfg(p)), - "model": load_model, - } - util.from_disk(path, deserialize, exclude) - return self - - -@component("nn_labeller") -class MultitaskObjective(Tagger): - """Experimental: Assist training of a parser or tagger, by training a - side-objective. - """ - - def __init__(self, vocab, model, **cfg): - self.vocab = vocab - self.model = model - target = cfg["target"] # default: 'dep_tag_offset' - if target == "dep": - self.make_label = self.make_dep - elif target == "tag": - self.make_label = self.make_tag - elif target == "ent": - self.make_label = self.make_ent - elif target == "dep_tag_offset": - self.make_label = self.make_dep_tag_offset - elif target == "ent_tag": - self.make_label = self.make_ent_tag - elif target == "sent_start": - self.make_label = self.make_sent_start - elif hasattr(target, "__call__"): - self.make_label = target - else: - raise ValueError(Errors.E016) - self.cfg = dict(cfg) - - @property - def labels(self): - return self.cfg.setdefault("labels", {}) - - @labels.setter - def labels(self, value): - self.cfg["labels"] = value - - def set_annotations(self, docs, dep_ids): - pass - - def begin_training(self, get_examples=lambda: [], pipeline=None, - sgd=None, **kwargs): - gold_examples = nonproj.preprocess_training_data(get_examples()) - # for raw_text, doc_annot in gold_tuples: - for example in gold_examples: - for token in example.y: - label = self.make_label(token) - if label is not None and label not in self.labels: - self.labels[label] = len(self.labels) - self.model.initialize() - link_vectors_to_models(self.vocab) - if sgd is None: - sgd = self.create_optimizer() - return sgd - - def predict(self, docs): - tokvecs = self.model.get_ref("tok2vec")(docs) - scores = self.model.get_ref("softmax")(tokvecs) - return tokvecs, scores - - def get_loss(self, examples, scores): - cdef int idx = 0 - correct = numpy.zeros((scores.shape[0],), dtype="i") - guesses = scores.argmax(axis=1) - docs = [eg.predicted for eg in examples] - for i, eg in enumerate(examples): - # Handles alignment for tokenization differences - doc_annots = eg.get_aligned() # TODO - for j in range(len(eg.predicted)): - tok_annots = {key: values[j] for key, values in tok_annots.items()} - label = self.make_label(j, tok_annots) - if label is None or label not in self.labels: - correct[idx] = guesses[idx] - else: - correct[idx] = self.labels[label] - idx += 1 - correct = self.model.ops.xp.array(correct, dtype="i") - d_scores = scores - to_categorical(correct, n_classes=scores.shape[1]) - loss = (d_scores**2).sum() - return float(loss), d_scores - - @staticmethod - def make_dep(token): - return token.dep_ - - @staticmethod - def make_tag(token): - return token.tag_ - - @staticmethod - def make_ent(token): - if token.ent_iob_ == "O": - return "O" - else: - return token.ent_iob_ + "-" + token.ent_type_ - - @staticmethod - def make_dep_tag_offset(token): - dep = token.dep_ - tag = token.tag_ - offset = token.head.i - token.i - offset = min(offset, 2) - offset = max(offset, -2) - return f"{dep}-{tag}:{offset}" - - @staticmethod - def make_ent_tag(token): - if token.ent_iob_ == "O": - ent = "O" - else: - ent = token.ent_iob_ + "-" + token.ent_type_ - tag = token.tag_ - return f"{tag}-{ent}" - - @staticmethod - def make_sent_start(token): - """A multi-task objective for representing sentence boundaries, - using BILU scheme. (O is impossible) - """ - if token.is_sent_start and token.is_sent_end: - return "U-SENT" - elif token.is_sent_start: - return "B-SENT" - else: - return "I-SENT" - - -class ClozeMultitask(Pipe): - def __init__(self, vocab, model, **cfg): - self.vocab = vocab - self.model = model - self.cfg = cfg - self.distance = CosineDistance(ignore_zeros=True, normalize=False) # TODO: in config - - def set_annotations(self, docs, dep_ids): - pass - - def begin_training(self, get_examples=lambda: [], pipeline=None, - sgd=None, **kwargs): - link_vectors_to_models(self.vocab) - self.model.initialize() - X = self.model.ops.alloc((5, self.model.get_ref("tok2vec").get_dim("nO"))) - self.model.output_layer.begin_training(X) - if sgd is None: - sgd = self.create_optimizer() - return sgd - - def predict(self, docs): - tokvecs = self.model.get_ref("tok2vec")(docs) - vectors = self.model.get_ref("output_layer")(tokvecs) - return tokvecs, vectors - - def get_loss(self, examples, vectors, prediction): - # The simplest way to implement this would be to vstack the - # token.vector values, but that's a bit inefficient, especially on GPU. - # Instead we fetch the index into the vectors table for each of our tokens, - # and look them up all at once. This prevents data copying. - ids = self.model.ops.flatten([eg.predicted.to_array(ID).ravel() for eg in examples]) - target = vectors[ids] - gradient = self.distance.get_grad(prediction, target) - loss = self.distance.get_loss(prediction, target) - return loss, gradient - - def update(self, examples, *, drop=0., set_annotations=False, sgd=None, losses=None): - pass - - def rehearse(self, examples, drop=0., sgd=None, losses=None): - if losses is not None and self.name not in losses: - losses[self.name] = 0. - set_dropout_rate(self.model, drop) - try: - predictions, bp_predictions = self.model.begin_update([eg.predicted for eg in examples]) - except AttributeError: - types = set([type(eg) for eg in examples]) - raise TypeError(Errors.E978.format(name="ClozeMultitask", method="rehearse", types=types)) - loss, d_predictions = self.get_loss(examples, self.vocab.vectors.data, predictions) - bp_predictions(d_predictions) - if sgd is not None: - self.model.finish_update(sgd) - - if losses is not None: - losses[self.name] += loss - - @staticmethod - def decode_utf8_predictions(char_array): - # The format alternates filling from start and end, and 255 is missing - words = [] - char_array = char_array.reshape((char_array.shape[0], -1, 256)) - nr_char = char_array.shape[1] - char_array = char_array.argmax(axis=-1) - for row in char_array: - starts = [chr(c) for c in row[::2] if c != 255] - ends = [chr(c) for c in row[1::2] if c != 255] - word = "".join(starts + list(reversed(ends))) - words.append(word) - return words - - -@component("textcat", assigns=["doc.cats"], default_model=default_textcat) -class TextCategorizer(Pipe): - """Pipeline component for text classification. - - DOCS: https://spacy.io/api/textcategorizer - """ - def __init__(self, vocab, model, **cfg): - self.vocab = vocab - self.model = model - self._rehearsal_model = None - self.cfg = dict(cfg) - - @property - def labels(self): - return tuple(self.cfg.setdefault("labels", [])) - - def require_labels(self): - """Raise an error if the component's model has no labels defined.""" - if not self.labels: - raise ValueError(Errors.E143.format(name=self.name)) - - @labels.setter - def labels(self, value): - self.cfg["labels"] = tuple(value) - - def pipe(self, stream, batch_size=128): - for docs in util.minibatch(stream, size=batch_size): - scores = self.predict(docs) - self.set_annotations(docs, scores) - yield from docs - - def predict(self, docs): - tensors = [doc.tensor for doc in docs] - - if not any(len(doc) for doc in docs): - # Handle cases where there are no tokens in any docs. - xp = get_array_module(tensors) - scores = xp.zeros((len(docs), len(self.labels))) - return scores - - scores = self.model.predict(docs) - scores = self.model.ops.asarray(scores) - return scores - - def set_annotations(self, docs, scores): - for i, doc in enumerate(docs): - for j, label in enumerate(self.labels): - doc.cats[label] = float(scores[i, j]) - - def update(self, examples, *, drop=0., set_annotations=False, sgd=None, losses=None): - if losses is None: - losses = {} - losses.setdefault(self.name, 0.0) - try: - if not any(len(eg.predicted) if eg.predicted else 0 for eg in examples): - # Handle cases where there are no tokens in any docs. - return losses - except AttributeError: - types = set([type(eg) for eg in examples]) - raise TypeError(Errors.E978.format(name="TextCategorizer", method="update", types=types)) - set_dropout_rate(self.model, drop) - scores, bp_scores = self.model.begin_update( - [eg.predicted for eg in examples] - ) - loss, d_scores = self.get_loss(examples, scores) - bp_scores(d_scores) - if sgd is not None: - self.model.finish_update(sgd) - losses[self.name] += loss - if set_annotations: - docs = [eg.predicted for eg in examples] - self.set_annotations(docs, scores=scores) - return losses - - def rehearse(self, examples, drop=0., sgd=None, losses=None): - if self._rehearsal_model is None: - return - try: - docs = [eg.predicted for eg in examples] - except AttributeError: - types = set([type(eg) for eg in examples]) - raise TypeError(Errors.E978.format(name="TextCategorizer", method="rehearse", types=types)) - if not any(len(doc) for doc in docs): - # Handle cases where there are no tokens in any docs. - return - set_dropout_rate(self.model, drop) - scores, bp_scores = self.model.begin_update(docs) - target = self._rehearsal_model(examples) - gradient = scores - target - bp_scores(gradient) - if sgd is not None: - self.model.finish_update(sgd) - if losses is not None: - losses.setdefault(self.name, 0.0) - losses[self.name] += (gradient**2).sum() - - def _examples_to_truth(self, examples): - truths = numpy.zeros((len(examples), len(self.labels)), dtype="f") - not_missing = numpy.ones((len(examples), len(self.labels)), dtype="f") - for i, eg in enumerate(examples): - for j, label in enumerate(self.labels): - if label in eg.reference.cats: - truths[i, j] = eg.reference.cats[label] - else: - not_missing[i, j] = 0. - truths = self.model.ops.asarray(truths) - return truths, not_missing - - def get_loss(self, examples, scores): - truths, not_missing = self._examples_to_truth(examples) - not_missing = self.model.ops.asarray(not_missing) - d_scores = (scores-truths) / scores.shape[0] - d_scores *= not_missing - mean_square_error = (d_scores**2).sum(axis=1).mean() - return float(mean_square_error), d_scores - - def add_label(self, label): - if not isinstance(label, str): - raise ValueError(Errors.E187) - if label in self.labels: - return 0 - if self.model.has_dim("nO"): - # This functionality was available previously, but was broken. - # The problem is that we resize the last layer, but the last layer - # is actually just an ensemble. We're not resizing the child layers - # - a huge problem. - raise ValueError(Errors.E116) - # smaller = self.model._layers[-1] - # larger = Linear(len(self.labels)+1, smaller.nI) - # copy_array(larger.W[:smaller.nO], smaller.W) - # copy_array(larger.b[:smaller.nO], smaller.b) - # self.model._layers[-1] = larger - self.labels = tuple(list(self.labels) + [label]) - return 1 - - def begin_training(self, get_examples=lambda: [], pipeline=None, sgd=None, **kwargs): - # TODO: begin_training is not guaranteed to see all data / labels ? - examples = list(get_examples()) - for example in examples: - try: - y = example.y - except AttributeError: - raise TypeError(Errors.E978.format(name="TextCategorizer", method="update", types=type(example))) - for cat in y.cats: - self.add_label(cat) - self.require_labels() - docs = [Doc(Vocab(), words=["hello"])] - truths, _ = self._examples_to_truth(examples) - self.set_output(len(self.labels)) - link_vectors_to_models(self.vocab) - self.model.initialize(X=docs, Y=truths) - if sgd is None: - sgd = self.create_optimizer() - return sgd - - -cdef class DependencyParser(Parser): - """Pipeline component for dependency parsing. - - DOCS: https://spacy.io/api/dependencyparser - """ - # cdef classes can't have decorators, so we're defining this here - name = "parser" - factory = "parser" - assigns = ["token.dep", "token.is_sent_start", "doc.sents"] - requires = [] - TransitionSystem = ArcEager - nr_feature = 8 - - @property - def postprocesses(self): - output = [nonproj.deprojectivize] - if self.cfg.get("learn_tokens") is True: - output.append(merge_subtokens) - return tuple(output) - - def add_multitask_objective(self, mt_component): - self._multitasks.append(mt_component) - - def init_multitask_objectives(self, get_examples, pipeline, sgd=None, **cfg): - # TODO: transfer self.model.get_ref("tok2vec") to the multitask's model ? - for labeller in self._multitasks: - labeller.model.set_dim("nO", len(self.labels)) - if labeller.model.has_ref("output_layer"): - labeller.model.get_ref("output_layer").set_dim("nO", len(self.labels)) - labeller.begin_training(get_examples, pipeline=pipeline, sgd=sgd) - - def __reduce__(self): - return (DependencyParser, (self.vocab, self.model), (self.moves, self.cfg)) - - def __getstate__(self): - return (self.moves, self.cfg) - - def __setstate__(self, state): - moves, config = state - self.moves = moves - self.cfg = config - - @property - def labels(self): - labels = set() - # Get the labels from the model by looking at the available moves - for move in self.move_names: - if "-" in move: - label = move.split("-")[1] - if "||" in label: - label = label.split("||")[1] - labels.add(label) - return tuple(sorted(labels)) - - -cdef class EntityRecognizer(Parser): - """Pipeline component for named entity recognition. - - DOCS: https://spacy.io/api/entityrecognizer - """ - name = "ner" - factory = "ner" - assigns = ["doc.ents", "token.ent_iob", "token.ent_type"] - requires = [] - TransitionSystem = BiluoPushDown - - def add_multitask_objective(self, mt_component): - self._multitasks.append(mt_component) - - def init_multitask_objectives(self, get_examples, pipeline, sgd=None, **cfg): - # TODO: transfer self.model.get_ref("tok2vec") to the multitask's model ? - for labeller in self._multitasks: - labeller.model.set_dim("nO", len(self.labels)) - if labeller.model.has_ref("output_layer"): - labeller.model.get_ref("output_layer").set_dim("nO", len(self.labels)) - labeller.begin_training(get_examples, pipeline=pipeline) - - def __reduce__(self): - return (EntityRecognizer, (self.vocab, self.model), (self.moves, self.cfg)) - - def __getstate__(self): - return self.moves, self.cfg - - def __setstate__(self, state): - moves, config = state - self.moves = moves - self.cfg = config - - @property - def labels(self): - # Get the labels from the model by looking at the available moves, e.g. - # B-PERSON, I-PERSON, L-PERSON, U-PERSON - labels = set(move.split("-")[1] for move in self.move_names - if move[0] in ("B", "I", "L", "U")) - return tuple(sorted(labels)) - - -@component( - "entity_linker", - requires=["doc.ents", "doc.sents", "token.ent_iob", "token.ent_type"], - assigns=["token.ent_kb_id"], - default_model=default_nel, -) -class EntityLinker(Pipe): - """Pipeline component for named entity linking. - - DOCS: https://spacy.io/api/entitylinker - """ - NIL = "NIL" # string used to refer to a non-existing link - - def __init__(self, vocab, model, **cfg): - self.vocab = vocab - self.model = model - self.kb = None - self.kb = cfg.get("kb", None) - if self.kb is None: - # create an empty KB that should be filled by calling from_disk - self.kb = KnowledgeBase(vocab=vocab) - else: - del cfg["kb"] # we don't want to duplicate its serialization - if not isinstance(self.kb, KnowledgeBase): - raise ValueError(Errors.E990.format(type=type(self.kb))) - self.cfg = dict(cfg) - self.distance = CosineDistance(normalize=False) - # how many neightbour sentences to take into account - self.n_sents = cfg.get("n_sents", 0) - - def require_kb(self): - # Raise an error if the knowledge base is not initialized. - if len(self.kb) == 0: - raise ValueError(Errors.E139.format(name=self.name)) - - def begin_training(self, get_examples=lambda: [], pipeline=None, sgd=None, **kwargs): - self.require_kb() - nO = self.kb.entity_vector_length - self.set_output(nO) - self.model.initialize() - if sgd is None: - sgd = self.create_optimizer() - return sgd - - def update(self, examples, *, set_annotations=False, drop=0.0, sgd=None, losses=None): - self.require_kb() - if losses is None: - losses = {} - losses.setdefault(self.name, 0.0) - if not examples: - return losses - sentence_docs = [] - try: - docs = [eg.predicted for eg in examples] - except AttributeError: - types = set([type(eg) for eg in examples]) - raise TypeError(Errors.E978.format(name="EntityLinker", method="update", types=types)) - if set_annotations: - # This seems simpler than other ways to get that exact output -- but - # it does run the model twice :( - predictions = self.model.predict(docs) - - for eg in examples: - sentences = [s for s in eg.predicted.sents] - kb_ids = eg.get_aligned("ENT_KB_ID", as_string=True) - for ent in eg.predicted.ents: - kb_id = kb_ids[ent.start] # KB ID of the first token is the same as the whole span - if kb_id: - try: - # find the sentence in the list of sentences. - sent_index = sentences.index(ent.sent) - except AttributeError: - # Catch the exception when ent.sent is None and provide a user-friendly warning - raise RuntimeError(Errors.E030) - # get n previous sentences, if there are any - start_sentence = max(0, sent_index - self.n_sents) - - # get n posterior sentences, or as many < n as there are - end_sentence = min(len(sentences) -1, sent_index + self.n_sents) - - # get token positions - start_token = sentences[start_sentence].start - end_token = sentences[end_sentence].end - - # append that span as a doc to training - sent_doc = eg.predicted[start_token:end_token].as_doc() - sentence_docs.append(sent_doc) - set_dropout_rate(self.model, drop) - if not sentence_docs: - warnings.warn(Warnings.W093.format(name="Entity Linker")) - return 0.0 - sentence_encodings, bp_context = self.model.begin_update(sentence_docs) - loss, d_scores = self.get_similarity_loss( - sentence_encodings=sentence_encodings, - examples=examples - ) - bp_context(d_scores) - if sgd is not None: - self.model.finish_update(sgd) - - losses[self.name] += loss - if set_annotations: - self.set_annotations(docs, predictions) - return losses - - def get_similarity_loss(self, examples, sentence_encodings): - entity_encodings = [] - for eg in examples: - kb_ids = eg.get_aligned("ENT_KB_ID", as_string=True) - for ent in eg.predicted.ents: - kb_id = kb_ids[ent.start] - if kb_id: - entity_encoding = self.kb.get_vector(kb_id) - entity_encodings.append(entity_encoding) - - entity_encodings = self.model.ops.asarray(entity_encodings, dtype="float32") - - if sentence_encodings.shape != entity_encodings.shape: - raise RuntimeError(Errors.E147.format(method="get_similarity_loss", msg="gold entities do not match up")) - - gradients = self.distance.get_grad(sentence_encodings, entity_encodings) - loss = self.distance.get_loss(sentence_encodings, entity_encodings) - loss = loss / len(entity_encodings) - return loss, gradients - - def __call__(self, doc): - kb_ids = self.predict([doc]) - self.set_annotations([doc], kb_ids) - return doc - - def pipe(self, stream, batch_size=128): - for docs in util.minibatch(stream, size=batch_size): - kb_ids = self.predict(docs) - self.set_annotations(docs, kb_ids) - yield from docs - - def predict(self, docs): - """ Return the KB IDs for each entity in each doc, including NIL if there is no prediction """ - self.require_kb() - entity_count = 0 - final_kb_ids = [] - - if not docs: - return final_kb_ids - - if isinstance(docs, Doc): - docs = [docs] - - for i, doc in enumerate(docs): - sentences = [s for s in doc.sents] - - if len(doc) > 0: - # Looping through each sentence and each entity - # This may go wrong if there are entities across sentences - which shouldn't happen normally. - for sent_index, sent in enumerate(sentences): - if sent.ents: - # get n_neightbour sentences, clipped to the length of the document - start_sentence = max(0, sent_index - self.n_sents) - end_sentence = min(len(sentences) -1, sent_index + self.n_sents) - - start_token = sentences[start_sentence].start - end_token = sentences[end_sentence].end - - sent_doc = doc[start_token:end_token].as_doc() - # currently, the context is the same for each entity in a sentence (should be refined) - sentence_encoding = self.model.predict([sent_doc])[0] - xp = get_array_module(sentence_encoding) - sentence_encoding_t = sentence_encoding.T - sentence_norm = xp.linalg.norm(sentence_encoding_t) - - for ent in sent.ents: - entity_count += 1 - - to_discard = self.cfg.get("labels_discard", []) - if to_discard and ent.label_ in to_discard: - # ignoring this entity - setting to NIL - final_kb_ids.append(self.NIL) - - else: - candidates = self.kb.get_candidates(ent.text) - if not candidates: - # no prediction possible for this entity - setting to NIL - final_kb_ids.append(self.NIL) - - elif len(candidates) == 1: - # shortcut for efficiency reasons: take the 1 candidate - - # TODO: thresholding - final_kb_ids.append(candidates[0].entity_) - - else: - random.shuffle(candidates) - - # this will set all prior probabilities to 0 if they should be excluded from the model - prior_probs = xp.asarray([c.prior_prob for c in candidates]) - if not self.cfg.get("incl_prior", True): - prior_probs = xp.asarray([0.0 for c in candidates]) - scores = prior_probs - - # add in similarity from the context - if self.cfg.get("incl_context", True): - entity_encodings = xp.asarray([c.entity_vector for c in candidates]) - entity_norm = xp.linalg.norm(entity_encodings, axis=1) - - if len(entity_encodings) != len(prior_probs): - raise RuntimeError(Errors.E147.format(method="predict", msg="vectors not of equal length")) - - # cosine similarity - sims = xp.dot(entity_encodings, sentence_encoding_t) / (sentence_norm * entity_norm) - if sims.shape != prior_probs.shape: - raise ValueError(Errors.E161) - scores = prior_probs + sims - (prior_probs*sims) - - # TODO: thresholding - best_index = scores.argmax().item() - best_candidate = candidates[best_index] - final_kb_ids.append(best_candidate.entity_) - - if not (len(final_kb_ids) == entity_count): - raise RuntimeError(Errors.E147.format(method="predict", msg="result variables not of equal length")) - - return final_kb_ids - - def set_annotations(self, docs, kb_ids): - count_ents = len([ent for doc in docs for ent in doc.ents]) - if count_ents != len(kb_ids): - raise ValueError(Errors.E148.format(ents=count_ents, ids=len(kb_ids))) - - i=0 - for doc in docs: - for ent in doc.ents: - kb_id = kb_ids[i] - i += 1 - for token in ent: - token.ent_kb_id_ = kb_id - - def to_disk(self, path, exclude=tuple()): - serialize = {} - self.cfg["entity_width"] = self.kb.entity_vector_length - serialize["cfg"] = lambda p: srsly.write_json(p, self.cfg) - serialize["vocab"] = lambda p: self.vocab.to_disk(p) - serialize["kb"] = lambda p: self.kb.dump(p) - serialize["model"] = lambda p: self.model.to_disk(p) - util.to_disk(path, serialize, exclude) - - def from_disk(self, path, exclude=tuple()): - def load_model(p): - try: - self.model.from_bytes(p.open("rb").read()) - except AttributeError: - raise ValueError(Errors.E149) - - def load_kb(p): - self.kb = KnowledgeBase(vocab=self.vocab, entity_vector_length=self.cfg["entity_width"]) - self.kb.load_bulk(p) - - deserialize = {} - deserialize["vocab"] = lambda p: self.vocab.from_disk(p) - deserialize["cfg"] = lambda p: self.cfg.update(_load_cfg(p)) - deserialize["kb"] = load_kb - deserialize["model"] = load_model - util.from_disk(path, deserialize, exclude) - return self - - def rehearse(self, examples, sgd=None, losses=None, **config): - raise NotImplementedError - - def add_label(self, label): - raise NotImplementedError - - -@component("sentencizer", assigns=["token.is_sent_start", "doc.sents"]) -class Sentencizer(Pipe): - """Segment the Doc into sentences using a rule-based strategy. - - DOCS: https://spacy.io/api/sentencizer - """ - - default_punct_chars = ['!', '.', '?', '։', '؟', '۔', '܀', '܁', '܂', '߹', - '।', '॥', '၊', '။', '።', '፧', '፨', '᙮', '᜵', '᜶', '᠃', '᠉', '᥄', - '᥅', '᪨', '᪩', '᪪', '᪫', '᭚', '᭛', '᭞', '᭟', '᰻', '᰼', '᱾', '᱿', - '‼', '‽', '⁇', '⁈', '⁉', '⸮', '⸼', '꓿', '꘎', '꘏', '꛳', '꛷', '꡶', - '꡷', '꣎', '꣏', '꤯', '꧈', '꧉', '꩝', '꩞', '꩟', '꫰', '꫱', '꯫', '﹒', - '﹖', '﹗', '！', '．', '？', '𐩖', '𐩗', '𑁇', '𑁈', '𑂾', '𑂿', '𑃀', - '𑃁', '𑅁', '𑅂', '𑅃', '𑇅', '𑇆', '𑇍', '𑇞', '𑇟', '𑈸', '𑈹', '𑈻', '𑈼', - '𑊩', '𑑋', '𑑌', '𑗂', '𑗃', '𑗉', '𑗊', '𑗋', '𑗌', '𑗍', '𑗎', '𑗏', '𑗐', - '𑗑', '𑗒', '𑗓', '𑗔', '𑗕', '𑗖', '𑗗', '𑙁', '𑙂', '𑜼', '𑜽', '𑜾', '𑩂', - '𑩃', '𑪛', '𑪜', '𑱁', '𑱂', '𖩮', '𖩯', '𖫵', '𖬷', '𖬸', '𖭄', '𛲟', '𝪈', - '｡', '。'] - - def __init__(self, punct_chars=None, **kwargs): - """Initialize the sentencizer. - - punct_chars (list): Punctuation characters to split on. Will be - serialized with the nlp object. - RETURNS (Sentencizer): The sentencizer component. - - DOCS: https://spacy.io/api/sentencizer#init - """ - if punct_chars: - self.punct_chars = set(punct_chars) - else: - self.punct_chars = set(self.default_punct_chars) - - @classmethod - def from_nlp(cls, nlp, model=None, **cfg): - return cls(**cfg) - - def begin_training( - self, get_examples=lambda: [], pipeline=None, sgd=None, **kwargs - ): - pass - - def __call__(self, doc): - """Apply the sentencizer to a Doc and set Token.is_sent_start. - - example (Doc or Example): The document to process. - RETURNS (Doc or Example): The processed Doc or Example. - - DOCS: https://spacy.io/api/sentencizer#call - """ - start = 0 - seen_period = False - for i, token in enumerate(doc): - is_in_punct_chars = token.text in self.punct_chars - token.is_sent_start = i == 0 - if seen_period and not token.is_punct and not is_in_punct_chars: - doc[start].is_sent_start = True - start = token.i - seen_period = False - elif is_in_punct_chars: - seen_period = True - if start < len(doc): - doc[start].is_sent_start = True - return doc - - def pipe(self, stream, batch_size=128): - for docs in util.minibatch(stream, size=batch_size): - predictions = self.predict(docs) - self.set_annotations(docs, predictions) - yield from docs - - def predict(self, docs): - """Apply the pipeline's model to a batch of docs, without - modifying them. - """ - if not any(len(doc) for doc in docs): - # Handle cases where there are no tokens in any docs. - guesses = [[] for doc in docs] - return guesses - guesses = [] - for doc in docs: - doc_guesses = [False] * len(doc) - if len(doc) > 0: - start = 0 - seen_period = False - doc_guesses[0] = True - for i, token in enumerate(doc): - is_in_punct_chars = token.text in self.punct_chars - if seen_period and not token.is_punct and not is_in_punct_chars: - doc_guesses[start] = True - start = token.i - seen_period = False - elif is_in_punct_chars: - seen_period = True - if start < len(doc): - doc_guesses[start] = True - guesses.append(doc_guesses) - return guesses - - def set_annotations(self, docs, batch_tag_ids): - if isinstance(docs, Doc): - docs = [docs] - cdef Doc doc - cdef int idx = 0 - for i, doc in enumerate(docs): - doc_tag_ids = batch_tag_ids[i] - for j, tag_id in enumerate(doc_tag_ids): - # Don't clobber existing sentence boundaries - if doc.c[j].sent_start == 0: - if tag_id: - doc.c[j].sent_start = 1 - else: - doc.c[j].sent_start = -1 - - def to_bytes(self, **kwargs): - """Serialize the sentencizer to a bytestring. - - RETURNS (bytes): The serialized object. - - DOCS: https://spacy.io/api/sentencizer#to_bytes - """ - return srsly.msgpack_dumps({"punct_chars": list(self.punct_chars)}) - - def from_bytes(self, bytes_data, **kwargs): - """Load the sentencizer from a bytestring. - - bytes_data (bytes): The data to load. - returns (Sentencizer): The loaded object. - - DOCS: https://spacy.io/api/sentencizer#from_bytes - """ - cfg = srsly.msgpack_loads(bytes_data) - self.punct_chars = set(cfg.get("punct_chars", self.default_punct_chars)) - return self - - def to_disk(self, path, exclude=tuple(), **kwargs): - """Serialize the sentencizer to disk. - - DOCS: https://spacy.io/api/sentencizer#to_disk - """ - path = util.ensure_path(path) - path = path.with_suffix(".json") - srsly.write_json(path, {"punct_chars": list(self.punct_chars)}) - - - def from_disk(self, path, exclude=tuple(), **kwargs): - """Load the sentencizer from disk. - - DOCS: https://spacy.io/api/sentencizer#from_disk - """ - path = util.ensure_path(path) - path = path.with_suffix(".json") - cfg = srsly.read_json(path) - self.punct_chars = set(cfg.get("punct_chars", self.default_punct_chars)) - return self - - -# Cython classes can't be decorated, so we need to add the factories here -Language.factories["parser"] = lambda nlp, model, **cfg: parser_factory(nlp, model, **cfg) -Language.factories["ner"] = lambda nlp, model, **cfg: ner_factory(nlp, model, **cfg) - -def parser_factory(nlp, model, **cfg): - default_config = {"learn_tokens": False, "min_action_freq": 30, "beam_width": 1, "beam_update_prob": 1.0} - if model is None: - model = default_parser() - warnings.warn(Warnings.W098.format(name="parser")) - for key, value in default_config.items(): - if key not in cfg: - cfg[key] = value - return DependencyParser.from_nlp(nlp, model, **cfg) - -def ner_factory(nlp, model, **cfg): - default_config = {"learn_tokens": False, "min_action_freq": 30, "beam_width": 1, "beam_update_prob": 1.0} - if model is None: - model = default_ner() - warnings.warn(Warnings.W098.format(name="ner")) - for key, value in default_config.items(): - if key not in cfg: - cfg[key] = value - return EntityRecognizer.from_nlp(nlp, model, **cfg) - -__all__ = ["Tagger", "DependencyParser", "EntityRecognizer", "TextCategorizer", "EntityLinker", "Sentencizer", "SentenceRecognizer"] diff --git a/spacy/pipeline/sentencizer.pyx b/spacy/pipeline/sentencizer.pyx new file mode 100644 index 000000000..c827ffc5c --- /dev/null +++ b/spacy/pipeline/sentencizer.pyx @@ -0,0 +1,173 @@ +# cython: infer_types=True, profile=True, binding=True +import srsly +from typing import Optional, List + +from ..tokens.doc cimport Doc + +from .pipe import Pipe +from ..language import Language +from .. import util + + +@Language.factory( + "sentencizer", + assigns=["token.is_sent_start", "doc.sents"], + default_config={"punct_chars": None} +) +def make_sentencizer( + nlp: Language, + name: str, + punct_chars: Optional[List[str]] +): + return Sentencizer(name, punct_chars=punct_chars) + + +class Sentencizer(Pipe): + """Segment the Doc into sentences using a rule-based strategy. + + DOCS: https://spacy.io/api/sentencizer + """ + + default_punct_chars = ['!', '.', '?', '։', '؟', '۔', '܀', '܁', '܂', '߹', + '।', '॥', '၊', '။', '።', '፧', '፨', '᙮', '᜵', '᜶', '᠃', '᠉', '᥄', + '᥅', '᪨', '᪩', '᪪', '᪫', '᭚', '᭛', '᭞', '᭟', '᰻', '᰼', '᱾', '᱿', + '‼', '‽', '⁇', '⁈', '⁉', '⸮', '⸼', '꓿', '꘎', '꘏', '꛳', '꛷', '꡶', + '꡷', '꣎', '꣏', '꤯', '꧈', '꧉', '꩝', '꩞', '꩟', '꫰', '꫱', '꯫', '﹒', + '﹖', '﹗', '！', '．', '？', '𐩖', '𐩗', '𑁇', '𑁈', '𑂾', '𑂿', '𑃀', + '𑃁', '𑅁', '𑅂', '𑅃', '𑇅', '𑇆', '𑇍', '𑇞', '𑇟', '𑈸', '𑈹', '𑈻', '𑈼', + '𑊩', '𑑋', '𑑌', '𑗂', '𑗃', '𑗉', '𑗊', '𑗋', '𑗌', '𑗍', '𑗎', '𑗏', '𑗐', + '𑗑', '𑗒', '𑗓', '𑗔', '𑗕', '𑗖', '𑗗', '𑙁', '𑙂', '𑜼', '𑜽', '𑜾', '𑩂', + '𑩃', '𑪛', '𑪜', '𑱁', '𑱂', '𖩮', '𖩯', '𖫵', '𖬷', '𖬸', '𖭄', '𛲟', '𝪈', + '｡', '。'] + + def __init__(self, name="sentencizer", *, punct_chars): + """Initialize the sentencizer. + + punct_chars (list): Punctuation characters to split on. Will be + serialized with the nlp object. + RETURNS (Sentencizer): The sentencizer component. + + DOCS: https://spacy.io/api/sentencizer#init + """ + self.name = name + if punct_chars: + self.punct_chars = set(punct_chars) + else: + self.punct_chars = set(self.default_punct_chars) + + def begin_training(self, get_examples=lambda: [], pipeline=None, sgd=None): + pass + + def __call__(self, doc): + """Apply the sentencizer to a Doc and set Token.is_sent_start. + + example (Doc or Example): The document to process. + RETURNS (Doc or Example): The processed Doc or Example. + + DOCS: https://spacy.io/api/sentencizer#call + """ + start = 0 + seen_period = False + for i, token in enumerate(doc): + is_in_punct_chars = token.text in self.punct_chars + token.is_sent_start = i == 0 + if seen_period and not token.is_punct and not is_in_punct_chars: + doc[start].is_sent_start = True + start = token.i + seen_period = False + elif is_in_punct_chars: + seen_period = True + if start < len(doc): + doc[start].is_sent_start = True + return doc + + def pipe(self, stream, batch_size=128): + for docs in util.minibatch(stream, size=batch_size): + predictions = self.predict(docs) + self.set_annotations(docs, predictions) + yield from docs + + def predict(self, docs): + """Apply the pipeline's model to a batch of docs, without + modifying them. + """ + if not any(len(doc) for doc in docs): + # Handle cases where there are no tokens in any docs. + guesses = [[] for doc in docs] + return guesses + guesses = [] + for doc in docs: + doc_guesses = [False] * len(doc) + if len(doc) > 0: + start = 0 + seen_period = False + doc_guesses[0] = True + for i, token in enumerate(doc): + is_in_punct_chars = token.text in self.punct_chars + if seen_period and not token.is_punct and not is_in_punct_chars: + doc_guesses[start] = True + start = token.i + seen_period = False + elif is_in_punct_chars: + seen_period = True + if start < len(doc): + doc_guesses[start] = True + guesses.append(doc_guesses) + return guesses + + def set_annotations(self, docs, batch_tag_ids): + if isinstance(docs, Doc): + docs = [docs] + cdef Doc doc + cdef int idx = 0 + for i, doc in enumerate(docs): + doc_tag_ids = batch_tag_ids[i] + for j, tag_id in enumerate(doc_tag_ids): + # Don't clobber existing sentence boundaries + if doc.c[j].sent_start == 0: + if tag_id: + doc.c[j].sent_start = 1 + else: + doc.c[j].sent_start = -1 + + def to_bytes(self, exclude=tuple()): + """Serialize the sentencizer to a bytestring. + + RETURNS (bytes): The serialized object. + + DOCS: https://spacy.io/api/sentencizer#to_bytes + """ + return srsly.msgpack_dumps({"punct_chars": list(self.punct_chars)}) + + def from_bytes(self, bytes_data, exclude=tuple()): + """Load the sentencizer from a bytestring. + + bytes_data (bytes): The data to load. + returns (Sentencizer): The loaded object. + + DOCS: https://spacy.io/api/sentencizer#from_bytes + """ + cfg = srsly.msgpack_loads(bytes_data) + self.punct_chars = set(cfg.get("punct_chars", self.default_punct_chars)) + return self + + def to_disk(self, path, exclude=tuple()): + """Serialize the sentencizer to disk. + + DOCS: https://spacy.io/api/sentencizer#to_disk + """ + path = util.ensure_path(path) + path = path.with_suffix(".json") + srsly.write_json(path, {"punct_chars": list(self.punct_chars)}) + + + def from_disk(self, path, exclude=tuple()): + """Load the sentencizer from disk. + + DOCS: https://spacy.io/api/sentencizer#from_disk + """ + path = util.ensure_path(path) + path = path.with_suffix(".json") + cfg = srsly.read_json(path) + self.punct_chars = set(cfg.get("punct_chars", self.default_punct_chars)) + return self diff --git a/spacy/pipeline/senter.pyx b/spacy/pipeline/senter.pyx new file mode 100644 index 000000000..603b7965e --- /dev/null +++ b/spacy/pipeline/senter.pyx @@ -0,0 +1,151 @@ +# cython: infer_types=True, profile=True, binding=True +import srsly +from thinc.api import Model, SequenceCategoricalCrossentropy, Config + +from ..tokens.doc cimport Doc + +from .pipe import deserialize_config +from .tagger import Tagger +from ..language import Language +from ..errors import Errors +from .. import util + + +default_model_config = """ +[model] +@architectures = "spacy.Tagger.v1" + +[model.tok2vec] +@architectures = "spacy.HashEmbedCNN.v1" +pretrained_vectors = null +width = 12 +depth = 1 +embed_size = 2000 +window_size = 1 +maxout_pieces = 2 +subword_features = true +dropout = null +""" +DEFAULT_SENTER_MODEL = Config().from_str(default_model_config)["model"] + + +@Language.factory( + "senter", + assigns=["token.is_sent_start"], + default_config={"model": DEFAULT_SENTER_MODEL} +) +def make_senter(nlp: Language, name: str, model: Model): + return SentenceRecognizer(nlp.vocab, model, name) + + +class SentenceRecognizer(Tagger): + """Pipeline component for sentence segmentation. + + DOCS: https://spacy.io/api/sentencerecognizer + """ + def __init__(self, vocab, model, name="senter"): + self.vocab = vocab + self.model = model + self.name = name + self._rehearsal_model = None + self.cfg = {} + + @property + def labels(self): + # labels are numbered by index internally, so this matches GoldParse + # and Example where the sentence-initial tag is 1 and other positions + # are 0 + return tuple(["I", "S"]) + + def set_annotations(self, docs, batch_tag_ids): + if isinstance(docs, Doc): + docs = [docs] + cdef Doc doc + for i, doc in enumerate(docs): + doc_tag_ids = batch_tag_ids[i] + if hasattr(doc_tag_ids, "get"): + doc_tag_ids = doc_tag_ids.get() + for j, tag_id in enumerate(doc_tag_ids): + # Don't clobber existing sentence boundaries + if doc.c[j].sent_start == 0: + if tag_id == 1: + doc.c[j].sent_start = 1 + else: + doc.c[j].sent_start = -1 + + def get_loss(self, examples, scores): + labels = self.labels + loss_func = SequenceCategoricalCrossentropy(names=labels, normalize=False) + truths = [] + for eg in examples: + eg_truth = [] + for x in eg.get_aligned("sent_start"): + if x == None: + eg_truth.append(None) + elif x == 1: + eg_truth.append(labels[1]) + else: + # anything other than 1: 0, -1, -1 as uint64 + eg_truth.append(labels[0]) + truths.append(eg_truth) + d_scores, loss = loss_func(scores, truths) + if self.model.ops.xp.isnan(loss): + raise ValueError("nan value when computing loss") + return float(loss), d_scores + + def begin_training(self, get_examples=lambda: [], pipeline=None, sgd=None): + self.set_output(len(self.labels)) + self.model.initialize() + util.link_vectors_to_models(self.vocab) + if sgd is None: + sgd = self.create_optimizer() + return sgd + + def add_label(self, label, values=None): + raise NotImplementedError + + def to_bytes(self, exclude=tuple()): + serialize = {} + serialize["model"] = self.model.to_bytes + serialize["vocab"] = self.vocab.to_bytes + serialize["cfg"] = lambda: srsly.json_dumps(self.cfg) + return util.to_bytes(serialize, exclude) + + def from_bytes(self, bytes_data, exclude=tuple()): + def load_model(b): + try: + self.model.from_bytes(b) + except AttributeError: + raise ValueError(Errors.E149) + + deserialize = { + "vocab": lambda b: self.vocab.from_bytes(b), + "cfg": lambda b: self.cfg.update(srsly.json_loads(b)), + "model": lambda b: load_model(b), + } + util.from_bytes(bytes_data, deserialize, exclude) + return self + + def to_disk(self, path, exclude=tuple()): + serialize = { + "vocab": lambda p: self.vocab.to_disk(p), + "model": lambda p: p.open("wb").write(self.model.to_bytes()), + "cfg": lambda p: srsly.write_json(p, self.cfg), + } + util.to_disk(path, serialize, exclude) + + def from_disk(self, path, exclude=tuple()): + def load_model(p): + with p.open("rb") as file_: + try: + self.model.from_bytes(file_.read()) + except AttributeError: + raise ValueError(Errors.E149) + + deserialize = { + "vocab": lambda p: self.vocab.from_disk(p), + "cfg": lambda p: self.cfg.update(deserialize_config(p)), + "model": load_model, + } + util.from_disk(path, deserialize, exclude) + return self diff --git a/spacy/pipeline/simple_ner.py b/spacy/pipeline/simple_ner.py index bf5783b1a..4086c0710 100644 --- a/spacy/pipeline/simple_ner.py +++ b/spacy/pipeline/simple_ner.py @@ -1,43 +1,76 @@ -from typing import List +from typing import List, Iterable, Optional, Dict, Tuple, Callable from thinc.types import Floats2d -from thinc.api import SequenceCategoricalCrossentropy, set_dropout_rate +from thinc.api import SequenceCategoricalCrossentropy, set_dropout_rate, Model +from thinc.api import Optimizer, Config from thinc.util import to_numpy -from .defaults import default_simple_ner from ..gold import Example, spans_from_biluo_tags, iob_to_biluo, biluo_to_iob from ..tokens import Doc -from ..language import component -from ..util import link_vectors_to_models -from .pipes import Pipe +from ..language import Language +from ..vocab import Vocab +from .. import util +from .pipe import Pipe + + +default_model_config = """ +[model] +@architectures = "spacy.BiluoTagger.v1" + +[model.tok2vec] +@architectures = "spacy.HashEmbedCNN.v1" +pretrained_vectors = null +width = 128 +depth = 4 +embed_size = 7000 +window_size = 1 +maxout_pieces = 3 +subword_features = true +dropout = null +""" +DEFAULT_SIMPLE_NER_MODEL = Config().from_str(default_model_config)["model"] + + +@Language.factory( + "simple_ner", + assigns=["doc.ents"], + default_config={"labels": [], "model": DEFAULT_SIMPLE_NER_MODEL}, +) +def make_simple_ner( + nlp: Language, name: str, model: Model, labels: Iterable[str] +) -> "SimpleNER": + return SimpleNER(nlp.vocab, model, name, labels=labels) -@component("simple_ner", assigns=["doc.ents"], default_model=default_simple_ner) class SimpleNER(Pipe): """Named entity recognition with a tagging model. The model should include validity constraints to ensure that only valid tag sequences are returned.""" - def __init__(self, vocab, model): + def __init__( + self, + vocab: Vocab, + model: Model, + name: str = "simple_ner", + *, + labels: Iterable[str], + ) -> None: self.vocab = vocab self.model = model - self.cfg = {"labels": []} + self.name = name + self.labels = labels self.loss_func = SequenceCategoricalCrossentropy( names=self.get_tag_names(), normalize=True, missing_value=None ) assert self.model is not None @property - def labels(self): - return self.cfg["labels"] - - @property - def is_biluo(self): + def is_biluo(self) -> bool: return self.model.name.startswith("biluo") - def add_label(self, label): - if label not in self.cfg["labels"]: - self.cfg["labels"].append(label) + def add_label(self, label: str) -> None: + if label not in self.labels: + self.labels.append(label) - def get_tag_names(self): + def get_tag_names(self) -> List[str]: if self.is_biluo: return ( [f"B-{label}" for label in self.labels] @@ -57,7 +90,7 @@ class SimpleNER(Pipe): scores = self.model.predict(docs) return scores - def set_annotations(self, docs: List[Doc], scores: List[Floats2d]): + def set_annotations(self, docs: List[Doc], scores: List[Floats2d]) -> None: """Set entities on a batch of documents from a batch of scores.""" tag_names = self.get_tag_names() for i, doc in enumerate(docs): @@ -67,7 +100,15 @@ class SimpleNER(Pipe): tags = iob_to_biluo(tags) doc.ents = spans_from_biluo_tags(doc, tags) - def update(self, examples, *, set_annotations=False, drop=0.0, sgd=None, losses=None): + def update( + self, + examples: List[Example], + *, + set_annotations: bool = False, + drop: float = 0.0, + sgd: Optional[Optimizer] = None, + losses: Optional[Dict[str, float]] = None, + ) -> Dict[str, float]: if losses is None: losses = {} losses.setdefault("ner", 0.0) @@ -85,7 +126,7 @@ class SimpleNER(Pipe): losses["ner"] += loss return losses - def get_loss(self, examples, scores): + def get_loss(self, examples: List[Example], scores) -> Tuple[List[Floats2d], float]: loss = 0 d_scores = [] truths = [] @@ -105,8 +146,12 @@ class SimpleNER(Pipe): d_scores, loss = self.loss_func(scores, truths) return loss, d_scores - def begin_training(self, get_examples, pipeline=None, sgd=None, **kwargs): - self.cfg.update(kwargs) + def begin_training( + self, + get_examples: Callable, + pipeline: Optional[List[Tuple[str, Callable[[Doc], Doc]]]] = None, + sgd: Optional[Optimizer] = None, + ): if not hasattr(get_examples, "__call__"): gold_tuples = get_examples get_examples = lambda: gold_tuples @@ -119,18 +164,17 @@ class SimpleNER(Pipe): self.model.initialize() if pipeline is not None: self.init_multitask_objectives(get_examples, pipeline, sgd=sgd, **self.cfg) - link_vectors_to_models(self.vocab) + util.link_vectors_to_models(self.vocab) self.loss_func = SequenceCategoricalCrossentropy( names=self.get_tag_names(), normalize=True, missing_value=None ) - return sgd def init_multitask_objectives(self, *args, **kwargs): pass -def _has_ner(example): +def _has_ner(example: Example) -> bool: for ner_tag in example.get_aligned_ner(): if ner_tag != "-" and ner_tag is not None: return True @@ -138,7 +182,7 @@ def _has_ner(example): return False -def _get_labels(examples): +def _get_labels(examples: List[Example]) -> List[str]: labels = set() for eg in examples: for ner_tag in eg.get_aligned("ENT_TYPE", as_string=True): diff --git a/spacy/pipeline/tagger.pyx b/spacy/pipeline/tagger.pyx new file mode 100644 index 000000000..e4250b932 --- /dev/null +++ b/spacy/pipeline/tagger.pyx @@ -0,0 +1,331 @@ +# cython: infer_types=True, profile=True, binding=True +import numpy +import srsly + +from thinc.api import Model, set_dropout_rate, SequenceCategoricalCrossentropy, Config +import warnings + +from ..tokens.doc cimport Doc +from ..morphology cimport Morphology +from ..vocab cimport Vocab + +from .pipe import Pipe, deserialize_config +from ..language import Language +from ..attrs import POS, ID +from ..parts_of_speech import X +from ..errors import Errors, TempErrors, Warnings +from .. import util + + +default_model_config = """ +[model] +@architectures = "spacy.Tagger.v1" + +[model.tok2vec] +@architectures = "spacy.HashEmbedCNN.v1" +pretrained_vectors = null +width = 96 +depth = 4 +embed_size = 2000 +window_size = 1 +maxout_pieces = 3 +subword_features = true +dropout = null +""" +DEFAULT_TAGGER_MODEL = Config().from_str(default_model_config)["model"] + + +@Language.factory( + "tagger", + assigns=["token.tag"], + default_config={"model": DEFAULT_TAGGER_MODEL, "set_morphology": False} +) +def make_tagger(nlp: Language, name: str, model: Model, set_morphology: bool): + return Tagger(nlp.vocab, model, name, set_morphology=set_morphology) + + +class Tagger(Pipe): + """Pipeline component for part-of-speech tagging. + + DOCS: https://spacy.io/api/tagger + """ + def __init__(self, vocab, model, name="tagger", *, set_morphology=False): + self.vocab = vocab + self.model = model + self.name = name + self._rehearsal_model = None + cfg = {"set_morphology": set_morphology} + self.cfg = dict(sorted(cfg.items())) + + @property + def labels(self): + return tuple(self.vocab.morphology.tag_names) + + def __call__(self, doc): + tags = self.predict([doc]) + self.set_annotations([doc], tags) + return doc + + def pipe(self, stream, batch_size=128): + for docs in util.minibatch(stream, size=batch_size): + tag_ids = self.predict(docs) + self.set_annotations(docs, tag_ids) + yield from docs + + def predict(self, docs): + if not any(len(doc) for doc in docs): + # Handle cases where there are no tokens in any docs. + n_labels = len(self.labels) + guesses = [self.model.ops.alloc((0, n_labels)) for doc in docs] + assert len(guesses) == len(docs) + return guesses + scores = self.model.predict(docs) + assert len(scores) == len(docs), (len(scores), len(docs)) + guesses = self._scores2guesses(scores) + assert len(guesses) == len(docs) + return guesses + + def _scores2guesses(self, scores): + guesses = [] + for doc_scores in scores: + doc_guesses = doc_scores.argmax(axis=1) + if not isinstance(doc_guesses, numpy.ndarray): + doc_guesses = doc_guesses.get() + guesses.append(doc_guesses) + return guesses + + def set_annotations(self, docs, batch_tag_ids): + if isinstance(docs, Doc): + docs = [docs] + cdef Doc doc + cdef int idx = 0 + cdef Vocab vocab = self.vocab + assign_morphology = self.cfg.get("set_morphology", True) + for i, doc in enumerate(docs): + doc_tag_ids = batch_tag_ids[i] + if hasattr(doc_tag_ids, "get"): + doc_tag_ids = doc_tag_ids.get() + for j, tag_id in enumerate(doc_tag_ids): + # Don't clobber preset POS tags + if doc.c[j].tag == 0: + if doc.c[j].pos == 0 and assign_morphology: + # Don't clobber preset lemmas + lemma = doc.c[j].lemma + vocab.morphology.assign_tag_id(&doc.c[j], tag_id) + if lemma != 0 and lemma != doc.c[j].lex.orth: + doc.c[j].lemma = lemma + else: + doc.c[j].tag = self.vocab.strings[self.labels[tag_id]] + idx += 1 + doc.is_tagged = True + + def update(self, examples, *, drop=0., sgd=None, losses=None, set_annotations=False): + if losses is None: + losses = {} + losses.setdefault(self.name, 0.0) + + try: + if not any(len(eg.predicted) if eg.predicted else 0 for eg in examples): + # Handle cases where there are no tokens in any docs. + return + except AttributeError: + types = set([type(eg) for eg in examples]) + raise TypeError(Errors.E978.format(name="Tagger", method="update", types=types)) + set_dropout_rate(self.model, drop) + tag_scores, bp_tag_scores = self.model.begin_update( + [eg.predicted for eg in examples]) + for sc in tag_scores: + if self.model.ops.xp.isnan(sc.sum()): + raise ValueError("nan value in scores") + loss, d_tag_scores = self.get_loss(examples, tag_scores) + bp_tag_scores(d_tag_scores) + if sgd not in (None, False): + self.model.finish_update(sgd) + + losses[self.name] += loss + if set_annotations: + docs = [eg.predicted for eg in examples] + self.set_annotations(docs, self._scores2guesses(tag_scores)) + return losses + + def rehearse(self, examples, drop=0., sgd=None, losses=None): + """Perform a 'rehearsal' update, where we try to match the output of + an initial model. + """ + try: + docs = [eg.predicted for eg in examples] + except AttributeError: + types = set([type(eg) for eg in examples]) + raise TypeError(Errors.E978.format(name="Tagger", method="rehearse", types=types)) + if self._rehearsal_model is None: + return + if not any(len(doc) for doc in docs): + # Handle cases where there are no tokens in any docs. + return + set_dropout_rate(self.model, drop) + guesses, backprop = self.model.begin_update(docs) + target = self._rehearsal_model(examples) + gradient = guesses - target + backprop(gradient) + self.model.finish_update(sgd) + if losses is not None: + losses.setdefault(self.name, 0.0) + losses[self.name] += (gradient**2).sum() + + def get_loss(self, examples, scores): + loss_func = SequenceCategoricalCrossentropy(names=self.labels, normalize=False) + truths = [eg.get_aligned("tag", as_string=True) for eg in examples] + d_scores, loss = loss_func(scores, truths) + if self.model.ops.xp.isnan(loss): + raise ValueError("nan value when computing loss") + return float(loss), d_scores + + def begin_training(self, get_examples=lambda: [], pipeline=None, sgd=None): + lemma_tables = ["lemma_rules", "lemma_index", "lemma_exc", "lemma_lookup"] + if not any(table in self.vocab.lookups for table in lemma_tables): + warnings.warn(Warnings.W022) + if len(self.vocab.lookups.get_table("lexeme_norm", {})) == 0: + warnings.warn(Warnings.W033.format(model="part-of-speech tagger")) + orig_tag_map = dict(self.vocab.morphology.tag_map) + new_tag_map = {} + for example in get_examples(): + try: + y = example.y + except AttributeError: + raise TypeError(Errors.E978.format(name="Tagger", method="begin_training", types=type(example))) + for token in y: + tag = token.tag_ + if tag in orig_tag_map: + new_tag_map[tag] = orig_tag_map[tag] + else: + new_tag_map[tag] = {POS: X} + + cdef Vocab vocab = self.vocab + if new_tag_map: + if "_SP" in orig_tag_map: + new_tag_map["_SP"] = orig_tag_map["_SP"] + vocab.morphology.load_tag_map(new_tag_map) + self.set_output(len(self.labels)) + doc_sample = [Doc(self.vocab, words=["hello", "world"])] + if pipeline is not None: + for name, component in pipeline: + if component is self: + break + if hasattr(component, "pipe"): + doc_sample = list(component.pipe(doc_sample)) + else: + doc_sample = [component(doc) for doc in doc_sample] + self.model.initialize(X=doc_sample) + # Get batch of example docs, example outputs to call begin_training(). + # This lets the model infer shapes. + util.link_vectors_to_models(self.vocab) + if sgd is None: + sgd = self.create_optimizer() + return sgd + + def add_label(self, label, values=None): + if not isinstance(label, str): + raise ValueError(Errors.E187) + if label in self.labels: + return 0 + if self.model.has_dim("nO"): + # Here's how the model resizing will work, once the + # neuron-to-tag mapping is no longer controlled by + # the Morphology class, which sorts the tag names. + # The sorting makes adding labels difficult. + # smaller = self.model._layers[-1] + # larger = Softmax(len(self.labels)+1, smaller.nI) + # copy_array(larger.W[:smaller.nO], smaller.W) + # copy_array(larger.b[:smaller.nO], smaller.b) + # self.model._layers[-1] = larger + raise ValueError(TempErrors.T003) + tag_map = dict(self.vocab.morphology.tag_map) + if values is None: + values = {POS: "X"} + tag_map[label] = values + self.vocab.morphology.load_tag_map(tag_map) + return 1 + + def use_params(self, params): + with self.model.use_params(params): + yield + + def to_bytes(self, exclude=tuple()): + serialize = {} + serialize["model"] = self.model.to_bytes + serialize["vocab"] = self.vocab.to_bytes + serialize["cfg"] = lambda: srsly.json_dumps(self.cfg) + tag_map = dict(sorted(self.vocab.morphology.tag_map.items())) + serialize["tag_map"] = lambda: srsly.msgpack_dumps(tag_map) + morph_rules = dict(self.vocab.morphology.exc) + serialize["morph_rules"] = lambda: srsly.msgpack_dumps(morph_rules) + return util.to_bytes(serialize, exclude) + + def from_bytes(self, bytes_data, exclude=tuple()): + def load_model(b): + try: + self.model.from_bytes(b) + except AttributeError: + raise ValueError(Errors.E149) + + def load_tag_map(b): + tag_map = srsly.msgpack_loads(b) + self.vocab.morphology.load_tag_map(tag_map) + + def load_morph_rules(b): + morph_rules = srsly.msgpack_loads(b) + self.vocab.morphology.load_morph_exceptions(morph_rules) + + self.vocab.morphology = Morphology(self.vocab.strings, dict(), + lemmatizer=self.vocab.morphology.lemmatizer) + + deserialize = { + "vocab": lambda b: self.vocab.from_bytes(b), + "tag_map": load_tag_map, + "morph_rules": load_morph_rules, + "cfg": lambda b: self.cfg.update(srsly.json_loads(b)), + "model": lambda b: load_model(b), + } + util.from_bytes(bytes_data, deserialize, exclude) + return self + + def to_disk(self, path, exclude=tuple()): + tag_map = dict(sorted(self.vocab.morphology.tag_map.items())) + morph_rules = dict(self.vocab.morphology.exc) + serialize = { + "vocab": lambda p: self.vocab.to_disk(p), + "tag_map": lambda p: srsly.write_msgpack(p, tag_map), + "morph_rules": lambda p: srsly.write_msgpack(p, morph_rules), + "model": lambda p: self.model.to_disk(p), + "cfg": lambda p: srsly.write_json(p, self.cfg), + } + util.to_disk(path, serialize, exclude) + + def from_disk(self, path, exclude=tuple()): + def load_model(p): + with p.open("rb") as file_: + try: + self.model.from_bytes(file_.read()) + except AttributeError: + raise ValueError(Errors.E149) + + def load_tag_map(p): + tag_map = srsly.read_msgpack(p) + self.vocab.morphology.load_tag_map(tag_map) + + def load_morph_rules(p): + morph_rules = srsly.read_msgpack(p) + self.vocab.morphology.load_morph_exceptions(morph_rules) + + self.vocab.morphology = Morphology(self.vocab.strings, dict(), + lemmatizer=self.vocab.morphology.lemmatizer) + + deserialize = { + "vocab": lambda p: self.vocab.from_disk(p), + "cfg": lambda p: self.cfg.update(deserialize_config(p)), + "tag_map": load_tag_map, + "morph_rules": load_morph_rules, + "model": load_model, + } + util.from_disk(path, deserialize, exclude) + return self diff --git a/spacy/pipeline/textcat.py b/spacy/pipeline/textcat.py new file mode 100644 index 000000000..ff79a600a --- /dev/null +++ b/spacy/pipeline/textcat.py @@ -0,0 +1,252 @@ +from typing import Iterable, Tuple, Optional, Dict, List, Callable +from thinc.api import get_array_module, Model, Optimizer, set_dropout_rate, Config +import numpy + +from .pipe import Pipe +from ..language import Language +from ..gold import Example +from ..errors import Errors +from .. import util +from ..tokens import Doc +from ..vocab import Vocab + + +default_model_config = """ +[model] +@architectures = "spacy.TextCat.v1" +exclusive_classes = false +pretrained_vectors = null +width = 64 +conv_depth = 2 +embed_size = 2000 +window_size = 1 +ngram_size = 1 +dropout = null +""" +DEFAULT_TEXTCAT_MODEL = Config().from_str(default_model_config)["model"] + +bow_model_config = """ +[model] +@architectures = "spacy.TextCatBOW.v1" +exclusive_classes = false +ngram_size: 1 +no_output_layer: false +""" + +cnn_model_config = """ +[model] +@architectures = "spacy.TextCatCNN.v1" +exclusive_classes = false + +[model.tok2vec] +@architectures = "spacy.HashEmbedCNN.v1" +pretrained_vectors = null +width = 96 +depth = 4 +embed_size = 2000 +window_size = 1 +maxout_pieces = 3 +subword_features = true +dropout = null +""" + + +@Language.factory( + "textcat", + assigns=["doc.cats"], + default_config={"labels": [], "model": DEFAULT_TEXTCAT_MODEL}, +) +def make_textcat( + nlp: Language, name: str, model: Model, labels: Iterable[str] +) -> "TextCategorizer": + return TextCategorizer(nlp.vocab, model, name, labels=labels) + + +class TextCategorizer(Pipe): + """Pipeline component for text classification. + + DOCS: https://spacy.io/api/textcategorizer + """ + + def __init__( + self, + vocab: Vocab, + model: Model, + name: str = "textcat", + *, + labels: Iterable[str], + ) -> None: + self.vocab = vocab + self.model = model + self.name = name + self._rehearsal_model = None + cfg = {"labels": labels} + self.cfg = dict(cfg) + + @property + def labels(self) -> Tuple[str]: + return tuple(self.cfg.setdefault("labels", [])) + + def require_labels(self) -> None: + """Raise an error if the component's model has no labels defined.""" + if not self.labels: + raise ValueError(Errors.E143.format(name=self.name)) + + @labels.setter + def labels(self, value: Iterable[str]) -> None: + self.cfg["labels"] = tuple(value) + + def pipe(self, stream, batch_size=128): + for docs in util.minibatch(stream, size=batch_size): + scores = self.predict(docs) + self.set_annotations(docs, scores) + yield from docs + + def predict(self, docs: Iterable[Doc]): + tensors = [doc.tensor for doc in docs] + if not any(len(doc) for doc in docs): + # Handle cases where there are no tokens in any docs. + xp = get_array_module(tensors) + scores = xp.zeros((len(docs), len(self.labels))) + return scores + scores = self.model.predict(docs) + scores = self.model.ops.asarray(scores) + return scores + + def set_annotations(self, docs: Iterable[Doc], scores) -> None: + for i, doc in enumerate(docs): + for j, label in enumerate(self.labels): + doc.cats[label] = float(scores[i, j]) + + def update( + self, + examples: Iterable[Example], + *, + drop: float = 0.0, + set_annotations: bool = False, + sgd: Optional[Optimizer] = None, + losses: Optional[Dict[str, float]] = None, + ) -> Dict[str, float]: + if losses is None: + losses = {} + losses.setdefault(self.name, 0.0) + try: + if not any(len(eg.predicted) if eg.predicted else 0 for eg in examples): + # Handle cases where there are no tokens in any docs. + return losses + except AttributeError: + types = set([type(eg) for eg in examples]) + raise TypeError( + Errors.E978.format(name="TextCategorizer", method="update", types=types) + ) + set_dropout_rate(self.model, drop) + scores, bp_scores = self.model.begin_update([eg.predicted for eg in examples]) + loss, d_scores = self.get_loss(examples, scores) + bp_scores(d_scores) + if sgd is not None: + self.model.finish_update(sgd) + losses[self.name] += loss + if set_annotations: + docs = [eg.predicted for eg in examples] + self.set_annotations(docs, scores=scores) + return losses + + def rehearse( + self, + examples: Iterable[Example], + drop: float = 0.0, + sgd: Optional[Optimizer] = None, + losses: Optional[Dict[str, float]] = None, + ) -> None: + if self._rehearsal_model is None: + return + try: + docs = [eg.predicted for eg in examples] + except AttributeError: + types = set([type(eg) for eg in examples]) + err = Errors.E978.format( + name="TextCategorizer", method="rehearse", types=types + ) + raise TypeError(err) + if not any(len(doc) for doc in docs): + # Handle cases where there are no tokens in any docs. + return + set_dropout_rate(self.model, drop) + scores, bp_scores = self.model.begin_update(docs) + target = self._rehearsal_model(examples) + gradient = scores - target + bp_scores(gradient) + if sgd is not None: + self.model.finish_update(sgd) + if losses is not None: + losses.setdefault(self.name, 0.0) + losses[self.name] += (gradient ** 2).sum() + + def _examples_to_truth( + self, examples: List[Example] + ) -> Tuple[numpy.ndarray, numpy.ndarray]: + truths = numpy.zeros((len(examples), len(self.labels)), dtype="f") + not_missing = numpy.ones((len(examples), len(self.labels)), dtype="f") + for i, eg in enumerate(examples): + for j, label in enumerate(self.labels): + if label in eg.reference.cats: + truths[i, j] = eg.reference.cats[label] + else: + not_missing[i, j] = 0.0 + truths = self.model.ops.asarray(truths) + return truths, not_missing + + def get_loss(self, examples: Iterable[Example], scores) -> Tuple[float, float]: + truths, not_missing = self._examples_to_truth(examples) + not_missing = self.model.ops.asarray(not_missing) + d_scores = (scores - truths) / scores.shape[0] + d_scores *= not_missing + mean_square_error = (d_scores ** 2).sum(axis=1).mean() + return float(mean_square_error), d_scores + + def add_label(self, label: str) -> int: + if not isinstance(label, str): + raise ValueError(Errors.E187) + if label in self.labels: + return 0 + if self.model.has_dim("nO"): + # This functionality was available previously, but was broken. + # The problem is that we resize the last layer, but the last layer + # is actually just an ensemble. We're not resizing the child layers + # - a huge problem. + raise ValueError(Errors.E116) + # smaller = self.model._layers[-1] + # larger = Linear(len(self.labels)+1, smaller.nI) + # copy_array(larger.W[:smaller.nO], smaller.W) + # copy_array(larger.b[:smaller.nO], smaller.b) + # self.model._layers[-1] = larger + self.labels = tuple(list(self.labels) + [label]) + return 1 + + def begin_training( + self, + get_examples: Callable = lambda: [], + pipeline: Optional[List[Tuple[str, Callable[[Doc], Doc]]]] = None, + sgd: Optional[Optimizer] = None, + ) -> Optimizer: + # TODO: begin_training is not guaranteed to see all data / labels ? + examples = list(get_examples()) + for example in examples: + try: + y = example.y + except AttributeError: + err = Errors.E978.format( + name="TextCategorizer", method="update", types=type(example) + ) + raise TypeError(err) + for cat in y.cats: + self.add_label(cat) + self.require_labels() + docs = [Doc(Vocab(), words=["hello"])] + truths, _ = self._examples_to_truth(examples) + self.set_output(len(self.labels)) + util.link_vectors_to_models(self.vocab) + self.model.initialize(X=docs, Y=truths) + if sgd is None: + sgd = self.create_optimizer() + return sgd diff --git a/spacy/pipeline/tok2vec.py b/spacy/pipeline/tok2vec.py index 56afb3925..0322ef26c 100644 --- a/spacy/pipeline/tok2vec.py +++ b/spacy/pipeline/tok2vec.py @@ -1,47 +1,61 @@ -from thinc.api import Model, set_dropout_rate +from typing import Iterator, Sequence, Iterable, Optional, Dict, Callable, List, Tuple +from thinc.api import Model, set_dropout_rate, Optimizer, Config -from .pipes import Pipe +from .pipe import Pipe from ..gold import Example from ..tokens import Doc from ..vocab import Vocab -from ..language import component +from ..language import Language from ..util import link_vectors_to_models, minibatch -from .defaults import default_tok2vec -@component("tok2vec", assigns=["doc.tensor"], default_model=default_tok2vec) +default_model_config = """ +[model] +@architectures = "spacy.HashEmbedCNN.v1" +pretrained_vectors = null +width = 96 +depth = 4 +embed_size = 2000 +window_size = 1 +maxout_pieces = 3 +subword_features = true +dropout = null +""" +DEFAULT_TOK2VEC_MODEL = Config().from_str(default_model_config)["model"] + + +@Language.factory( + "tok2vec", assigns=["doc.tensor"], default_config={"model": DEFAULT_TOK2VEC_MODEL} +) +def make_tok2vec(nlp: Language, name: str, model: Model) -> "Tok2Vec": + return Tok2Vec(nlp.vocab, model, name) + + class Tok2Vec(Pipe): - @classmethod - def from_nlp(cls, nlp, model, **cfg): - return cls(nlp.vocab, model, **cfg) - - def __init__(self, vocab, model, **cfg): + def __init__(self, vocab: Vocab, model: Model, name: str = "tok2vec") -> None: """Construct a new statistical model. Weights are not allocated on initialisation. vocab (Vocab): A `Vocab` instance. The model must share the same `Vocab` instance with the `Doc` objects it will process. - **cfg: Config parameters. """ self.vocab = vocab self.model = model - self.cfg = dict(cfg) + self.name = name self.listeners = [] + self.cfg = {} - def create_listener(self): - listener = Tok2VecListener( - upstream_name="tok2vec", width=self.model.get_dim("nO") - ) + def add_listener(self, listener: "Tok2VecListener") -> None: self.listeners.append(listener) - def add_listener(self, listener): - self.listeners.append(listener) - - def find_listeners(self, model): + def find_listeners(self, model: Model) -> None: for node in model.walk(): - if isinstance(node, Tok2VecListener) and node.upstream_name == self.name: + if isinstance(node, Tok2VecListener) and node.upstream_name in ( + "*", + self.name, + ): self.add_listener(node) - def __call__(self, doc): + def __call__(self, doc: Doc) -> Doc: """Add context-sensitive vectors to a `Doc`, e.g. from a CNN or LSTM model. Vectors are set to the `Doc.tensor` attribute. docs (Doc or iterable): One or more documents to add vectors to. @@ -51,7 +65,7 @@ class Tok2Vec(Pipe): self.set_annotations([doc], tokvecses) return doc - def pipe(self, stream, batch_size=128): + def pipe(self, stream: Iterator[Doc], batch_size: int = 128) -> Iterator[Doc]: """Process `Doc` objects as a stream. stream (iterator): A sequence of `Doc` objects to process. batch_size (int): Number of `Doc` objects to group. @@ -63,7 +77,7 @@ class Tok2Vec(Pipe): self.set_annotations(docs, tokvecses) yield from docs - def predict(self, docs): + def predict(self, docs: Sequence[Doc]): """Return a single tensor for a batch of documents. docs (iterable): A sequence of `Doc` objects. RETURNS (object): Vector representations for each token in the documents. @@ -74,7 +88,7 @@ class Tok2Vec(Pipe): listener.receive(batch_id, tokvecs, None) return tokvecs - def set_annotations(self, docs, tokvecses): + def set_annotations(self, docs: Sequence[Doc], tokvecses) -> None: """Set the tensor attribute for a batch of documents. docs (iterable): A sequence of `Doc` objects. tokvecs (object): Vector representation for each token in the documents. @@ -83,7 +97,15 @@ class Tok2Vec(Pipe): assert tokvecs.shape[0] == len(doc) doc.tensor = tokvecs - def update(self, examples, *, drop=0.0, sgd=None, losses=None, set_annotations=False): + def update( + self, + examples: Iterable[Example], + *, + drop: float = 0.0, + sgd: Optional[Optimizer] = None, + losses: Optional[Dict[str, float]] = None, + set_annotations: bool = False, + ): """Update the model. examples (Iterable[Example]): A batch of examples drop (float): The droput rate. @@ -128,11 +150,14 @@ class Tok2Vec(Pipe): self.set_annotations(docs, tokvecs) return losses - def get_loss(self, docs, golds, scores): + def get_loss(self, examples, scores): pass def begin_training( - self, get_examples=lambda: [], pipeline=None, sgd=None, **kwargs + self, + get_examples: Callable = lambda: [], + pipeline: Optional[List[Tuple[str, Callable[[Doc], Doc]]]] = None, + sgd: Optional[Optimizer] = None, ): """Allocate models and pre-process training data @@ -151,7 +176,7 @@ class Tok2VecListener(Model): name = "tok2vec-listener" - def __init__(self, upstream_name, width): + def __init__(self, upstream_name: str, width: int) -> None: Model.__init__(self, name=self.name, forward=forward, dims={"nO": width}) self.upstream_name = upstream_name self._batch_id = None diff --git a/spacy/schemas.py b/spacy/schemas.py index b577eadb4..bd4939392 100644 --- a/spacy/schemas.py +++ b/spacy/schemas.py @@ -1,10 +1,10 @@ -from typing import Dict, List, Union, Optional, Sequence, Any +from typing import Dict, List, Union, Optional, Sequence, Any, Callable from enum import Enum from pydantic import BaseModel, Field, ValidationError, validator from pydantic import StrictStr, StrictInt, StrictFloat, StrictBool from pydantic import root_validator from collections import defaultdict -from thinc.api import Model, Optimizer +from thinc.api import Optimizer from .attrs import NAMES @@ -52,7 +52,7 @@ class TokenPatternString(BaseModel): class Config: extra = "forbid" - @validator("*", pre=True, whole=True) + @validator("*", pre=True, each_item=True) def raise_for_none(cls, v): if v is None: raise ValueError("None / null is not allowed") @@ -73,7 +73,7 @@ class TokenPatternNumber(BaseModel): class Config: extra = "forbid" - @validator("*", pre=True, whole=True) + @validator("*", pre=True, each_item=True) def raise_for_none(cls, v): if v is None: raise ValueError("None / null is not allowed") @@ -192,6 +192,8 @@ class TrainingSchema(BaseModel): class ConfigSchemaTraining(BaseModel): # fmt: off + base_model: Optional[StrictStr] = Field(..., title="The base model to use") + vectors: Optional[StrictStr] = Field(..., title="Path to vectors") gold_preproc: StrictBool = Field(..., title="Whether to train on gold-standard sentences and tokens") max_length: StrictInt = Field(..., title="Maximum length of examples (longer examples are divided into sentences if possible)") limit: StrictInt = Field(..., title="Number of examples to use (0 for all)") @@ -201,10 +203,10 @@ class ConfigSchemaTraining(BaseModel): max_epochs: StrictInt = Field(..., title="Maximum number of epochs to train for") max_steps: StrictInt = Field(..., title="Maximum number of update steps to train for") eval_frequency: StrictInt = Field(..., title="How often to evaluate during training (steps)") + eval_batch_size: StrictInt = Field(..., title="Evaluation batch size") seed: Optional[StrictInt] = Field(..., title="Random seed") accumulate_gradient: StrictInt = Field(..., title="Whether to divide the batch up into substeps") use_pytorch_for_gpu_memory: StrictBool = Field(..., title="Allocate memory via PyTorch") - use_gpu: StrictInt = Field(..., title="GPU ID or -1 for CPU") scores: List[StrictStr] = Field(..., title="Score types to be printed in overview") score_weights: Dict[StrictStr, Union[StrictFloat, StrictInt]] = Field(..., title="Weights of each score type for selecting final model") init_tok2vec: Optional[StrictStr] = Field(..., title="Path to pretrained tok2vec weights") @@ -213,6 +215,7 @@ class ConfigSchemaTraining(BaseModel): batch_by: StrictStr = Field(..., title="Batch examples by type") raw_text: Optional[StrictStr] = Field(..., title="Raw text") tag_map: Optional[StrictStr] = Field(..., title="Path to JSON-formatted tag map") + morph_rules: Optional[StrictStr] = Field(..., title="Path to morphology rules") batch_size: Union[Sequence[int], int] = Field(..., title="The batch size or batch size schedule") optimizer: Optimizer = Field(..., title="The optimizer to use") # fmt: on @@ -222,29 +225,25 @@ class ConfigSchemaTraining(BaseModel): arbitrary_types_allowed = True -class ConfigSchemaNlpComponent(BaseModel): - factory: StrictStr = Field(..., title="Component factory name") - model: Model = Field(..., title="Component model") - # TODO: add config schema / types for components so we can fill and validate - # component options like learn_tokens, min_action_freq etc. - - class Config: - extra = "allow" - arbitrary_types_allowed = True - - -class ConfigSchemaPipeline(BaseModel): - __root__: Dict[str, ConfigSchemaNlpComponent] +class ConfigSchemaNlpWritingSystem(BaseModel): + direction: StrictStr = Field(..., title="The writing direction, e.g. 'rtl'") + has_case: StrictBool = Field(..., title="Whether the language has case") + has_letters: StrictBool = Field(..., title="Whether the language has letters") class Config: extra = "allow" class ConfigSchemaNlp(BaseModel): + # fmt: off lang: StrictStr = Field(..., title="The base language to use") - base_model: Optional[StrictStr] = Field(..., title="The base model to use") - vectors: Optional[StrictStr] = Field(..., title="Path to vectors") - pipeline: Optional[ConfigSchemaPipeline] + pipeline: List[StrictStr] = Field(..., title="The pipeline component names in order") + tokenizer: Callable = Field(..., title="The tokenizer to use") + lemmatizer: Callable = Field(..., title="The lemmatizer to use") + writing_system: ConfigSchemaNlpWritingSystem = Field(..., title="The language's writing system") + stop_words: Sequence[StrictStr] = Field(..., title="Stop words to mark via Token/Lexeme.is_stop") + lex_attr_getters: Dict[StrictStr, Callable] = Field(..., title="Custom getter functions for lexical attributes (e.g. like_num)") + # fmt: on class Config: extra = "forbid" @@ -261,7 +260,7 @@ class ConfigSchemaPretrain(BaseModel): batch_size: Union[Sequence[int], int] = Field(..., title="The batch size or batch size schedule") seed: Optional[StrictInt] = Field(..., title="Random seed") use_pytorch_for_gpu_memory: StrictBool = Field(..., title="Allocate memory via PyTorch") - tok2vec_model: StrictStr = Field(..., title="tok2vec model in config, e.g. nlp.pipeline.tok2vec.model") + tok2vec_model: StrictStr = Field(..., title="tok2vec model in config, e.g. components.tok2vec.model") optimizer: Optimizer = Field(..., title="The optimizer to use") # TODO: use a more detailed schema for this? objective: Dict[str, Any] = Field(..., title="Pretraining objective") @@ -276,6 +275,7 @@ class ConfigSchema(BaseModel): training: ConfigSchemaTraining nlp: ConfigSchemaNlp pretraining: Optional[ConfigSchemaPretrain] + components: Dict[str, Dict[str, Any]] @root_validator def validate_config(cls, values): diff --git a/spacy/syntax/nn_parser.pxd b/spacy/syntax/nn_parser.pxd index d77a04420..7840ec27a 100644 --- a/spacy/syntax/nn_parser.pxd +++ b/spacy/syntax/nn_parser.pxd @@ -10,13 +10,14 @@ from ._parser_model cimport WeightsC, ActivationsC, SizesC cdef class Parser: cdef readonly Vocab vocab cdef public object model + cdef public str name cdef public object _rehearsal_model cdef readonly TransitionSystem moves cdef readonly object cfg cdef public object _multitasks - + cdef void _parseC(self, StateC** states, WeightsC weights, SizesC sizes) nogil - + cdef void c_transition_batch(self, StateC** states, const float* scores, int nr_class, int batch_size) nogil diff --git a/spacy/syntax/nn_parser.pyx b/spacy/syntax/nn_parser.pyx index 591afe5ab..09616ee75 100644 --- a/spacy/syntax/nn_parser.pyx +++ b/spacy/syntax/nn_parser.pyx @@ -28,7 +28,6 @@ from ._parser_model cimport get_c_weights, get_c_sizes from .stateclass cimport StateClass from ._state cimport StateC from .transition_system cimport Transition -from ..gold.example cimport Example from ..util import link_vectors_to_models, create_default_optimizer, registry from ..compat import copy_array @@ -41,10 +40,19 @@ cdef class Parser: """ Base class of the DependencyParser and EntityRecognizer. """ - name = 'base_parser' - - def __init__(self, Vocab vocab, model, **cfg): + def __init__( + self, + Vocab vocab, + model, + name="base_parser", + moves=None, + *, + update_with_oracle_cut_size, + multitasks=tuple(), + min_action_freq, + learn_tokens, + ): """Create a Parser. vocab (Vocab): The vocabulary object. Must be shared with documents @@ -55,7 +63,14 @@ cdef class Parser: parse-state is created, updated and evaluated. """ self.vocab = vocab - moves = cfg.get("moves", None) + self.name = name + cfg = { + "moves": moves, + "update_with_oracle_cut_size": update_with_oracle_cut_size, + "multitasks": list(multitasks), + "min_action_freq": min_action_freq, + "learn_tokens": learn_tokens + } if moves is None: # defined by EntityRecognizer as a BiluoPushDown moves = self.TransitionSystem(self.vocab.strings) @@ -63,28 +78,17 @@ cdef class Parser: self.model = model if self.moves.n_moves != 0: self.set_output(self.moves.n_moves) - self.cfg = dict(cfg) - self.cfg.setdefault("update_with_oracle_cut_size", 100) + self.cfg = cfg self._multitasks = [] - for multitask in cfg.get("multitasks", []): + for multitask in cfg["multitasks"]: self.add_multitask_objective(multitask) self._rehearsal_model = None - @classmethod - def from_nlp(cls, nlp, model, **cfg): - return cls(nlp.vocab, model, **cfg) - - def __reduce__(self): - return (Parser, (self.vocab, self.model), (self.moves, self.cfg)) - - def __getstate__(self): - return (self.moves, self.cfg) - - def __setstate__(self, state): - moves, config = state - self.moves = moves - self.cfg = config + def __getnewargs_ex__(self): + """This allows pickling the Parser and its keyword-only init arguments""" + args = (self.vocab, self.model, self.name, self.moves) + return args, self.cfg @property def move_names(self): @@ -286,7 +290,7 @@ cdef class Parser: cut_size = self.cfg["update_with_oracle_cut_size"] states, golds, max_steps = self._init_gold_batch( examples, - max_length=cut_size + max_length=cut_size ) else: states, golds, _ = self.moves.init_gold_batch(examples) diff --git a/spacy/tests/conftest.py b/spacy/tests/conftest.py index 907b37205..a2e319e12 100644 --- a/spacy/tests/conftest.py +++ b/spacy/tests/conftest.py @@ -26,47 +26,47 @@ def pytest_runtest_setup(item): @pytest.fixture(scope="module") def tokenizer(): - return get_lang_class("xx").Defaults.create_tokenizer() + return get_lang_class("xx")().tokenizer @pytest.fixture(scope="session") def ar_tokenizer(): - return get_lang_class("ar").Defaults.create_tokenizer() + return get_lang_class("ar")().tokenizer @pytest.fixture(scope="session") def bn_tokenizer(): - return get_lang_class("bn").Defaults.create_tokenizer() + return get_lang_class("bn")().tokenizer @pytest.fixture(scope="session") def ca_tokenizer(): - return get_lang_class("ca").Defaults.create_tokenizer() + return get_lang_class("ca")().tokenizer @pytest.fixture(scope="session") def da_tokenizer(): - return get_lang_class("da").Defaults.create_tokenizer() + return get_lang_class("da")().tokenizer @pytest.fixture(scope="session") def de_tokenizer(): - return get_lang_class("de").Defaults.create_tokenizer() + return get_lang_class("de")().tokenizer @pytest.fixture(scope="session") def el_tokenizer(): - return get_lang_class("el").Defaults.create_tokenizer() + return get_lang_class("el")().tokenizer @pytest.fixture(scope="session") def en_tokenizer(): - return get_lang_class("en").Defaults.create_tokenizer() + return get_lang_class("en")().tokenizer @pytest.fixture(scope="session") def en_vocab(): - return get_lang_class("en").Defaults.create_vocab() + return get_lang_class("en")().vocab @pytest.fixture(scope="session") @@ -77,197 +77,205 @@ def en_parser(en_vocab): @pytest.fixture(scope="session") def es_tokenizer(): - return get_lang_class("es").Defaults.create_tokenizer() + return get_lang_class("es")().tokenizer @pytest.fixture(scope="session") def eu_tokenizer(): - return get_lang_class("eu").Defaults.create_tokenizer() + return get_lang_class("eu")().tokenizer @pytest.fixture(scope="session") def fa_tokenizer(): - return get_lang_class("fa").Defaults.create_tokenizer() + return get_lang_class("fa")().tokenizer @pytest.fixture(scope="session") def fi_tokenizer(): - return get_lang_class("fi").Defaults.create_tokenizer() + return get_lang_class("fi")().tokenizer @pytest.fixture(scope="session") def fr_tokenizer(): - return get_lang_class("fr").Defaults.create_tokenizer() + return get_lang_class("fr")().tokenizer @pytest.fixture(scope="session") def ga_tokenizer(): - return get_lang_class("ga").Defaults.create_tokenizer() + return get_lang_class("ga")().tokenizer @pytest.fixture(scope="session") def gu_tokenizer(): - return get_lang_class("gu").Defaults.create_tokenizer() + return get_lang_class("gu")().tokenizer @pytest.fixture(scope="session") def he_tokenizer(): - return get_lang_class("he").Defaults.create_tokenizer() + return get_lang_class("he")().tokenizer @pytest.fixture(scope="session") def hr_tokenizer(): - return get_lang_class("hr").Defaults.create_tokenizer() + return get_lang_class("hr")().tokenizer @pytest.fixture def hu_tokenizer(): - return get_lang_class("hu").Defaults.create_tokenizer() + return get_lang_class("hu")().tokenizer @pytest.fixture(scope="session") def id_tokenizer(): - return get_lang_class("id").Defaults.create_tokenizer() + return get_lang_class("id")().tokenizer @pytest.fixture(scope="session") def it_tokenizer(): - return get_lang_class("it").Defaults.create_tokenizer() + return get_lang_class("it")().tokenizer @pytest.fixture(scope="session") def ja_tokenizer(): pytest.importorskip("sudachipy") - return get_lang_class("ja").Defaults.create_tokenizer() + return get_lang_class("ja")().tokenizer @pytest.fixture(scope="session") def ko_tokenizer(): pytest.importorskip("natto") - return get_lang_class("ko").Defaults.create_tokenizer() + return get_lang_class("ko")().tokenizer @pytest.fixture(scope="session") def lb_tokenizer(): - return get_lang_class("lb").Defaults.create_tokenizer() + return get_lang_class("lb")().tokenizer @pytest.fixture(scope="session") def lt_tokenizer(): - return get_lang_class("lt").Defaults.create_tokenizer() + return get_lang_class("lt")().tokenizer @pytest.fixture(scope="session") def ml_tokenizer(): - return get_lang_class("ml").Defaults.create_tokenizer() + return get_lang_class("ml")().tokenizer @pytest.fixture(scope="session") def nb_tokenizer(): - return get_lang_class("nb").Defaults.create_tokenizer() + return get_lang_class("nb")().tokenizer @pytest.fixture(scope="session") def ne_tokenizer(): - return get_lang_class("ne").Defaults.create_tokenizer() + return get_lang_class("ne")().tokenizer @pytest.fixture(scope="session") def nl_tokenizer(): - return get_lang_class("nl").Defaults.create_tokenizer() + return get_lang_class("nl")().tokenizer @pytest.fixture(scope="session") def pl_tokenizer(): - return get_lang_class("pl").Defaults.create_tokenizer() + return get_lang_class("pl")().tokenizer @pytest.fixture(scope="session") def pt_tokenizer(): - return get_lang_class("pt").Defaults.create_tokenizer() + return get_lang_class("pt")().tokenizer @pytest.fixture(scope="session") def ro_tokenizer(): - return get_lang_class("ro").Defaults.create_tokenizer() + return get_lang_class("ro")().tokenizer @pytest.fixture(scope="session") def ru_tokenizer(): pytest.importorskip("pymorphy2") - return get_lang_class("ru").Defaults.create_tokenizer() + return get_lang_class("ru")().tokenizer @pytest.fixture def ru_lemmatizer(): pytest.importorskip("pymorphy2") - return get_lang_class("ru").Defaults.create_lemmatizer() + return get_lang_class("ru")().vocab.morphology.lemmatizer @pytest.fixture(scope="session") def sr_tokenizer(): - return get_lang_class("sr").Defaults.create_tokenizer() + return get_lang_class("sr")().tokenizer @pytest.fixture(scope="session") def sv_tokenizer(): - return get_lang_class("sv").Defaults.create_tokenizer() + return get_lang_class("sv")().tokenizer @pytest.fixture(scope="session") def th_tokenizer(): pytest.importorskip("pythainlp") - return get_lang_class("th").Defaults.create_tokenizer() + return get_lang_class("th")().tokenizer @pytest.fixture(scope="session") def tr_tokenizer(): - return get_lang_class("tr").Defaults.create_tokenizer() + return get_lang_class("tr")().tokenizer @pytest.fixture(scope="session") def tt_tokenizer(): - return get_lang_class("tt").Defaults.create_tokenizer() + return get_lang_class("tt")().tokenizer @pytest.fixture(scope="session") def uk_tokenizer(): pytest.importorskip("pymorphy2") pytest.importorskip("pymorphy2.lang") - return get_lang_class("uk").Defaults.create_tokenizer() + return get_lang_class("uk")().tokenizer @pytest.fixture(scope="session") def ur_tokenizer(): - return get_lang_class("ur").Defaults.create_tokenizer() + return get_lang_class("ur")().tokenizer @pytest.fixture(scope="session") def yo_tokenizer(): - return get_lang_class("yo").Defaults.create_tokenizer() + return get_lang_class("yo")().tokenizer @pytest.fixture(scope="session") def zh_tokenizer_char(): - return get_lang_class("zh").Defaults.create_tokenizer() + nlp = get_lang_class("zh")() + return nlp.tokenizer @pytest.fixture(scope="session") def zh_tokenizer_jieba(): pytest.importorskip("jieba") - return get_lang_class("zh").Defaults.create_tokenizer( - config={"segmenter": "jieba"} - ) + config = { + "@tokenizers": "spacy.ChineseTokenizer.v1", + "segmenter": "jieba", + } + nlp = get_lang_class("zh").from_config({"nlp": {"tokenizer": config}}) + return nlp.tokenizer @pytest.fixture(scope="session") def zh_tokenizer_pkuseg(): pytest.importorskip("pkuseg") - return get_lang_class("zh").Defaults.create_tokenizer( - config={"pkuseg_model": "default", "segmenter": "pkuseg"} - ) + config = { + "@tokenizers": "spacy.ChineseTokenizer.v1", + "segmenter": "pkuseg", + "pkuseg_model": "default", + } + nlp = get_lang_class("zh").from_config({"nlp": {"tokenizer": config}}) + return nlp.tokenizer @pytest.fixture(scope="session") def hy_tokenizer(): - return get_lang_class("hy").Defaults.create_tokenizer() + return get_lang_class("hy")().tokenizer diff --git a/spacy/tests/doc/test_add_entities.py b/spacy/tests/doc/test_add_entities.py index b9c230516..c4167f878 100644 --- a/spacy/tests/doc/test_add_entities.py +++ b/spacy/tests/doc/test_add_entities.py @@ -1,9 +1,10 @@ from spacy.pipeline import EntityRecognizer from spacy.tokens import Span +from spacy import registry import pytest from ..util import get_doc -from spacy.pipeline.defaults import default_ner +from spacy.pipeline.ner import DEFAULT_NER_MODEL def test_doc_add_entities_set_ents_iob(en_vocab): @@ -12,10 +13,10 @@ def test_doc_add_entities_set_ents_iob(en_vocab): config = { "learn_tokens": False, "min_action_freq": 30, - "beam_width": 1, - "beam_update_prob": 1.0, + "update_with_oracle_cut_size": 100, } - ner = EntityRecognizer(en_vocab, default_ner(), **config) + model = registry.make_from_config({"model": DEFAULT_NER_MODEL}, validate=True)["model"] + ner = EntityRecognizer(en_vocab, model, **config) ner.begin_training([]) ner(doc) assert len(list(doc.ents)) == 0 @@ -34,10 +35,10 @@ def test_ents_reset(en_vocab): config = { "learn_tokens": False, "min_action_freq": 30, - "beam_width": 1, - "beam_update_prob": 1.0, + "update_with_oracle_cut_size": 100, } - ner = EntityRecognizer(en_vocab, default_ner(), **config) + model = registry.make_from_config({"model": DEFAULT_NER_MODEL}, validate=True)["model"] + ner = EntityRecognizer(en_vocab, model, **config) ner.begin_training([]) ner(doc) assert [t.ent_iob_ for t in doc] == (["O"] * len(doc)) diff --git a/spacy/tests/doc/test_doc_api.py b/spacy/tests/doc/test_doc_api.py index 8a35b1236..a0106348d 100644 --- a/spacy/tests/doc/test_doc_api.py +++ b/spacy/tests/doc/test_doc_api.py @@ -2,6 +2,7 @@ import pytest import numpy from spacy.tokens import Doc, Span from spacy.vocab import Vocab +from spacy.lang.en import English from spacy.attrs import ENT_TYPE, ENT_IOB, SENT_START, HEAD, DEP, MORPH from ..util import get_doc @@ -380,3 +381,11 @@ def test_doc_lang(en_vocab): doc = Doc(en_vocab, words=["Hello", "world"]) assert doc.lang_ == "en" assert doc.lang == en_vocab.strings["en"] + assert doc[0].lang_ == "en" + assert doc[0].lang == en_vocab.strings["en"] + nlp = English() + doc = nlp("Hello world") + assert doc.lang_ == "en" + assert doc.lang == en_vocab.strings["en"] + assert doc[0].lang_ == "en" + assert doc[0].lang == en_vocab.strings["en"] diff --git a/spacy/tests/lang/fr/test_prefix_suffix_infix.py b/spacy/tests/lang/fr/test_prefix_suffix_infix.py index 01d50b0a6..2ead34069 100644 --- a/spacy/tests/lang/fr/test_prefix_suffix_infix.py +++ b/spacy/tests/lang/fr/test_prefix_suffix_infix.py @@ -15,7 +15,7 @@ def test_issue768(text, expected_tokens): class Defaults(Language.Defaults): infixes = TOKENIZER_INFIXES + [SPLIT_INFIX] - fr_tokenizer_w_infix = FrenchTest.Defaults.create_tokenizer() + fr_tokenizer_w_infix = FrenchTest().tokenizer tokens = fr_tokenizer_w_infix(text) assert len(tokens) == 2 assert [t.text for t in tokens] == expected_tokens diff --git a/spacy/tests/lang/ja/test_serialize.py b/spacy/tests/lang/ja/test_serialize.py index 4d4174b03..e05a363bf 100644 --- a/spacy/tests/lang/ja/test_serialize.py +++ b/spacy/tests/lang/ja/test_serialize.py @@ -18,7 +18,7 @@ def test_ja_tokenizer_serialize(ja_tokenizer): assert nlp.tokenizer.split_mode is None # split mode is (de)serialized correctly - nlp = Japanese(meta={"tokenizer": {"config": {"split_mode": "B"}}}) + nlp = Japanese.from_config({"nlp": {"tokenizer": {"split_mode": "B"}}}) nlp_r = Japanese() nlp_bytes = nlp.to_bytes() nlp_r.from_bytes(nlp_bytes) diff --git a/spacy/tests/lang/ja/test_tokenizer.py b/spacy/tests/lang/ja/test_tokenizer.py index 943b8ceeb..8354aa80c 100644 --- a/spacy/tests/lang/ja/test_tokenizer.py +++ b/spacy/tests/lang/ja/test_tokenizer.py @@ -83,9 +83,9 @@ def test_ja_tokenizer_naughty_strings(ja_tokenizer, text): ], ) def test_ja_tokenizer_split_modes(ja_tokenizer, text, len_a, len_b, len_c): - nlp_a = Japanese(meta={"tokenizer": {"config": {"split_mode": "A"}}}) - nlp_b = Japanese(meta={"tokenizer": {"config": {"split_mode": "B"}}}) - nlp_c = Japanese(meta={"tokenizer": {"config": {"split_mode": "C"}}}) + nlp_a = Japanese.from_config({"nlp": {"tokenizer": {"split_mode": "A"}}}) + nlp_b = Japanese.from_config({"nlp": {"tokenizer": {"split_mode": "B"}}}) + nlp_c = Japanese.from_config({"nlp": {"tokenizer": {"split_mode": "C"}}}) assert len(ja_tokenizer(text)) == len_a assert len(nlp_a(text)) == len_a diff --git a/spacy/tests/lang/ru/test_lemmatizer.py b/spacy/tests/lang/ru/test_lemmatizer.py index 40dcf4cf8..8a87a7506 100644 --- a/spacy/tests/lang/ru/test_lemmatizer.py +++ b/spacy/tests/lang/ru/test_lemmatizer.py @@ -3,6 +3,7 @@ import pytest from ...util import get_doc +@pytest.mark.xfail(reason="TODO: investigate why lemmatizer fails here") def test_ru_doc_lemmatization(ru_tokenizer): words = ["мама", "мыла", "раму"] tags = [ diff --git a/spacy/tests/lang/zh/test_tokenizer.py b/spacy/tests/lang/zh/test_tokenizer.py index 1ebb1e7b7..70e753ba2 100644 --- a/spacy/tests/lang/zh/test_tokenizer.py +++ b/spacy/tests/lang/zh/test_tokenizer.py @@ -1,5 +1,6 @@ import pytest from spacy.lang.zh import Chinese, _get_pkuseg_trie_data +from thinc.config import ConfigValidationError # fmt: off @@ -64,12 +65,14 @@ def test_zh_extra_spaces(zh_tokenizer_char): def test_zh_unsupported_segmenter(): - with pytest.warns(UserWarning): - nlp = Chinese(meta={"tokenizer": {"config": {"segmenter": "unk"}}}) + config = {"nlp": {"tokenizer": {"segmenter": "unk"}}} + with pytest.raises(ConfigValidationError): + Chinese.from_config(config) def test_zh_uninitialized_pkuseg(): - nlp = Chinese(meta={"tokenizer": {"config": {"segmenter": "char"}}}) + config = {"nlp": {"tokenizer": {"segmenter": "char"}}} + nlp = Chinese.from_config(config) nlp.tokenizer.segmenter = "pkuseg" with pytest.raises(ValueError): - doc = nlp("test") + nlp("test") diff --git a/spacy/tests/parser/test_add_label.py b/spacy/tests/parser/test_add_label.py index a9ccda995..a1e7dd388 100644 --- a/spacy/tests/parser/test_add_label.py +++ b/spacy/tests/parser/test_add_label.py @@ -1,11 +1,13 @@ import pytest from thinc.api import Adam, fix_random_seed +from spacy import registry from spacy.attrs import NORM from spacy.vocab import Vocab from spacy.gold import Example -from spacy.pipeline.defaults import default_parser, default_ner from spacy.tokens import Doc from spacy.pipeline import DependencyParser, EntityRecognizer +from spacy.pipeline.ner import DEFAULT_NER_MODEL +from spacy.pipeline.dep_parser import DEFAULT_PARSER_MODEL @pytest.fixture @@ -18,10 +20,10 @@ def parser(vocab): config = { "learn_tokens": False, "min_action_freq": 30, - "beam_width": 1, - "beam_update_prob": 1.0, + "update_with_oracle_cut_size": 100, } - parser = DependencyParser(vocab, default_parser(), **config) + model = registry.make_from_config({"model": DEFAULT_PARSER_MODEL}, validate=True)["model"] + parser = DependencyParser(vocab, model, **config) return parser @@ -64,15 +66,15 @@ def test_add_label_deserializes_correctly(): config = { "learn_tokens": False, "min_action_freq": 30, - "beam_width": 1, - "beam_update_prob": 1.0, + "update_with_oracle_cut_size": 100, } - ner1 = EntityRecognizer(Vocab(), default_ner(), **config) + model = registry.make_from_config({"model": DEFAULT_NER_MODEL}, validate=True)["model"] + ner1 = EntityRecognizer(Vocab(), model, **config) ner1.add_label("C") ner1.add_label("B") ner1.add_label("A") ner1.begin_training([]) - ner2 = EntityRecognizer(Vocab(), default_ner(), **config) + ner2 = EntityRecognizer(Vocab(), model, **config) # the second model needs to be resized before we can call from_bytes ner2.model.attrs["resize_output"](ner2.model, ner1.moves.n_moves) @@ -83,16 +85,22 @@ def test_add_label_deserializes_correctly(): @pytest.mark.parametrize( - "pipe_cls,n_moves,model", - [(DependencyParser, 5, default_parser()), (EntityRecognizer, 4, default_ner())], + "pipe_cls,n_moves,model_config", + [(DependencyParser, 5, DEFAULT_PARSER_MODEL), (EntityRecognizer, 4, DEFAULT_NER_MODEL)], ) -def test_add_label_get_label(pipe_cls, n_moves, model): +def test_add_label_get_label(pipe_cls, n_moves, model_config): """Test that added labels are returned correctly. This test was added to test for a bug in DependencyParser.labels that'd cause it to fail when splitting the move names. """ labels = ["A", "B", "C"] - pipe = pipe_cls(Vocab(), model) + model = registry.make_from_config({"model": model_config}, validate=True)["model"] + config = { + "learn_tokens": False, + "min_action_freq": 30, + "update_with_oracle_cut_size": 100, + } + pipe = pipe_cls(Vocab(), model, **config) for label in labels: pipe.add_label(label) assert len(pipe.move_names) == len(labels) * n_moves diff --git a/spacy/tests/parser/test_arc_eager_oracle.py b/spacy/tests/parser/test_arc_eager_oracle.py index 0ef978bfa..9781f71ed 100644 --- a/spacy/tests/parser/test_arc_eager_oracle.py +++ b/spacy/tests/parser/test_arc_eager_oracle.py @@ -1,12 +1,12 @@ import pytest from spacy.vocab import Vocab - +from spacy import registry from spacy.gold import Example -from spacy.pipeline.defaults import default_parser from spacy.pipeline import DependencyParser from spacy.tokens import Doc from spacy.syntax.nonproj import projectivize from spacy.syntax.arc_eager import ArcEager +from spacy.pipeline.dep_parser import DEFAULT_PARSER_MODEL def get_sequence_costs(M, words, heads, deps, transitions): @@ -124,10 +124,10 @@ def test_get_oracle_actions(): config = { "learn_tokens": False, "min_action_freq": 0, - "beam_width": 1, - "beam_update_prob": 1.0, + "update_with_oracle_cut_size": 100, } - parser = DependencyParser(doc.vocab, default_parser(), **config) + model = registry.make_from_config({"model": DEFAULT_PARSER_MODEL}, validate=True)["model"] + parser = DependencyParser(doc.vocab, model, **config) parser.moves.add_action(0, "") parser.moves.add_action(1, "") parser.moves.add_action(1, "") diff --git a/spacy/tests/parser/test_ner.py b/spacy/tests/parser/test_ner.py index b71285a34..71539fe60 100644 --- a/spacy/tests/parser/test_ner.py +++ b/spacy/tests/parser/test_ner.py @@ -1,17 +1,14 @@ import pytest -from spacy.attrs import ENT_IOB from spacy import util from spacy.lang.en import English from spacy.language import Language from spacy.lookups import Lookups -from spacy.pipeline.defaults import default_ner -from spacy.pipeline import EntityRecognizer, EntityRuler -from spacy.vocab import Vocab from spacy.syntax.ner import BiluoPushDown from spacy.gold import Example from spacy.tokens import Doc +from spacy.vocab import Vocab from ..util import make_tempdir @@ -150,10 +147,8 @@ def test_accept_blocked_token(): config = { "learn_tokens": False, "min_action_freq": 30, - "beam_width": 1, - "beam_update_prob": 1.0, } - ner1 = EntityRecognizer(doc1.vocab, default_ner(), **config) + ner1 = nlp1.create_pipe("ner", config=config) assert [token.ent_iob_ for token in doc1] == ["", "", "", "", ""] assert [token.ent_type_ for token in doc1] == ["", "", "", "", ""] @@ -174,10 +169,8 @@ def test_accept_blocked_token(): config = { "learn_tokens": False, "min_action_freq": 30, - "beam_width": 1, - "beam_update_prob": 1.0, } - ner2 = EntityRecognizer(doc2.vocab, default_ner(), **config) + ner2 = nlp2.create_pipe("ner", config=config) # set "New York" to a blocked entity doc2.ents = [(0, 3, 5)] @@ -212,11 +205,8 @@ def test_train_empty(): train_examples = [] for t in train_data: train_examples.append(Example.from_dict(nlp.make_doc(t[0]), t[1])) - - ner = nlp.create_pipe("ner") + ner = nlp.add_pipe("ner", last=True) ner.add_label("PERSON") - nlp.add_pipe(ner, last=True) - nlp.begin_training() for itn in range(2): losses = {} @@ -227,23 +217,18 @@ def test_train_empty(): def test_overwrite_token(): nlp = English() - ner1 = nlp.create_pipe("ner") - nlp.add_pipe(ner1, name="ner") + nlp.add_pipe("ner") nlp.begin_training() - # The untrained NER will predict O for each token doc = nlp("I live in New York") assert [token.ent_iob_ for token in doc] == ["O", "O", "O", "O", "O"] assert [token.ent_type_ for token in doc] == ["", "", "", "", ""] - # Check that a new ner can overwrite O config = { "learn_tokens": False, "min_action_freq": 30, - "beam_width": 1, - "beam_update_prob": 1.0, } - ner2 = EntityRecognizer(doc.vocab, default_ner(), **config) + ner2 = nlp.create_pipe("ner", config=config) ner2.moves.add_action(5, "") ner2.add_label("GPE") state = ner2.moves.init_batch([doc])[0] @@ -256,9 +241,8 @@ def test_overwrite_token(): def test_empty_ner(): nlp = English() - ner = nlp.create_pipe("ner") + ner = nlp.add_pipe("ner") ner.add_label("MY_LABEL") - nlp.add_pipe(ner) nlp.begin_training() doc = nlp("John is watching the news about Croatia's elections") # if this goes wrong, the initialization of the parser's upper layer is probably broken @@ -271,15 +255,13 @@ def test_ruler_before_ner(): nlp = English() # 1 : Entity Ruler - should set "this" to B and everything else to empty - ruler = EntityRuler(nlp) patterns = [{"label": "THING", "pattern": "This"}] + ruler = nlp.add_pipe("entity_ruler") ruler.add_patterns(patterns) - nlp.add_pipe(ruler) # 2: untrained NER - should set everything else to O - untrained_ner = nlp.create_pipe("ner") + untrained_ner = nlp.add_pipe("ner") untrained_ner.add_label("MY_LABEL") - nlp.add_pipe(untrained_ner) nlp.begin_training() doc = nlp("This is Antti Korhonen speaking in Finland") expected_iobs = ["B", "O", "O", "O", "O", "O", "O"] @@ -293,16 +275,14 @@ def test_ner_before_ruler(): nlp = English() # 1: untrained NER - should set everything to O - untrained_ner = nlp.create_pipe("ner") + untrained_ner = nlp.add_pipe("ner", name="uner") untrained_ner.add_label("MY_LABEL") - nlp.add_pipe(untrained_ner, name="uner") nlp.begin_training() # 2 : Entity Ruler - should set "this" to B and keep everything else O - ruler = EntityRuler(nlp) patterns = [{"label": "THING", "pattern": "This"}] + ruler = nlp.add_pipe("entity_ruler") ruler.add_patterns(patterns) - nlp.add_pipe(ruler) doc = nlp("This is Antti Korhonen speaking in Finland") expected_iobs = ["B", "O", "O", "O", "O", "O", "O"] @@ -315,10 +295,9 @@ def test_block_ner(): """ Test functionality for blocking tokens so they can't be in a named entity """ # block "Antti L Korhonen" from being a named entity nlp = English() - nlp.add_pipe(BlockerComponent1(2, 5)) - untrained_ner = nlp.create_pipe("ner") + nlp.add_pipe("blocker", config={"start": 2, "end": 5}) + untrained_ner = nlp.add_pipe("ner") untrained_ner.add_label("MY_LABEL") - nlp.add_pipe(untrained_ner, name="uner") nlp.begin_training() doc = nlp("This is Antti L Korhonen speaking in Finland") expected_iobs = ["O", "O", "B", "B", "B", "O", "O", "O"] @@ -330,13 +309,12 @@ def test_block_ner(): def test_overfitting_IO(): # Simple test to try and quickly overfit the NER component - ensuring the ML models work correctly nlp = English() - ner = nlp.create_pipe("ner") + ner = nlp.add_pipe("ner") train_examples = [] for text, annotations in TRAIN_DATA: train_examples.append(Example.from_dict(nlp.make_doc(text), annotations)) for ent in annotations.get("entities"): ner.add_label(ent[2]) - nlp.add_pipe(ner) optimizer = nlp.begin_training() for i in range(50): @@ -367,8 +345,7 @@ def test_ner_warns_no_lookups(): nlp = Language() nlp.vocab.lookups = Lookups() assert not len(nlp.vocab.lookups) - ner = nlp.create_pipe("ner") - nlp.add_pipe(ner) + nlp.add_pipe("ner") with pytest.warns(UserWarning): nlp.begin_training() nlp.vocab.lookups.add_table("lexeme_norm") @@ -378,12 +355,12 @@ def test_ner_warns_no_lookups(): assert not record.list +@Language.factory("blocker") class BlockerComponent1: - name = "my_blocker" - - def __init__(self, start, end): + def __init__(self, nlp, start, end, name="my_blocker"): self.start = start self.end = end + self.name = name def __call__(self, doc): doc.ents = [(0, self.start, self.end)] diff --git a/spacy/tests/parser/test_neural_parser.py b/spacy/tests/parser/test_neural_parser.py index b75d2f9e5..a53a0f37a 100644 --- a/spacy/tests/parser/test_neural_parser.py +++ b/spacy/tests/parser/test_neural_parser.py @@ -1,12 +1,14 @@ import pytest +from spacy import registry from spacy.gold import Example -from spacy.pipeline.defaults import default_parser, default_tok2vec from spacy.vocab import Vocab from spacy.syntax.arc_eager import ArcEager from spacy.syntax.nn_parser import Parser from spacy.tokens.doc import Doc from thinc.api import Model +from spacy.pipeline.tok2vec import DEFAULT_TOK2VEC_MODEL +from spacy.pipeline.dep_parser import DEFAULT_PARSER_MODEL @pytest.fixture @@ -22,7 +24,7 @@ def arc_eager(vocab): @pytest.fixture def tok2vec(): - tok2vec = default_tok2vec() + tok2vec = registry.make_from_config({"model": DEFAULT_TOK2VEC_MODEL}, validate=True)["model"] tok2vec.initialize() return tok2vec @@ -32,15 +34,15 @@ def parser(vocab, arc_eager): config = { "learn_tokens": False, "min_action_freq": 30, - "beam_width": 1, - "beam_update_prob": 1.0, + "update_with_oracle_cut_size": 100, } - return Parser(vocab, model=default_parser(), moves=arc_eager, **config) + model = registry.make_from_config({"model": DEFAULT_PARSER_MODEL}, validate=True)["model"] + return Parser(vocab, model, moves=arc_eager, **config) @pytest.fixture def model(arc_eager, tok2vec, vocab): - model = default_parser() + model = registry.make_from_config({"model": DEFAULT_PARSER_MODEL}, validate=True)["model"] model.attrs["resize_output"](model, arc_eager.n_moves) model.initialize() return model @@ -61,7 +63,13 @@ def test_can_init_nn_parser(parser): def test_build_model(parser, vocab): - parser.model = Parser(vocab, model=default_parser(), moves=parser.moves).model + config = { + "learn_tokens": False, + "min_action_freq": 0, + "update_with_oracle_cut_size": 100, + } + model = registry.make_from_config({"model": DEFAULT_PARSER_MODEL}, validate=True)["model"] + parser.model = Parser(vocab, model=model, moves=parser.moves, **config).model assert parser.model is not None diff --git a/spacy/tests/parser/test_parse.py b/spacy/tests/parser/test_parse.py index b1c74e2e9..3f3fabbb8 100644 --- a/spacy/tests/parser/test_parse.py +++ b/spacy/tests/parser/test_parse.py @@ -187,13 +187,12 @@ def test_parser_set_sent_starts(en_vocab): def test_overfitting_IO(): # Simple test to try and quickly overfit the dependency parser - ensuring the ML models work correctly nlp = English() - parser = nlp.create_pipe("parser") + parser = nlp.add_pipe("parser") train_examples = [] for text, annotations in TRAIN_DATA: train_examples.append(Example.from_dict(nlp.make_doc(text), annotations)) for dep in annotations.get("deps", []): parser.add_label(dep) - nlp.add_pipe(parser) optimizer = nlp.begin_training() for i in range(100): diff --git a/spacy/tests/parser/test_preset_sbd.py b/spacy/tests/parser/test_preset_sbd.py index ffd0c5df4..747203c2f 100644 --- a/spacy/tests/parser/test_preset_sbd.py +++ b/spacy/tests/parser/test_preset_sbd.py @@ -2,9 +2,9 @@ import pytest from thinc.api import Adam from spacy.attrs import NORM from spacy.vocab import Vocab - +from spacy import registry from spacy.gold import Example -from spacy.pipeline.defaults import default_parser +from spacy.pipeline.dep_parser import DEFAULT_PARSER_MODEL from spacy.tokens import Doc from spacy.pipeline import DependencyParser @@ -19,10 +19,10 @@ def parser(vocab): config = { "learn_tokens": False, "min_action_freq": 30, - "beam_width": 1, - "beam_update_prob": 1.0, + "update_with_oracle_cut_size": 100, } - parser = DependencyParser(vocab, default_parser(), **config) + model = registry.make_from_config({"model": DEFAULT_PARSER_MODEL}, validate=True)["model"] + parser = DependencyParser(vocab, model, **config) parser.cfg["token_vector_width"] = 4 parser.cfg["hidden_width"] = 32 # parser.add_label('right') diff --git a/spacy/tests/pipeline/test_analysis.py b/spacy/tests/pipeline/test_analysis.py index 85f88c22c..4e1407707 100644 --- a/spacy/tests/pipeline/test_analysis.py +++ b/spacy/tests/pipeline/test_analysis.py @@ -1,99 +1,55 @@ import spacy.language -from spacy.language import Language, component +from spacy.language import Language from spacy.pipe_analysis import print_summary, validate_attrs from spacy.pipe_analysis import get_assigns_for_attr, get_requires_for_attr from spacy.pipe_analysis import count_pipeline_interdependencies -from mock import Mock, ANY +from mock import Mock import pytest -def test_component_decorator_function(): - @component(name="test") - def test_component(doc): - """docstring""" - return doc - - assert test_component.name == "test" - assert test_component.__doc__ == "docstring" - assert test_component("foo") == "foo" - - -def test_component_decorator_class(): - @component(name="test") - class TestComponent: - """docstring1""" - - foo = "bar" - - def __call__(self, doc): - """docstring2""" - return doc - - def custom(self, x): - """docstring3""" - return x - - assert TestComponent.name == "test" - assert TestComponent.foo == "bar" - assert hasattr(TestComponent, "custom") - test_component = TestComponent() - assert test_component.foo == "bar" - assert test_component("foo") == "foo" - assert hasattr(test_component, "custom") - assert test_component.custom("bar") == "bar" - assert TestComponent.__doc__ == "docstring1" - assert TestComponent.__call__.__doc__ == "docstring2" - assert TestComponent.custom.__doc__ == "docstring3" - assert test_component.__doc__ == "docstring1" - assert test_component.__call__.__doc__ == "docstring2" - assert test_component.custom.__doc__ == "docstring3" - - def test_component_decorator_assigns(): spacy.language.ENABLE_PIPELINE_ANALYSIS = True - @component("c1", assigns=["token.tag", "doc.tensor"]) + @Language.component("c1", assigns=["token.tag", "doc.tensor"]) def test_component1(doc): return doc - @component( + @Language.component( "c2", requires=["token.tag", "token.pos"], assigns=["token.lemma", "doc.tensor"] ) def test_component2(doc): return doc - @component("c3", requires=["token.lemma"], assigns=["token._.custom_lemma"]) + @Language.component( + "c3", requires=["token.lemma"], assigns=["token._.custom_lemma"] + ) def test_component3(doc): return doc - assert "c1" in Language.factories - assert "c2" in Language.factories - assert "c3" in Language.factories + assert Language.has_factory("c1") + assert Language.has_factory("c2") + assert Language.has_factory("c3") nlp = Language() - nlp.add_pipe(test_component1) + nlp.add_pipe("c1") with pytest.warns(UserWarning): - nlp.add_pipe(test_component2) - nlp.add_pipe(test_component3) - assigns_tensor = get_assigns_for_attr(nlp.pipeline, "doc.tensor") - assert [name for name, _ in assigns_tensor] == ["c1", "c2"] - test_component4 = nlp.create_pipe("c1") - assert test_component4.name == "c1" - assert test_component4.factory == "c1" - nlp.add_pipe(test_component4, name="c4") + nlp.add_pipe("c2") + nlp.add_pipe("c3") + assert get_assigns_for_attr(nlp, "doc.tensor") == ["c1", "c2"] + nlp.add_pipe("c1", name="c4") + test_component4_meta = nlp.get_pipe_meta("c1") + assert test_component4_meta.factory == "c1" assert nlp.pipe_names == ["c1", "c2", "c3", "c4"] - assert "c4" not in Language.factories + assert not Language.has_factory("c4") assert nlp.pipe_factories["c1"] == "c1" assert nlp.pipe_factories["c4"] == "c1" - assigns_tensor = get_assigns_for_attr(nlp.pipeline, "doc.tensor") - assert [name for name, _ in assigns_tensor] == ["c1", "c2", "c4"] - requires_pos = get_requires_for_attr(nlp.pipeline, "token.pos") - assert [name for name, _ in requires_pos] == ["c2"] + assert get_assigns_for_attr(nlp, "doc.tensor") == ["c1", "c2", "c4"] + assert get_requires_for_attr(nlp, "token.pos") == ["c2"] assert print_summary(nlp, no_print=True) assert nlp("hello world") -def test_component_factories_from_nlp(): +def test_component_factories_class_func(): """Test that class components can implement a from_nlp classmethod that gives them access to the nlp object and config via the factory.""" @@ -103,17 +59,16 @@ def test_component_factories_from_nlp(): mock = Mock() mock.return_value = TestComponent5() - TestComponent5.from_nlp = classmethod(mock) - TestComponent5 = component("c5")(TestComponent5) - assert "c5" in Language.factories + def test_componen5_factory(nlp, foo: str = "bar", name="c5"): + return mock(nlp, foo=foo) + + Language.factory("c5", func=test_componen5_factory) + assert Language.has_factory("c5") nlp = Language() - pipe = nlp.create_pipe("c5", config={"foo": "bar"}) - nlp.add_pipe(pipe) + nlp.add_pipe("c5", config={"foo": "bar"}) assert nlp("hello world") - # The first argument here is the class itself, so we're accepting any here - # The model will be initialized to None by the factory - mock.assert_called_once_with(ANY, nlp, None, foo="bar") + mock.assert_called_once_with(nlp, foo="bar") def test_analysis_validate_attrs_valid(): @@ -147,34 +102,36 @@ def test_analysis_validate_attrs_remove_pipe(): """Test that attributes are validated correctly on remove.""" spacy.language.ENABLE_PIPELINE_ANALYSIS = True - @component("c1", assigns=["token.tag"]) + @Language.component("pipe_analysis_c6", assigns=["token.tag"]) def c1(doc): return doc - @component("c2", requires=["token.pos"]) + @Language.component("pipe_analysis_c7", requires=["token.pos"]) def c2(doc): return doc nlp = Language() - nlp.add_pipe(c1) + nlp.add_pipe("pipe_analysis_c6") with pytest.warns(UserWarning): - nlp.add_pipe(c2) + nlp.add_pipe("pipe_analysis_c7") with pytest.warns(None) as record: - nlp.remove_pipe("c2") + nlp.remove_pipe("pipe_analysis_c7") assert not record.list def test_pipe_interdependencies(): - class Fancifier: - name = "fancifier" - assigns = ("doc._.fancy",) - requires = tuple() + prefix = "test_pipe_interdependencies" - class FancyNeeder: - name = "needer" - assigns = tuple() - requires = ("doc._.fancy",) + @Language.component(f"{prefix}.fancifier", assigns=("doc._.fancy",)) + def fancifier(doc): + return doc - pipeline = [("fancifier", Fancifier()), ("needer", FancyNeeder())] - counts = count_pipeline_interdependencies(pipeline) + @Language.component(f"{prefix}.needer", requires=("doc._.fancy",)) + def needer(doc): + return doc + + nlp = Language() + nlp.add_pipe(f"{prefix}.fancifier") + nlp.add_pipe(f"{prefix}.needer") + counts = count_pipeline_interdependencies(nlp) assert counts == [1, 0] diff --git a/spacy/tests/pipeline/test_entity_linker.py b/spacy/tests/pipeline/test_entity_linker.py index f91cc6f70..4002eafe3 100644 --- a/spacy/tests/pipeline/test_entity_linker.py +++ b/spacy/tests/pipeline/test_entity_linker.py @@ -2,10 +2,9 @@ import pytest from spacy.kb import KnowledgeBase -from spacy import util +from spacy import util, registry from spacy.gold import Example from spacy.lang.en import English -from spacy.pipeline import EntityRuler from spacy.tests.util import make_tempdir from spacy.tokens import Span @@ -182,34 +181,31 @@ def test_append_invalid_alias(nlp): def test_preserving_links_asdoc(nlp): """Test that Span.as_doc preserves the existing entity links""" - mykb = KnowledgeBase(nlp.vocab, entity_vector_length=1) - # adding entities - mykb.add_entity(entity="Q1", freq=19, entity_vector=[1]) - mykb.add_entity(entity="Q2", freq=8, entity_vector=[1]) - - # adding aliases - mykb.add_alias(alias="Boston", entities=["Q1"], probabilities=[0.7]) - mykb.add_alias(alias="Denver", entities=["Q2"], probabilities=[0.6]) + @registry.assets.register("myLocationsKB.v1") + def dummy_kb() -> KnowledgeBase: + mykb = KnowledgeBase(nlp.vocab, entity_vector_length=1) + # adding entities + mykb.add_entity(entity="Q1", freq=19, entity_vector=[1]) + mykb.add_entity(entity="Q2", freq=8, entity_vector=[1]) + # adding aliases + mykb.add_alias(alias="Boston", entities=["Q1"], probabilities=[0.7]) + mykb.add_alias(alias="Denver", entities=["Q2"], probabilities=[0.6]) + return mykb # set up pipeline with NER (Entity Ruler) and NEL (prior probability only, model not trained) - sentencizer = nlp.create_pipe("sentencizer") - nlp.add_pipe(sentencizer) - - ruler = EntityRuler(nlp) + nlp.add_pipe("sentencizer") patterns = [ {"label": "GPE", "pattern": "Boston"}, {"label": "GPE", "pattern": "Denver"}, ] + ruler = nlp.add_pipe("entity_ruler") ruler.add_patterns(patterns) - nlp.add_pipe(ruler) - - cfg = {"kb": mykb, "incl_prior": False} - el_pipe = nlp.create_pipe(name="entity_linker", config=cfg) + el_config = {"kb": {"@assets": "myLocationsKB.v1"}, "incl_prior": False} + el_pipe = nlp.add_pipe("entity_linker", config=el_config, last=True) el_pipe.begin_training() el_pipe.incl_context = False el_pipe.incl_prior = True - nlp.add_pipe(el_pipe, last=True) # test whether the entity links are preserved by the `as_doc()` function text = "She lives in Boston. He lives in Denver." @@ -273,15 +269,14 @@ GOLD_entities = ["Q2146908", "Q7381115", "Q7381115", "Q2146908"] def test_overfitting_IO(): # Simple test to try and quickly overfit the NEL component - ensuring the ML models work correctly nlp = English() - nlp.add_pipe(nlp.create_pipe("sentencizer")) + nlp.add_pipe("sentencizer") # Add a custom component to recognize "Russ Cochran" as an entity for the example training data - ruler = EntityRuler(nlp) patterns = [ {"label": "PERSON", "pattern": [{"LOWER": "russ"}, {"LOWER": "cochran"}]} ] + ruler = nlp.add_pipe("entity_ruler") ruler.add_patterns(patterns) - nlp.add_pipe(ruler) # Convert the texts to docs to make sure we have doc.ents set for the training examples train_examples = [] @@ -289,21 +284,25 @@ def test_overfitting_IO(): doc = nlp(text) train_examples.append(Example.from_dict(doc, annotation)) - # create artificial KB - assign same prior weight to the two russ cochran's - # Q2146908 (Russ Cochran): American golfer - # Q7381115 (Russ Cochran): publisher - mykb = KnowledgeBase(nlp.vocab, entity_vector_length=3) - mykb.add_entity(entity="Q2146908", freq=12, entity_vector=[6, -4, 3]) - mykb.add_entity(entity="Q7381115", freq=12, entity_vector=[9, 1, -7]) - mykb.add_alias( - alias="Russ Cochran", - entities=["Q2146908", "Q7381115"], - probabilities=[0.5, 0.5], - ) + @registry.assets.register("myOverfittingKB.v1") + def dummy_kb() -> KnowledgeBase: + # create artificial KB - assign same prior weight to the two russ cochran's + # Q2146908 (Russ Cochran): American golfer + # Q7381115 (Russ Cochran): publisher + mykb = KnowledgeBase(nlp.vocab, entity_vector_length=3) + mykb.add_entity(entity="Q2146908", freq=12, entity_vector=[6, -4, 3]) + mykb.add_entity(entity="Q7381115", freq=12, entity_vector=[9, 1, -7]) + mykb.add_alias( + alias="Russ Cochran", + entities=["Q2146908", "Q7381115"], + probabilities=[0.5, 0.5], + ) + return mykb # Create the Entity Linker component and add it to the pipeline - entity_linker = nlp.create_pipe("entity_linker", config={"kb": mykb}) - nlp.add_pipe(entity_linker, last=True) + nlp.add_pipe( + "entity_linker", config={"kb": {"@assets": "myOverfittingKB.v1"}}, last=True + ) # train the NEL pipe optimizer = nlp.begin_training() @@ -324,6 +323,7 @@ def test_overfitting_IO(): with make_tempdir() as tmp_dir: nlp.to_disk(tmp_dir) nlp2 = util.load_model_from_path(tmp_dir) + assert nlp2.pipe_names == nlp.pipe_names predictions = [] for text, annotation in TRAIN_DATA: doc2 = nlp2(text) diff --git a/spacy/tests/pipeline/test_entity_ruler.py b/spacy/tests/pipeline/test_entity_ruler.py index b04569e22..e4e1631b1 100644 --- a/spacy/tests/pipeline/test_entity_ruler.py +++ b/spacy/tests/pipeline/test_entity_ruler.py @@ -22,13 +22,10 @@ def patterns(): ] -@pytest.fixture -def add_ent(): - def add_ent_component(doc): - doc.ents = [Span(doc, 0, 3, label=doc.vocab.strings["ORG"])] - return doc - - return add_ent_component +@Language.component("add_ent") +def add_ent_component(doc): + doc.ents = [Span(doc, 0, 3, label="ORG")] + return doc def test_entity_ruler_init(nlp, patterns): @@ -37,27 +34,28 @@ def test_entity_ruler_init(nlp, patterns): assert len(ruler.labels) == 4 assert "HELLO" in ruler assert "BYE" in ruler - nlp.add_pipe(ruler) + ruler = nlp.add_pipe("entity_ruler") + ruler.add_patterns(patterns) doc = nlp("hello world bye bye") assert len(doc.ents) == 2 assert doc.ents[0].label_ == "HELLO" assert doc.ents[1].label_ == "BYE" -def test_entity_ruler_existing(nlp, patterns, add_ent): - ruler = EntityRuler(nlp, patterns=patterns) - nlp.add_pipe(add_ent) - nlp.add_pipe(ruler) +def test_entity_ruler_existing(nlp, patterns): + ruler = nlp.add_pipe("entity_ruler") + ruler.add_patterns(patterns) + nlp.add_pipe("add_ent", before="entity_ruler") doc = nlp("OH HELLO WORLD bye bye") assert len(doc.ents) == 2 assert doc.ents[0].label_ == "ORG" assert doc.ents[1].label_ == "BYE" -def test_entity_ruler_existing_overwrite(nlp, patterns, add_ent): - ruler = EntityRuler(nlp, patterns=patterns, overwrite_ents=True) - nlp.add_pipe(add_ent) - nlp.add_pipe(ruler) +def test_entity_ruler_existing_overwrite(nlp, patterns): + ruler = nlp.add_pipe("entity_ruler", config={"overwrite_ents": True}) + ruler.add_patterns(patterns) + nlp.add_pipe("add_ent", before="entity_ruler") doc = nlp("OH HELLO WORLD bye bye") assert len(doc.ents) == 2 assert doc.ents[0].label_ == "HELLO" @@ -65,10 +63,10 @@ def test_entity_ruler_existing_overwrite(nlp, patterns, add_ent): assert doc.ents[1].label_ == "BYE" -def test_entity_ruler_existing_complex(nlp, patterns, add_ent): - ruler = EntityRuler(nlp, patterns=patterns, overwrite_ents=True) - nlp.add_pipe(add_ent) - nlp.add_pipe(ruler) +def test_entity_ruler_existing_complex(nlp, patterns): + ruler = nlp.add_pipe("entity_ruler", config={"overwrite_ents": True}) + ruler.add_patterns(patterns) + nlp.add_pipe("add_ent", before="entity_ruler") doc = nlp("foo foo bye bye") assert len(doc.ents) == 2 assert doc.ents[0].label_ == "COMPLEX" @@ -78,8 +76,8 @@ def test_entity_ruler_existing_complex(nlp, patterns, add_ent): def test_entity_ruler_entity_id(nlp, patterns): - ruler = EntityRuler(nlp, patterns=patterns, overwrite_ents=True) - nlp.add_pipe(ruler) + ruler = nlp.add_pipe("entity_ruler", config={"overwrite_ents": True}) + ruler.add_patterns(patterns) doc = nlp("Apple is a technology company") assert len(doc.ents) == 1 assert doc.ents[0].label_ == "TECH_ORG" @@ -87,9 +85,10 @@ def test_entity_ruler_entity_id(nlp, patterns): def test_entity_ruler_cfg_ent_id_sep(nlp, patterns): - ruler = EntityRuler(nlp, patterns=patterns, overwrite_ents=True, ent_id_sep="**") + config = {"overwrite_ents": True, "ent_id_sep": "**"} + ruler = nlp.add_pipe("entity_ruler", config=config) + ruler.add_patterns(patterns) assert "TECH_ORG**a1" in ruler.phrase_patterns - nlp.add_pipe(ruler) doc = nlp("Apple is a technology company") assert len(doc.ents) == 1 assert doc.ents[0].label_ == "TECH_ORG" diff --git a/spacy/tests/pipeline/test_factories.py b/spacy/tests/pipeline/test_factories.py deleted file mode 100644 index 0a9a4d3c9..000000000 --- a/spacy/tests/pipeline/test_factories.py +++ /dev/null @@ -1,47 +0,0 @@ -import pytest -from spacy.language import Language -from spacy.tokens import Span - -from ..util import get_doc - - -@pytest.fixture -def doc(en_tokenizer): - text = "I like New York in Autumn." - heads = [1, 0, 1, -2, -3, -1, -5] - tags = ["PRP", "IN", "NNP", "NNP", "IN", "NNP", "."] - pos = ["PRON", "VERB", "PROPN", "PROPN", "ADP", "PROPN", "PUNCT"] - deps = ["ROOT", "prep", "compound", "pobj", "prep", "pobj", "punct"] - tokens = en_tokenizer(text) - doc = get_doc( - tokens.vocab, - words=[t.text for t in tokens], - heads=heads, - tags=tags, - pos=pos, - deps=deps, - ) - doc.ents = [Span(doc, 2, 4, doc.vocab.strings["GPE"])] - doc.is_parsed = True - doc.is_tagged = True - return doc - - -def test_factories_merge_noun_chunks(doc): - assert len(doc) == 7 - nlp = Language() - merge_noun_chunks = nlp.create_pipe("merge_noun_chunks") - merge_noun_chunks(doc) - assert len(doc) == 6 - assert doc[2].text == "New York" - - -def test_factories_merge_ents(doc): - assert len(doc) == 7 - assert len(list(doc.ents)) == 1 - nlp = Language() - merge_entities = nlp.create_pipe("merge_entities") - merge_entities(doc) - assert len(doc) == 6 - assert len(list(doc.ents)) == 1 - assert doc[2].text == "New York" diff --git a/spacy/tests/pipeline/test_functions.py b/spacy/tests/pipeline/test_functions.py index ca983267f..0ec8a5ec2 100644 --- a/spacy/tests/pipeline/test_functions.py +++ b/spacy/tests/pipeline/test_functions.py @@ -1,5 +1,8 @@ import pytest from spacy.pipeline.functions import merge_subtokens +from spacy.language import Language +from spacy.tokens import Span + from ..util import get_doc @@ -15,6 +18,28 @@ def doc(en_tokenizer): return get_doc(tokens.vocab, words=[t.text for t in tokens], heads=heads, deps=deps) +@pytest.fixture +def doc2(en_tokenizer): + text = "I like New York in Autumn." + heads = [1, 0, 1, -2, -3, -1, -5] + tags = ["PRP", "IN", "NNP", "NNP", "IN", "NNP", "."] + pos = ["PRON", "VERB", "PROPN", "PROPN", "ADP", "PROPN", "PUNCT"] + deps = ["ROOT", "prep", "compound", "pobj", "prep", "pobj", "punct"] + tokens = en_tokenizer(text) + doc = get_doc( + tokens.vocab, + words=[t.text for t in tokens], + heads=heads, + tags=tags, + pos=pos, + deps=deps, + ) + doc.ents = [Span(doc, 2, 4, doc.vocab.strings["GPE"])] + doc.is_parsed = True + doc.is_tagged = True + return doc + + def test_merge_subtokens(doc): doc = merge_subtokens(doc) # get_doc() doesn't set spaces, so the result is "And a third ." @@ -29,3 +54,23 @@ def test_merge_subtokens(doc): ".", "And a third .", ] + + +def test_factories_merge_noun_chunks(doc2): + assert len(doc2) == 7 + nlp = Language() + merge_noun_chunks = nlp.create_pipe("merge_noun_chunks") + merge_noun_chunks(doc2) + assert len(doc2) == 6 + assert doc2[2].text == "New York" + + +def test_factories_merge_ents(doc2): + assert len(doc2) == 7 + assert len(list(doc2.ents)) == 1 + nlp = Language() + merge_entities = nlp.create_pipe("merge_entities") + merge_entities(doc2) + assert len(doc2) == 6 + assert len(list(doc2.ents)) == 1 + assert doc2[2].text == "New York" diff --git a/spacy/tests/pipeline/test_morphologizer.py b/spacy/tests/pipeline/test_morphologizer.py index 757c9214c..501c00f84 100644 --- a/spacy/tests/pipeline/test_morphologizer.py +++ b/spacy/tests/pipeline/test_morphologizer.py @@ -10,10 +10,10 @@ from spacy.morphology import Morphology def test_label_types(): nlp = Language() - nlp.add_pipe(nlp.create_pipe("morphologizer")) - nlp.get_pipe("morphologizer").add_label("Feat=A") + morphologizer = nlp.add_pipe("morphologizer") + morphologizer.add_label("Feat=A") with pytest.raises(ValueError): - nlp.get_pipe("morphologizer").add_label(9) + morphologizer.add_label(9) TRAIN_DATA = [ @@ -25,28 +25,26 @@ TRAIN_DATA = [ }, ), # test combinations of morph+POS - ( - "Eat blue ham", - {"morphs": ["Feat=V", "", ""], "pos": ["", "ADJ", ""]}, - ), + ("Eat blue ham", {"morphs": ["Feat=V", "", ""], "pos": ["", "ADJ", ""]},), ] def test_overfitting_IO(): # Simple test to try and quickly overfit the morphologizer - ensuring the ML models work correctly nlp = English() - morphologizer = nlp.create_pipe("morphologizer") + morphologizer = nlp.add_pipe("morphologizer") train_examples = [] for inst in TRAIN_DATA: train_examples.append(Example.from_dict(nlp.make_doc(inst[0]), inst[1])) for morph, pos in zip(inst[1]["morphs"], inst[1]["pos"]): if morph and pos: - morphologizer.add_label(morph + Morphology.FEATURE_SEP + "POS" + Morphology.FIELD_SEP + pos) + morphologizer.add_label( + morph + Morphology.FEATURE_SEP + "POS" + Morphology.FIELD_SEP + pos + ) elif pos: morphologizer.add_label("POS" + Morphology.FIELD_SEP + pos) elif morph: morphologizer.add_label(morph) - nlp.add_pipe(morphologizer) optimizer = nlp.begin_training() for i in range(50): diff --git a/spacy/tests/pipeline/test_pipe_factories.py b/spacy/tests/pipeline/test_pipe_factories.py new file mode 100644 index 000000000..db090fdf2 --- /dev/null +++ b/spacy/tests/pipeline/test_pipe_factories.py @@ -0,0 +1,330 @@ +import pytest +from spacy.language import Language +from spacy.lang.en import English +from spacy.lang.de import German +from spacy.tokens import Doc +from spacy.util import registry, SimpleFrozenDict +from thinc.api import Model, Linear +from thinc.config import ConfigValidationError +from pydantic import StrictInt, StrictStr + + +def test_pipe_function_component(): + name = "test_component" + + @Language.component(name) + def component(doc: Doc) -> Doc: + return doc + + assert name in registry.factories + nlp = Language() + with pytest.raises(ValueError): + nlp.add_pipe(component) + nlp.add_pipe(name) + assert name in nlp.pipe_names + assert nlp.pipe_factories[name] == name + assert Language.get_factory_meta(name) + assert nlp.get_pipe_meta(name) + pipe = nlp.get_pipe(name) + assert pipe == component + pipe = nlp.create_pipe(name) + assert pipe == component + + +def test_pipe_class_component_init(): + name1 = "test_class_component1" + name2 = "test_class_component2" + + @Language.factory(name1) + class Component1: + def __init__(self, nlp: Language, name: str): + self.nlp = nlp + + def __call__(self, doc: Doc) -> Doc: + return doc + + class Component2: + def __init__(self, nlp: Language, name: str): + self.nlp = nlp + + def __call__(self, doc: Doc) -> Doc: + return doc + + @Language.factory(name2) + def factory(nlp: Language, name=name2): + return Component2(nlp, name) + + nlp = Language() + for name, Component in [(name1, Component1), (name2, Component2)]: + assert name in registry.factories + with pytest.raises(ValueError): + nlp.add_pipe(Component(nlp, name)) + nlp.add_pipe(name) + assert name in nlp.pipe_names + assert nlp.pipe_factories[name] == name + assert Language.get_factory_meta(name) + assert nlp.get_pipe_meta(name) + pipe = nlp.get_pipe(name) + assert isinstance(pipe, Component) + assert isinstance(pipe.nlp, Language) + pipe = nlp.create_pipe(name) + assert isinstance(pipe, Component) + assert isinstance(pipe.nlp, Language) + + +def test_pipe_class_component_config(): + name = "test_class_component_config" + + @Language.factory(name) + class Component: + def __init__( + self, nlp: Language, name: str, value1: StrictInt, value2: StrictStr + ): + self.nlp = nlp + self.value1 = value1 + self.value2 = value2 + self.is_base = True + + def __call__(self, doc: Doc) -> Doc: + return doc + + @English.factory(name) + class ComponentEN: + def __init__( + self, nlp: Language, name: str, value1: StrictInt, value2: StrictStr + ): + self.nlp = nlp + self.value1 = value1 + self.value2 = value2 + self.is_base = False + + def __call__(self, doc: Doc) -> Doc: + return doc + + nlp = Language() + with pytest.raises(ConfigValidationError): # no config provided + nlp.add_pipe(name) + with pytest.raises(ConfigValidationError): # invalid config + nlp.add_pipe(name, config={"value1": "10", "value2": "hello"}) + nlp.add_pipe(name, config={"value1": 10, "value2": "hello"}) + pipe = nlp.get_pipe(name) + assert isinstance(pipe.nlp, Language) + assert pipe.value1 == 10 + assert pipe.value2 == "hello" + assert pipe.is_base is True + + nlp_en = English() + with pytest.raises(ConfigValidationError): # invalid config + nlp_en.add_pipe(name, config={"value1": "10", "value2": "hello"}) + nlp_en.add_pipe(name, config={"value1": 10, "value2": "hello"}) + pipe = nlp_en.get_pipe(name) + assert isinstance(pipe.nlp, English) + assert pipe.value1 == 10 + assert pipe.value2 == "hello" + assert pipe.is_base is False + + +def test_pipe_class_component_defaults(): + name = "test_class_component_defaults" + + @Language.factory(name) + class Component: + def __init__( + self, + nlp: Language, + name: str, + value1: StrictInt = 10, + value2: StrictStr = "hello", + ): + self.nlp = nlp + self.value1 = value1 + self.value2 = value2 + + def __call__(self, doc: Doc) -> Doc: + return doc + + nlp = Language() + nlp.add_pipe(name) + pipe = nlp.get_pipe(name) + assert isinstance(pipe.nlp, Language) + assert pipe.value1 == 10 + assert pipe.value2 == "hello" + + +def test_pipe_class_component_model(): + name = "test_class_component_model" + default_config = { + "model": { + "@architectures": "spacy.TextCat.v1", + "exclusive_classes": False, + "pretrained_vectors": None, + "width": 64, + "embed_size": 2000, + "window_size": 1, + "conv_depth": 2, + "ngram_size": 1, + "dropout": None, + }, + "value1": 10, + } + + @Language.factory(name, default_config=default_config) + class Component: + def __init__(self, nlp: Language, model: Model, name: str, value1: StrictInt): + self.nlp = nlp + self.model = model + self.value1 = value1 + self.name = name + + def __call__(self, doc: Doc) -> Doc: + return doc + + nlp = Language() + nlp.add_pipe(name) + pipe = nlp.get_pipe(name) + assert isinstance(pipe.nlp, Language) + assert pipe.value1 == 10 + assert isinstance(pipe.model, Model) + + +def test_pipe_class_component_model_custom(): + name = "test_class_component_model_custom" + arch = f"{name}.arch" + default_config = {"value1": 1, "model": {"@architectures": arch, "nO": 0, "nI": 0}} + + @Language.factory(name, default_config=default_config) + class Component: + def __init__( + self, nlp: Language, model: Model, name: str, value1: StrictInt = 10 + ): + self.nlp = nlp + self.model = model + self.value1 = value1 + self.name = name + + def __call__(self, doc: Doc) -> Doc: + return doc + + @registry.architectures(arch) + def make_custom_arch(nO: StrictInt, nI: StrictInt): + return Linear(nO, nI) + + nlp = Language() + config = {"value1": 20, "model": {"@architectures": arch, "nO": 1, "nI": 2}} + nlp.add_pipe(name, config=config) + pipe = nlp.get_pipe(name) + assert isinstance(pipe.nlp, Language) + assert pipe.value1 == 20 + assert isinstance(pipe.model, Model) + assert pipe.model.name == "linear" + + nlp = Language() + with pytest.raises(ConfigValidationError): + config = {"value1": "20", "model": {"@architectures": arch, "nO": 1, "nI": 2}} + nlp.add_pipe(name, config=config) + with pytest.raises(ConfigValidationError): + config = {"value1": 20, "model": {"@architectures": arch, "nO": 1.0, "nI": 2.0}} + nlp.add_pipe(name, config=config) + + +def test_pipe_factories_wrong_formats(): + with pytest.raises(ValueError): + # Decorator is not called + @Language.component + def component(foo: int, bar: str): + ... + + with pytest.raises(ValueError): + # Decorator is not called + @Language.factory + def factory1(foo: int, bar: str): + ... + + with pytest.raises(ValueError): + # Factory function is missing "nlp" and "name" arguments + @Language.factory("test_pipe_factories_missing_args") + def factory2(foo: int, bar: str): + ... + + +def test_pipe_factory_meta_config_cleanup(): + """Test that component-specific meta and config entries are represented + correctly and cleaned up when pipes are removed, replaced or renamed.""" + nlp = Language() + nlp.add_pipe("ner", name="ner_component") + nlp.add_pipe("textcat") + assert nlp.get_factory_meta("ner") + assert nlp.get_pipe_meta("ner_component") + assert nlp.get_pipe_config("ner_component") + assert nlp.get_factory_meta("textcat") + assert nlp.get_pipe_meta("textcat") + assert nlp.get_pipe_config("textcat") + nlp.rename_pipe("textcat", "tc") + assert nlp.get_pipe_meta("tc") + assert nlp.get_pipe_config("tc") + with pytest.raises(ValueError): + nlp.remove_pipe("ner") + nlp.remove_pipe("ner_component") + assert "ner_component" not in nlp._pipe_meta + assert "ner_component" not in nlp._pipe_configs + with pytest.raises(ValueError): + nlp.replace_pipe("textcat", "parser") + nlp.replace_pipe("tc", "parser") + assert nlp.get_factory_meta("parser") + assert nlp.get_pipe_meta("tc").factory == "parser" + + +def test_pipe_factories_empty_dict_default(): + """Test that default config values can be empty dicts and that no config + validation error is raised.""" + # TODO: fix this + name = "test_pipe_factories_empty_dict_default" + + @Language.factory(name, default_config={"foo": {}}) + def factory(nlp: Language, name: str, foo: dict): + ... + + nlp = Language() + nlp.create_pipe(name) + + +def test_pipe_factories_language_specific(): + """Test that language sub-classes can have their own factories, with + fallbacks to the base factories.""" + name1 = "specific_component1" + name2 = "specific_component2" + Language.component(name1, func=lambda: "base") + English.component(name1, func=lambda: "en") + German.component(name2, func=lambda: "de") + + assert Language.has_factory(name1) + assert not Language.has_factory(name2) + assert English.has_factory(name1) + assert not English.has_factory(name2) + assert German.has_factory(name1) + assert German.has_factory(name2) + + nlp = Language() + assert nlp.create_pipe(name1)() == "base" + with pytest.raises(ValueError): + nlp.create_pipe(name2) + nlp_en = English() + assert nlp_en.create_pipe(name1)() == "en" + with pytest.raises(ValueError): + nlp_en.create_pipe(name2) + nlp_de = German() + assert nlp_de.create_pipe(name1)() == "base" + assert nlp_de.create_pipe(name2)() == "de" + + +def test_language_factories_invalid(): + """Test that assigning directly to Language.factories is now invalid and + raises a custom error.""" + assert isinstance(Language.factories, SimpleFrozenDict) + with pytest.raises(NotImplementedError): + Language.factories["foo"] = "bar" + nlp = Language() + assert isinstance(nlp.factories, SimpleFrozenDict) + assert len(nlp.factories) + with pytest.raises(NotImplementedError): + nlp.factories["foo"] = "bar" diff --git a/spacy/tests/pipeline/test_pipe_methods.py b/spacy/tests/pipeline/test_pipe_methods.py index d42216655..e37375bf1 100644 --- a/spacy/tests/pipeline/test_pipe_methods.py +++ b/spacy/tests/pipeline/test_pipe_methods.py @@ -7,67 +7,74 @@ def nlp(): return Language() +@Language.component("new_pipe") def new_pipe(doc): return doc +@Language.component("other_pipe") +def other_pipe(doc): + return doc + + def test_add_pipe_no_name(nlp): - nlp.add_pipe(new_pipe) + nlp.add_pipe("new_pipe") assert "new_pipe" in nlp.pipe_names def test_add_pipe_duplicate_name(nlp): - nlp.add_pipe(new_pipe, name="duplicate_name") + nlp.add_pipe("new_pipe", name="duplicate_name") with pytest.raises(ValueError): - nlp.add_pipe(new_pipe, name="duplicate_name") + nlp.add_pipe("new_pipe", name="duplicate_name") @pytest.mark.parametrize("name", ["parser"]) def test_add_pipe_first(nlp, name): - nlp.add_pipe(new_pipe, name=name, first=True) + nlp.add_pipe("new_pipe", name=name, first=True) assert nlp.pipeline[0][0] == name @pytest.mark.parametrize("name1,name2", [("parser", "lambda_pipe")]) def test_add_pipe_last(nlp, name1, name2): - nlp.add_pipe(lambda doc: doc, name=name2) - nlp.add_pipe(new_pipe, name=name1, last=True) + Language.component("new_pipe2", func=lambda doc: doc) + nlp.add_pipe("new_pipe2", name=name2) + nlp.add_pipe("new_pipe", name=name1, last=True) assert nlp.pipeline[0][0] != name1 assert nlp.pipeline[-1][0] == name1 def test_cant_add_pipe_first_and_last(nlp): with pytest.raises(ValueError): - nlp.add_pipe(new_pipe, first=True, last=True) + nlp.add_pipe("new_pipe", first=True, last=True) @pytest.mark.parametrize("name", ["my_component"]) def test_get_pipe(nlp, name): with pytest.raises(KeyError): nlp.get_pipe(name) - nlp.add_pipe(new_pipe, name=name) + nlp.add_pipe("new_pipe", name=name) assert nlp.get_pipe(name) == new_pipe @pytest.mark.parametrize( - "name,replacement,not_callable", [("my_component", lambda doc: doc, {})] + "name,replacement,invalid_replacement", + [("my_component", "other_pipe", lambda doc: doc)], ) -def test_replace_pipe(nlp, name, replacement, not_callable): +def test_replace_pipe(nlp, name, replacement, invalid_replacement): with pytest.raises(ValueError): nlp.replace_pipe(name, new_pipe) - nlp.add_pipe(new_pipe, name=name) + nlp.add_pipe("new_pipe", name=name) with pytest.raises(ValueError): - nlp.replace_pipe(name, not_callable) + nlp.replace_pipe(name, invalid_replacement) nlp.replace_pipe(name, replacement) - assert nlp.get_pipe(name) != new_pipe - assert nlp.get_pipe(name) == replacement + assert nlp.get_pipe(name) == nlp.create_pipe(replacement) @pytest.mark.parametrize("old_name,new_name", [("old_pipe", "new_pipe")]) def test_rename_pipe(nlp, old_name, new_name): with pytest.raises(ValueError): nlp.rename_pipe(old_name, new_name) - nlp.add_pipe(new_pipe, name=old_name) + nlp.add_pipe("new_pipe", name=old_name) nlp.rename_pipe(old_name, new_name) assert nlp.pipeline[0][0] == new_name @@ -76,7 +83,7 @@ def test_rename_pipe(nlp, old_name, new_name): def test_remove_pipe(nlp, name): with pytest.raises(ValueError): nlp.remove_pipe(name) - nlp.add_pipe(new_pipe, name=name) + nlp.add_pipe("new_pipe", name=name) assert len(nlp.pipeline) == 1 removed_name, removed_component = nlp.remove_pipe(name) assert not len(nlp.pipeline) @@ -86,7 +93,7 @@ def test_remove_pipe(nlp, name): @pytest.mark.parametrize("name", ["my_component"]) def test_disable_pipes_method(nlp, name): - nlp.add_pipe(new_pipe, name=name) + nlp.add_pipe("new_pipe", name=name) assert nlp.has_pipe(name) disabled = nlp.select_pipes(disable=name) assert not nlp.has_pipe(name) @@ -95,7 +102,7 @@ def test_disable_pipes_method(nlp, name): @pytest.mark.parametrize("name", ["my_component"]) def test_enable_pipes_method(nlp, name): - nlp.add_pipe(new_pipe, name=name) + nlp.add_pipe("new_pipe", name=name) assert nlp.has_pipe(name) disabled = nlp.select_pipes(enable=[]) assert not nlp.has_pipe(name) @@ -104,7 +111,7 @@ def test_enable_pipes_method(nlp, name): @pytest.mark.parametrize("name", ["my_component"]) def test_disable_pipes_context(nlp, name): - nlp.add_pipe(new_pipe, name=name) + nlp.add_pipe("new_pipe", name=name) assert nlp.has_pipe(name) with nlp.select_pipes(disable=name): assert not nlp.has_pipe(name) @@ -113,7 +120,7 @@ def test_disable_pipes_context(nlp, name): def test_select_pipes_list_arg(nlp): for name in ["c1", "c2", "c3"]: - nlp.add_pipe(new_pipe, name=name) + nlp.add_pipe("new_pipe", name=name) assert nlp.has_pipe(name) with nlp.select_pipes(disable=["c1", "c2"]): assert not nlp.has_pipe("c1") @@ -143,7 +150,7 @@ def test_select_pipes_list_arg(nlp): def test_select_pipes_errors(nlp): for name in ["c1", "c2", "c3"]: - nlp.add_pipe(new_pipe, name=name) + nlp.add_pipe("new_pipe", name=name) assert nlp.has_pipe(name) with pytest.raises(ValueError): @@ -161,12 +168,13 @@ def test_select_pipes_errors(nlp): @pytest.mark.parametrize("n_pipes", [100]) def test_add_lots_of_pipes(nlp, n_pipes): + Language.component("n_pipes", func=lambda doc: doc) for i in range(n_pipes): - nlp.add_pipe(lambda doc: doc, name=f"pipe_{i}") + nlp.add_pipe("n_pipes", name=f"pipe_{i}") assert len(nlp.pipe_names) == n_pipes -@pytest.mark.parametrize("component", ["ner", {"hello": "world"}]) +@pytest.mark.parametrize("component", [lambda doc: doc, {"hello": "world"}]) def test_raise_for_invalid_components(nlp, component): with pytest.raises(ValueError): nlp.add_pipe(component) @@ -190,11 +198,38 @@ def test_pipe_labels(nlp): "textcat": ["POSITIVE", "NEGATIVE"], } for name, labels in input_labels.items(): - pipe = nlp.create_pipe(name) + nlp.add_pipe(name) + pipe = nlp.get_pipe(name) for label in labels: pipe.add_label(label) assert len(pipe.labels) == len(labels) - nlp.add_pipe(pipe) + assert len(nlp.pipe_labels) == len(input_labels) for name, labels in nlp.pipe_labels.items(): assert sorted(input_labels[name]) == sorted(labels) + + +def test_add_pipe_before_after(): + """Test that before/after works with strings and ints.""" + nlp = Language() + nlp.add_pipe("ner") + with pytest.raises(ValueError): + nlp.add_pipe("textcat", before="parser") + nlp.add_pipe("textcat", before="ner") + assert nlp.pipe_names == ["textcat", "ner"] + with pytest.raises(ValueError): + nlp.add_pipe("parser", before=3) + with pytest.raises(ValueError): + nlp.add_pipe("parser", after=3) + nlp.add_pipe("parser", after=0) + assert nlp.pipe_names == ["textcat", "parser", "ner"] + nlp.add_pipe("tagger", before=2) + assert nlp.pipe_names == ["textcat", "parser", "tagger", "ner"] + with pytest.raises(ValueError): + nlp.add_pipe("entity_ruler", after=1, first=True) + with pytest.raises(ValueError): + nlp.add_pipe("entity_ruler", before="ner", after=2) + with pytest.raises(ValueError): + nlp.add_pipe("entity_ruler", before=True) + with pytest.raises(ValueError): + nlp.add_pipe("entity_ruler", first=False) diff --git a/spacy/tests/pipeline/test_sentencizer.py b/spacy/tests/pipeline/test_sentencizer.py index 6dfa0acee..1b1c51f34 100644 --- a/spacy/tests/pipeline/test_sentencizer.py +++ b/spacy/tests/pipeline/test_sentencizer.py @@ -7,7 +7,7 @@ from spacy.lang.en import English def test_sentencizer(en_vocab): doc = Doc(en_vocab, words=["Hello", "!", "This", "is", "a", "test", "."]) - sentencizer = Sentencizer() + sentencizer = Sentencizer(punct_chars=None) doc = sentencizer(doc) assert doc.is_sentenced sent_starts = [t.is_sent_start for t in doc] @@ -20,7 +20,7 @@ def test_sentencizer(en_vocab): def test_sentencizer_pipe(): texts = ["Hello! This is a test.", "Hi! This is a test."] nlp = English() - nlp.add_pipe(nlp.create_pipe("sentencizer")) + nlp.add_pipe("sentencizer") for doc in nlp.pipe(texts): assert doc.is_sentenced sent_starts = [t.is_sent_start for t in doc] @@ -39,7 +39,7 @@ def test_sentencizer_empty_docs(): many_empty_texts = ["", "", ""] some_empty_texts = ["hi", "", "This is a test. Here are two sentences.", ""] nlp = English() - nlp.add_pipe(nlp.create_pipe("sentencizer")) + nlp.add_pipe("sentencizer") for texts in [one_empty_text, many_empty_texts, some_empty_texts]: for doc in nlp.pipe(texts): assert doc.is_sentenced @@ -80,7 +80,7 @@ def test_sentencizer_empty_docs(): ) def test_sentencizer_complex(en_vocab, words, sent_starts, sent_ends, n_sents): doc = Doc(en_vocab, words=words) - sentencizer = Sentencizer() + sentencizer = Sentencizer(punct_chars=None) doc = sentencizer(doc) assert doc.is_sentenced assert [t.is_sent_start for t in doc] == sent_starts @@ -126,7 +126,7 @@ def test_sentencizer_serialize_bytes(en_vocab): sentencizer = Sentencizer(punct_chars=punct_chars) assert sentencizer.punct_chars == set(punct_chars) bytes_data = sentencizer.to_bytes() - new_sentencizer = Sentencizer().from_bytes(bytes_data) + new_sentencizer = Sentencizer(punct_chars=None).from_bytes(bytes_data) assert new_sentencizer.punct_chars == set(punct_chars) @@ -147,7 +147,6 @@ def test_sentencizer_serialize_bytes(en_vocab): ) def test_sentencizer_across_scripts(lang, text): nlp = spacy.blank(lang) - sentencizer = Sentencizer() - nlp.add_pipe(sentencizer) + nlp.add_pipe("sentencizer") doc = nlp(text) assert len(list(doc.sents)) > 1 diff --git a/spacy/tests/pipeline/test_senter.py b/spacy/tests/pipeline/test_senter.py index 82f536076..b64fa8581 100644 --- a/spacy/tests/pipeline/test_senter.py +++ b/spacy/tests/pipeline/test_senter.py @@ -9,9 +9,9 @@ from spacy.tests.util import make_tempdir def test_label_types(): nlp = Language() - nlp.add_pipe(nlp.create_pipe("senter")) + senter = nlp.add_pipe("senter") with pytest.raises(NotImplementedError): - nlp.get_pipe("senter").add_label("A") + senter.add_label("A") SENT_STARTS = [0] * 14 @@ -34,7 +34,6 @@ TRAIN_DATA = [ def test_overfitting_IO(): # Simple test to try and quickly overfit the senter - ensuring the ML models work correctly nlp = English() - senter = nlp.create_pipe("senter") train_examples = [] for t in TRAIN_DATA: train_examples.append(Example.from_dict(nlp.make_doc(t[0]), t[1])) @@ -43,7 +42,7 @@ def test_overfitting_IO(): train_examples[1].reference[1].is_sent_start = False train_examples[1].reference[11].is_sent_start = False - nlp.add_pipe(senter) + nlp.add_pipe("senter") optimizer = nlp.begin_training() for i in range(200): diff --git a/spacy/tests/pipeline/test_tagger.py b/spacy/tests/pipeline/test_tagger.py index 941a3fc64..dd6739e17 100644 --- a/spacy/tests/pipeline/test_tagger.py +++ b/spacy/tests/pipeline/test_tagger.py @@ -11,20 +11,19 @@ from ..util import make_tempdir def test_label_types(): nlp = Language() - nlp.add_pipe(nlp.create_pipe("tagger")) - nlp.get_pipe("tagger").add_label("A") + tagger = nlp.add_pipe("tagger") + tagger.add_label("A") with pytest.raises(ValueError): - nlp.get_pipe("tagger").add_label(9) + tagger.add_label(9) def test_tagger_begin_training_tag_map(): """Test that Tagger.begin_training() without gold tuples does not clobber the tag map.""" nlp = Language() - tagger = nlp.create_pipe("tagger") + tagger = nlp.add_pipe("tagger") orig_tag_count = len(tagger.labels) tagger.add_label("A", {"POS": "NOUN"}) - nlp.add_pipe(tagger) nlp.begin_training() assert nlp.vocab.morphology.tag_map["A"] == {POS: NOUN} assert orig_tag_count + 1 == len(nlp.get_pipe("tagger").labels) @@ -45,12 +44,13 @@ def test_overfitting_IO(): nlp = English() nlp.vocab.morphology.load_tag_map(TAG_MAP) nlp.vocab.morphology.load_morph_exceptions(MORPH_RULES) - tagger = nlp.create_pipe("tagger") + tagger = nlp.add_pipe("tagger", config={"set_morphology": True}) nlp.vocab.morphology.load_tag_map(TAG_MAP) train_examples = [] for t in TRAIN_DATA: train_examples.append(Example.from_dict(nlp.make_doc(t[0]), t[1])) - nlp.add_pipe(tagger) + for tag, values in TAG_MAP.items(): + tagger.add_label(tag, values) optimizer = nlp.begin_training() for i in range(50): diff --git a/spacy/tests/pipeline/test_textcat.py b/spacy/tests/pipeline/test_textcat.py index a39b5075b..5eb09a007 100644 --- a/spacy/tests/pipeline/test_textcat.py +++ b/spacy/tests/pipeline/test_textcat.py @@ -7,7 +7,7 @@ from spacy.lang.en import English from spacy.language import Language from spacy.pipeline import TextCategorizer from spacy.tokens import Doc -from spacy.pipeline.defaults import default_tok2vec +from spacy.pipeline.tok2vec import DEFAULT_TOK2VEC_MODEL from ..util import make_tempdir from ...gold import Example @@ -22,8 +22,8 @@ TRAIN_DATA = [ @pytest.mark.skip(reason="Test is flakey when run with others") def test_simple_train(): nlp = Language() - nlp.add_pipe(nlp.create_pipe("textcat")) - nlp.get_pipe("textcat").add_label("answer") + textcat = nlp.add_pipe("textcat") + textcat.add_label("answer") nlp.begin_training() for i in range(5): for text, answer in [ @@ -74,23 +74,22 @@ def test_textcat_learns_multilabel(): def test_label_types(): nlp = Language() - nlp.add_pipe(nlp.create_pipe("textcat")) - nlp.get_pipe("textcat").add_label("answer") + textcat = nlp.add_pipe("textcat") + textcat.add_label("answer") with pytest.raises(ValueError): - nlp.get_pipe("textcat").add_label(9) + textcat.add_label(9) def test_overfitting_IO(): # Simple test to try and quickly overfit the textcat component - ensuring the ML models work correctly fix_random_seed(0) nlp = English() - textcat = nlp.create_pipe("textcat", config={"exclusive_classes": True}) + textcat = nlp.add_pipe("textcat") train_examples = [] for text, annotations in TRAIN_DATA: train_examples.append(Example.from_dict(nlp.make_doc(text), annotations)) for label, value in annotations.get("cats").items(): textcat.add_label(label) - nlp.add_pipe(textcat) optimizer = nlp.begin_training() for i in range(50): @@ -127,21 +126,20 @@ def test_overfitting_IO(): {"@architectures": "spacy.TextCat.v1", "exclusive_classes": False, "ngram_size": 1, "pretrained_vectors": False, "width": 64, "conv_depth": 2, "embed_size": 2000, "window_size": 2, "dropout": None}, {"@architectures": "spacy.TextCat.v1", "exclusive_classes": True, "ngram_size": 5, "pretrained_vectors": False, "width": 128, "conv_depth": 2, "embed_size": 2000, "window_size": 1, "dropout": None}, {"@architectures": "spacy.TextCat.v1", "exclusive_classes": True, "ngram_size": 2, "pretrained_vectors": False, "width": 32, "conv_depth": 3, "embed_size": 500, "window_size": 3, "dropout": None}, - {"@architectures": "spacy.TextCatCNN.v1", "tok2vec": default_tok2vec(), "exclusive_classes": True}, - {"@architectures": "spacy.TextCatCNN.v1", "tok2vec": default_tok2vec(), "exclusive_classes": False}, + {"@architectures": "spacy.TextCatCNN.v1", "tok2vec": DEFAULT_TOK2VEC_MODEL, "exclusive_classes": True}, + {"@architectures": "spacy.TextCatCNN.v1", "tok2vec": DEFAULT_TOK2VEC_MODEL, "exclusive_classes": False}, ], ) # fmt: on def test_textcat_configs(textcat_config): pipe_config = {"model": textcat_config} nlp = English() - textcat = nlp.create_pipe("textcat", pipe_config) + textcat = nlp.add_pipe("textcat", config=pipe_config) train_examples = [] for text, annotations in TRAIN_DATA: train_examples.append(Example.from_dict(nlp.make_doc(text), annotations)) for label, value in annotations.get("cats").items(): textcat.add_label(label) - nlp.add_pipe(textcat) optimizer = nlp.begin_training() for i in range(5): losses = {} diff --git a/spacy/tests/regression/test_issue1-1000.py b/spacy/tests/regression/test_issue1-1000.py index 889d1a154..c1b83c6c4 100644 --- a/spacy/tests/regression/test_issue1-1000.py +++ b/spacy/tests/regression/test_issue1-1000.py @@ -11,7 +11,7 @@ from spacy.lang.en import English from spacy.lemmatizer import Lemmatizer from spacy.lookups import Lookups from spacy.tokens import Doc, Span -from spacy.lang.en import EnglishDefaults +from spacy.lang.en.lemmatizer import is_base_form from ..util import get_doc, make_tempdir @@ -165,7 +165,7 @@ def test_issue595(): lookups.add_table("lemma_rules", {"verb": [["ed", "e"]]}) lookups.add_table("lemma_index", {"verb": {}}) lookups.add_table("lemma_exc", {"verb": {}}) - lemmatizer = Lemmatizer(lookups, is_base_form=EnglishDefaults.is_base_form) + lemmatizer = Lemmatizer(lookups, is_base_form=is_base_form) vocab = Vocab(lemmatizer=lemmatizer, tag_map=tag_map) doc = Doc(vocab, words=words) doc[2].tag_ = "VB" @@ -281,7 +281,9 @@ def test_control_issue792(en_tokenizer, text): assert "".join([token.text_with_ws for token in doc]) == text -@pytest.mark.skip(reason="Can not be fixed unless with variable-width lookbehinds, cf. PR #3218") +@pytest.mark.skip( + reason="Can not be fixed unless with variable-width lookbehinds, cf. PR #3218" +) @pytest.mark.parametrize( "text,tokens", [ @@ -431,10 +433,8 @@ def test_issue999(): ["show me chinese restaurants", [(8, 15, "CUISINE")]], ["show me chines restaurants", [(8, 14, "CUISINE")]], ] - nlp = English() - ner = nlp.create_pipe("ner", {"learn_rate": 0.001}) # will need to be {"model": ...} in upcoming PR - nlp.add_pipe(ner) + ner = nlp.add_pipe("ner") for _, offsets in TRAIN_DATA: for start, end, label in offsets: ner.add_label(label) @@ -442,7 +442,9 @@ def test_issue999(): for itn in range(20): random.shuffle(TRAIN_DATA) for raw_text, entity_offsets in TRAIN_DATA: - example = Example.from_dict(nlp.make_doc(raw_text), {"entities": entity_offsets}) + example = Example.from_dict( + nlp.make_doc(raw_text), {"entities": entity_offsets} + ) nlp.update([example]) with make_tempdir() as model_dir: diff --git a/spacy/tests/regression/test_issue1001-1500.py b/spacy/tests/regression/test_issue1001-1500.py index a9b54fc6d..d612150de 100644 --- a/spacy/tests/regression/test_issue1001-1500.py +++ b/spacy/tests/regression/test_issue1001-1500.py @@ -14,7 +14,7 @@ from spacy.symbols import ORTH, LEMMA, POS, VERB def test_issue1061(): """Test special-case works after tokenizing. Was caching problem.""" text = "I like _MATH_ even _MATH_ when _MATH_, except when _MATH_ is _MATH_! but not _MATH_." - tokenizer = English.Defaults.create_tokenizer() + tokenizer = English().tokenizer doc = tokenizer(text) assert "MATH" in [w.text for w in doc] assert "_MATH_" not in [w.text for w in doc] @@ -25,7 +25,7 @@ def test_issue1061(): assert "MATH" not in [w.text for w in doc] # For sanity, check it works when pipeline is clean. - tokenizer = English.Defaults.create_tokenizer() + tokenizer = English().tokenizer tokenizer.add_special_case("_MATH_", [{ORTH: "_MATH_"}]) doc = tokenizer(text) assert "_MATH_" in [w.text for w in doc] diff --git a/spacy/tests/regression/test_issue1501-2000.py b/spacy/tests/regression/test_issue1501-2000.py index 8c989a7eb..9d2ef999b 100644 --- a/spacy/tests/regression/test_issue1501-2000.py +++ b/spacy/tests/regression/test_issue1501-2000.py @@ -10,9 +10,7 @@ from spacy.lang.lex_attrs import is_stop from spacy.vectors import Vectors from spacy.vocab import Vocab from spacy.language import Language -from spacy.pipeline.defaults import default_ner, default_tagger from spacy.tokens import Doc, Span, Token -from spacy.pipeline import Tagger, EntityRecognizer from spacy.attrs import HEAD, DEP from spacy.matcher import Matcher @@ -100,15 +98,20 @@ def test_issue1612(en_tokenizer): def test_issue1654(): nlp = Language(Vocab()) assert not nlp.pipeline - nlp.add_pipe(lambda doc: doc, name="1") - nlp.add_pipe(lambda doc: doc, name="2", after="1") - nlp.add_pipe(lambda doc: doc, name="3", after="2") + + @Language.component("component") + def component(doc): + return doc + + nlp.add_pipe("component", name="1") + nlp.add_pipe("component", name="2", after="1") + nlp.add_pipe("component", name="3", after="2") assert nlp.pipe_names == ["1", "2", "3"] nlp2 = Language(Vocab()) assert not nlp2.pipeline - nlp2.add_pipe(lambda doc: doc, name="3") - nlp2.add_pipe(lambda doc: doc, name="2", before="3") - nlp2.add_pipe(lambda doc: doc, name="1", before="2") + nlp2.add_pipe("component", name="3") + nlp2.add_pipe("component", name="2", before="3") + nlp2.add_pipe("component", name="1", before="2") assert nlp2.pipe_names == ["1", "2", "3"] @@ -122,9 +125,10 @@ def test_issue1698(en_tokenizer, text): def test_issue1727(): """Test that models with no pretrained vectors can be deserialized correctly after vectors are added.""" + nlp = Language(Vocab()) data = numpy.ones((3, 300), dtype="f") vectors = Vectors(data=data, keys=["I", "am", "Matt"]) - tagger = Tagger(Vocab(), default_tagger()) + tagger = nlp.create_pipe("tagger") tagger.add_label("PRP") with pytest.warns(UserWarning): tagger.begin_training() @@ -132,7 +136,7 @@ def test_issue1727(): tagger.vocab.vectors = vectors with make_tempdir() as path: tagger.to_disk(path) - tagger = Tagger(Vocab(), default_tagger()).from_disk(path) + tagger = nlp.create_pipe("tagger").from_disk(path) assert tagger.cfg.get("pretrained_dims", 0) == 0 @@ -241,8 +245,8 @@ def test_issue1889(word): def test_issue1915(): cfg = {"hidden_depth": 2} # should error out nlp = Language() - nlp.add_pipe(nlp.create_pipe("ner")) - nlp.get_pipe("ner").add_label("answer") + ner = nlp.add_pipe("ner") + ner.add_label("answer") with pytest.raises(ValueError): nlp.begin_training(**cfg) @@ -270,13 +274,12 @@ def test_issue1963(en_tokenizer): @pytest.mark.parametrize("label", ["U-JOB-NAME"]) def test_issue1967(label): + nlp = Language() config = { "learn_tokens": False, "min_action_freq": 30, - "beam_width": 1, - "beam_update_prob": 1.0, } - ner = EntityRecognizer(Vocab(), default_ner(), **config) + ner = nlp.create_pipe("ner", config=config) example = Example.from_dict( Doc(ner.vocab, words=["word"]), { diff --git a/spacy/tests/regression/test_issue2001-2500.py b/spacy/tests/regression/test_issue2001-2500.py index 1965c0f05..a09c6f4fb 100644 --- a/spacy/tests/regression/test_issue2001-2500.py +++ b/spacy/tests/regression/test_issue2001-2500.py @@ -29,13 +29,11 @@ def test_issue2070(): def test_issue2179(): """Test that spurious 'extra_labels' aren't created when initializing NER.""" nlp = Italian() - ner = nlp.create_pipe("ner") + ner = nlp.add_pipe("ner") ner.add_label("CITIZENSHIP") - nlp.add_pipe(ner) nlp.begin_training() nlp2 = Italian() - nlp2.add_pipe(nlp2.create_pipe("ner")) - + nlp2.add_pipe("ner") assert len(nlp2.get_pipe("ner").labels) == 0 model = nlp2.get_pipe("ner").model model.attrs["resize_output"](model, nlp.get_pipe("ner").moves.n_moves) @@ -141,6 +139,6 @@ def test_issue2464(en_vocab): def test_issue2482(): """Test we can serialize and deserialize a blank NER or parser model.""" nlp = Italian() - nlp.add_pipe(nlp.create_pipe("ner")) + nlp.add_pipe("ner") b = nlp.to_bytes() Italian().from_bytes(b) diff --git a/spacy/tests/regression/test_issue2501-3000.py b/spacy/tests/regression/test_issue2501-3000.py index c013e2766..7917157aa 100644 --- a/spacy/tests/regression/test_issue2501-3000.py +++ b/spacy/tests/regression/test_issue2501-3000.py @@ -19,10 +19,9 @@ from ..util import get_doc def test_issue2564(): """Test the tagger sets is_tagged correctly when used via Language.pipe.""" nlp = Language() - tagger = nlp.create_pipe("tagger") + tagger = nlp.add_pipe("tagger") with pytest.warns(UserWarning): tagger.begin_training() # initialise weights - nlp.add_pipe(tagger) doc = nlp("hello world") assert doc.is_tagged docs = nlp.pipe(["hello", "world"]) @@ -149,8 +148,7 @@ def test_issue2800(): [Example.from_dict(nlp.make_doc("One sentence"), {"entities": []})] ) entity_types = [str(i) for i in range(1000)] - ner = nlp.create_pipe("ner") - nlp.add_pipe(ner) + ner = nlp.add_pipe("ner") for entity_type in list(entity_types): ner.add_label(entity_type) optimizer = nlp.begin_training() diff --git a/spacy/tests/regression/test_issue3001-3500.py b/spacy/tests/regression/test_issue3001-3500.py index 599f0900a..6b4a9ad1d 100644 --- a/spacy/tests/regression/test_issue3001-3500.py +++ b/spacy/tests/regression/test_issue3001-3500.py @@ -1,7 +1,8 @@ import pytest +from spacy import registry from spacy.lang.en import English from spacy.lang.de import German -from spacy.pipeline.defaults import default_ner +from spacy.pipeline.ner import DEFAULT_NER_MODEL from spacy.pipeline import EntityRuler, EntityRecognizer from spacy.matcher import Matcher, PhraseMatcher from spacy.tokens import Doc @@ -96,19 +97,17 @@ def test_issue3209(): were added using ner.add_label(). """ nlp = English() - ner = nlp.create_pipe("ner") - nlp.add_pipe(ner) - + ner = nlp.add_pipe("ner") ner.add_label("ANIMAL") nlp.begin_training() move_names = ["O", "B-ANIMAL", "I-ANIMAL", "L-ANIMAL", "U-ANIMAL"] assert ner.move_names == move_names nlp2 = English() - nlp2.add_pipe(nlp2.create_pipe("ner")) - model = nlp2.get_pipe("ner").model + ner2 = nlp2.add_pipe("ner") + model = ner2.model model.attrs["resize_output"](model, ner.moves.n_moves) nlp2.from_bytes(nlp.to_bytes()) - assert nlp2.get_pipe("ner").move_names == move_names + assert ner2.move_names == move_names def test_issue3248_1(): @@ -156,10 +155,10 @@ def test_issue3289(): """Test that Language.to_bytes handles serializing a pipeline component with an uninitialized model.""" nlp = English() - nlp.add_pipe(nlp.create_pipe("textcat")) + nlp.add_pipe("textcat") bytes_data = nlp.to_bytes() new_nlp = English() - new_nlp.add_pipe(nlp.create_pipe("textcat")) + new_nlp.add_pipe("textcat") new_nlp.from_bytes(bytes_data) @@ -200,10 +199,10 @@ def test_issue3345(): config = { "learn_tokens": False, "min_action_freq": 30, - "beam_width": 1, - "beam_update_prob": 1.0, + "update_with_oracle_cut_size": 100, } - ner = EntityRecognizer(doc.vocab, default_ner(), **config) + model = registry.make_from_config({"model": DEFAULT_NER_MODEL}, validate=True)["model"] + ner = EntityRecognizer(doc.vocab, model, **config) # Add the OUT action. I wouldn't have thought this would be necessary... ner.moves.add_action(5, "") ner.add_label("GPE") @@ -229,7 +228,7 @@ def test_issue3412(): @pytest.mark.skip(reason="default suffix rules avoid one upper-case letter before dot") def test_issue3449(): nlp = English() - nlp.add_pipe(nlp.create_pipe("sentencizer")) + nlp.add_pipe("sentencizer") text1 = "He gave the ball to I. Do you want to go to the movies with I?" text2 = "He gave the ball to I. Do you want to go to the movies with I?" text3 = "He gave the ball to I.\nDo you want to go to the movies with I?" @@ -245,7 +244,7 @@ def test_issue3449(): def test_issue3456(): # this crashed because of a padding error in layer.ops.unflatten in thinc nlp = English() - nlp.add_pipe(nlp.create_pipe("tagger")) + nlp.add_pipe("tagger") nlp.begin_training() list(nlp.pipe(["hi", ""])) @@ -254,7 +253,7 @@ def test_issue3468(): """Test that sentence boundaries are set correctly so Doc.is_sentenced can be restored after serialization.""" nlp = English() - nlp.add_pipe(nlp.create_pipe("sentencizer")) + nlp.add_pipe("sentencizer") doc = nlp("Hello world") assert doc[0].is_sent_start assert doc.is_sentenced diff --git a/spacy/tests/regression/test_issue3501-4000.py b/spacy/tests/regression/test_issue3501-4000.py index 80b32cfd6..6426c6c24 100644 --- a/spacy/tests/regression/test_issue3501-4000.py +++ b/spacy/tests/regression/test_issue3501-4000.py @@ -2,7 +2,7 @@ import pytest from spacy.language import Language from spacy.vocab import Vocab from spacy.pipeline import EntityRuler, DependencyParser -from spacy.pipeline.defaults import default_parser +from spacy.pipeline.dep_parser import DEFAULT_PARSER_MODEL from spacy import displacy, load from spacy.displacy import parse_deps from spacy.tokens import Doc, Token @@ -14,6 +14,7 @@ from spacy.lang.hi import Hindi from spacy.lang.es import Spanish from spacy.lang.en import English from spacy.attrs import IS_ALPHA +from spacy import registry from thinc.api import compounding import spacy import srsly @@ -93,9 +94,10 @@ def test_issue_3526_3(en_vocab): @pytest.mark.filterwarnings("ignore::UserWarning") def test_issue_3526_4(en_vocab): nlp = Language(vocab=en_vocab) - ruler = EntityRuler(nlp, overwrite_ents=True) - ruler.add_patterns([{"label": "ORG", "pattern": "Apple"}]) - nlp.add_pipe(ruler) + patterns = [{"label": "ORG", "pattern": "Apple"}] + config = {"overwrite_ents": True} + ruler = nlp.add_pipe("entity_ruler", config=config) + ruler.add_patterns(patterns) with make_tempdir() as tmpdir: nlp.to_disk(tmpdir) ruler = nlp.get_pipe("entity_ruler") @@ -205,16 +207,18 @@ def test_issue3611(): cat_dict = {label: label == train_instance for label in unique_classes} train_data.append(Example.from_dict(nlp.make_doc(text), {"cats": cat_dict})) # add a text categorizer component - textcat = nlp.create_pipe( - "textcat", - config={"exclusive_classes": True, "architecture": "bow", "ngram_size": 2}, - ) + model = { + "@architectures": "spacy.TextCatBOW.v1", + "exclusive_classes": True, + "ngram_size": 2, + "no_output_layer": False, + } + textcat = nlp.add_pipe("textcat", config={"model": model}, last=True) for label in unique_classes: textcat.add_label(label) - nlp.add_pipe(textcat, last=True) # training the network with nlp.select_pipes(enable="textcat"): - optimizer = nlp.begin_training(X=x_train, Y=y_train) + optimizer = nlp.begin_training() for i in range(3): losses = {} batches = minibatch(train_data, size=compounding(4.0, 32.0, 1.001)) @@ -248,10 +252,12 @@ def test_issue3830_no_subtok(): config = { "learn_tokens": False, "min_action_freq": 30, - "beam_width": 1, - "beam_update_prob": 1.0, + "update_with_oracle_cut_size": 100, } - parser = DependencyParser(Vocab(), default_parser(), **config) + model = registry.make_from_config({"model": DEFAULT_PARSER_MODEL}, validate=True)[ + "model" + ] + parser = DependencyParser(Vocab(), model, **config) parser.add_label("nsubj") assert "subtok" not in parser.labels parser.begin_training(lambda: []) @@ -264,10 +270,12 @@ def test_issue3830_with_subtok(): config = { "learn_tokens": True, "min_action_freq": 30, - "beam_width": 1, - "beam_update_prob": 1.0, + "update_with_oracle_cut_size": 100, } - parser = DependencyParser(Vocab(), default_parser(), **config) + model = registry.make_from_config({"model": DEFAULT_PARSER_MODEL}, validate=True)[ + "model" + ] + parser = DependencyParser(Vocab(), model, **config) parser.add_label("nsubj") assert "subtok" not in parser.labels parser.begin_training(lambda: []) @@ -327,12 +335,9 @@ def test_issue3880(): """ texts = ["hello", "world", "", ""] nlp = English() - nlp.add_pipe(nlp.create_pipe("parser")) - nlp.add_pipe(nlp.create_pipe("ner")) - nlp.add_pipe(nlp.create_pipe("tagger")) - nlp.get_pipe("parser").add_label("dep") - nlp.get_pipe("ner").add_label("PERSON") - nlp.get_pipe("tagger").add_label("NN") + nlp.add_pipe("parser").add_label("dep") + nlp.add_pipe("ner").add_label("PERSON") + nlp.add_pipe("tagger").add_label("NN") nlp.begin_training() for doc in nlp.pipe(texts): pass diff --git a/spacy/tests/regression/test_issue4001-4500.py b/spacy/tests/regression/test_issue4001-4500.py index 626856e9e..636cddcb7 100644 --- a/spacy/tests/regression/test_issue4001-4500.py +++ b/spacy/tests/regression/test_issue4001-4500.py @@ -1,6 +1,5 @@ import pytest -from spacy.pipeline import EntityRuler, EntityRecognizer, Pipe -from spacy.pipeline.defaults import default_ner +from spacy.pipeline import Pipe from spacy.matcher import PhraseMatcher, Matcher from spacy.tokens import Doc, Span, DocBin from spacy.gold import Example, Corpus @@ -56,13 +55,15 @@ def test_issue4030(): cat_dict = {label: label == train_instance for label in unique_classes} train_data.append(Example.from_dict(nlp.make_doc(text), {"cats": cat_dict})) # add a text categorizer component - textcat = nlp.create_pipe( - "textcat", - config={"exclusive_classes": True, "architecture": "bow", "ngram_size": 2}, - ) + model = { + "@architectures": "spacy.TextCatBOW.v1", + "exclusive_classes": True, + "ngram_size": 2, + "no_output_layer": False, + } + textcat = nlp.add_pipe("textcat", config={"model": model}, last=True) for label in unique_classes: textcat.add_label(label) - nlp.add_pipe(textcat, last=True) # training the network with nlp.select_pipes(enable="textcat"): optimizer = nlp.begin_training() @@ -84,21 +85,18 @@ def test_issue4030(): def test_issue4042(): """Test that serialization of an EntityRuler before NER works fine.""" nlp = English() - # add ner pipe - ner = nlp.create_pipe("ner") + ner = nlp.add_pipe("ner") ner.add_label("SOME_LABEL") - nlp.add_pipe(ner) nlp.begin_training() - # Add entity ruler - ruler = EntityRuler(nlp) patterns = [ {"label": "MY_ORG", "pattern": "Apple"}, {"label": "MY_GPE", "pattern": [{"lower": "san"}, {"lower": "francisco"}]}, ] + # works fine with "after" + ruler = nlp.add_pipe("entity_ruler", before="ner") ruler.add_patterns(patterns) - nlp.add_pipe(ruler, before="ner") # works fine with "after" doc1 = nlp("What do you think about Apple ?") assert doc1.ents[0].label_ == "MY_ORG" @@ -107,7 +105,6 @@ def test_issue4042(): if not output_dir.exists(): output_dir.mkdir() nlp.to_disk(output_dir) - nlp2 = load_model(output_dir) doc2 = nlp2("What do you think about Apple ?") assert doc2.ents[0].label_ == "MY_ORG" @@ -120,41 +117,32 @@ def test_issue4042_bug2(): This is the second bug of two bugs underlying the issue 4042. """ nlp1 = English() - vocab = nlp1.vocab - # add ner pipe - ner1 = nlp1.create_pipe("ner") + ner1 = nlp1.add_pipe("ner") ner1.add_label("SOME_LABEL") - nlp1.add_pipe(ner1) nlp1.begin_training() - # add a new label to the doc doc1 = nlp1("What do you think about Apple ?") assert len(ner1.labels) == 1 assert "SOME_LABEL" in ner1.labels apple_ent = Span(doc1, 5, 6, label="MY_ORG") doc1.ents = list(doc1.ents) + [apple_ent] - # reapply the NER - at this point it should resize itself ner1(doc1) assert len(ner1.labels) == 2 assert "SOME_LABEL" in ner1.labels assert "MY_ORG" in ner1.labels - with make_tempdir() as d: # assert IO goes fine output_dir = ensure_path(d) if not output_dir.exists(): output_dir.mkdir() ner1.to_disk(output_dir) - config = { "learn_tokens": False, "min_action_freq": 30, - "beam_width": 1, - "beam_update_prob": 1.0, } - ner2 = EntityRecognizer(vocab, default_ner(), **config) + ner2 = nlp1.create_pipe("ner", config=config) ner2.from_disk(output_dir) assert len(ner2.labels) == 2 @@ -170,7 +158,6 @@ def test_issue4054(en_vocab): vocab_dir.mkdir() vocab1.to_disk(vocab_dir) vocab2 = Vocab().from_disk(vocab_dir) - print("lang", vocab2.lang) nlp2 = spacy.blank("en", vocab=vocab2) nlp_dir = ensure_path(d / "nlp") if not nlp_dir.exists(): @@ -262,9 +249,8 @@ def test_issue4190(): def test_issue4267(): """ Test that running an entity_ruler after ner gives consistent results""" nlp = English() - ner = nlp.create_pipe("ner") + ner = nlp.add_pipe("ner") ner.add_label("PEOPLE") - nlp.add_pipe(ner) nlp.begin_training() assert "ner" in nlp.pipe_names # assert that we have correct IOB annotations @@ -273,10 +259,9 @@ def test_issue4267(): for token in doc1: assert token.ent_iob == 2 # add entity ruler and run again - ruler = EntityRuler(nlp) patterns = [{"label": "SOFTWARE", "pattern": "spacy"}] + ruler = nlp.add_pipe("entity_ruler") ruler.add_patterns(patterns) - nlp.add_pipe(ruler) assert "entity_ruler" in nlp.pipe_names assert "ner" in nlp.pipe_names # assert that we still have correct IOB annotations @@ -320,14 +305,10 @@ def test_issue4313(): config = { "learn_tokens": False, "min_action_freq": 30, - "beam_width": 1, - "beam_update_prob": 1.0, } - ner = EntityRecognizer(nlp.vocab, default_ner(), **config) + ner = nlp.create_pipe("ner", config=config) ner.add_label("SOME_LABEL") ner.begin_training([]) - nlp.add_pipe(ner) - # add a new label to the doc doc = nlp("What do you think about Apple ?") assert len(ner.labels) == 1 @@ -354,8 +335,7 @@ def test_issue4348(): nlp = English() example = Example.from_dict(nlp.make_doc(""), {"tags": []}) TRAIN_DATA = [example, example] - tagger = nlp.create_pipe("tagger") - nlp.add_pipe(tagger) + nlp.add_pipe("tagger") optimizer = nlp.begin_training() for i in range(5): losses = {} diff --git a/spacy/tests/regression/test_issue4501-5000.py b/spacy/tests/regression/test_issue4501-5000.py index 01d7a1dbb..08a21e690 100644 --- a/spacy/tests/regression/test_issue4501-5000.py +++ b/spacy/tests/regression/test_issue4501-5000.py @@ -1,6 +1,5 @@ import pytest from mock import Mock -from spacy.pipeline import EntityRuler from spacy.matcher import DependencyMatcher from spacy.tokens import Doc, Span, DocBin from spacy.gold import Example @@ -72,18 +71,16 @@ def test_issue4651_with_phrase_matcher_attr(): """ text = "Spacy is a python library for nlp" nlp = English() - ruler = EntityRuler(nlp, phrase_matcher_attr="LOWER") patterns = [{"label": "PYTHON_LIB", "pattern": "spacy", "id": "spaCy"}] + ruler = nlp.add_pipe("entity_ruler", config={"phrase_matcher_attr": "LOWER"}) ruler.add_patterns(patterns) - nlp.add_pipe(ruler) doc = nlp(text) res = [(ent.text, ent.label_, ent.ent_id_) for ent in doc.ents] nlp_reloaded = English() with make_tempdir() as d: file_path = d / "entityruler" ruler.to_disk(file_path) - ruler_reloaded = EntityRuler(nlp_reloaded).from_disk(file_path) - nlp_reloaded.add_pipe(ruler_reloaded) + nlp_reloaded.add_pipe("entity_ruler").from_disk(file_path) doc_reloaded = nlp_reloaded(text) res_reloaded = [(ent.text, ent.label_, ent.ent_id_) for ent in doc_reloaded.ents] assert res == res_reloaded @@ -96,18 +93,16 @@ def test_issue4651_without_phrase_matcher_attr(): """ text = "Spacy is a python library for nlp" nlp = English() - ruler = EntityRuler(nlp) patterns = [{"label": "PYTHON_LIB", "pattern": "spacy", "id": "spaCy"}] + ruler = nlp.add_pipe("entity_ruler") ruler.add_patterns(patterns) - nlp.add_pipe(ruler) doc = nlp(text) res = [(ent.text, ent.label_, ent.ent_id_) for ent in doc.ents] nlp_reloaded = English() with make_tempdir() as d: file_path = d / "entityruler" ruler.to_disk(file_path) - ruler_reloaded = EntityRuler(nlp_reloaded).from_disk(file_path) - nlp_reloaded.add_pipe(ruler_reloaded) + nlp_reloaded.add_pipe("entity_ruler").from_disk(file_path) doc_reloaded = nlp_reloaded(text) res_reloaded = [(ent.text, ent.label_, ent.ent_id_) for ent in doc_reloaded.ents] assert res == res_reloaded @@ -171,8 +166,8 @@ def test_issue4707(): by default when loading a model. """ nlp = English() - nlp.add_pipe(nlp.create_pipe("sentencizer")) - nlp.add_pipe(nlp.create_pipe("entity_ruler")) + nlp.add_pipe("sentencizer") + nlp.add_pipe("entity_ruler") assert nlp.pipe_names == ["sentencizer", "entity_ruler"] exclude = ["tokenizer", "sentencizer"] with make_tempdir() as tmpdir: @@ -187,21 +182,29 @@ def test_issue4725_1(): """ Ensure the pickling of the NER goes well""" vocab = Vocab(vectors_name="test_vocab_add_vector") nlp = English(vocab=vocab) - ner = nlp.create_pipe("ner", config={"min_action_freq": 342}) + config = { + "learn_tokens": False, + "min_action_freq": 342, + "update_with_oracle_cut_size": 111, + } + ner = nlp.create_pipe("ner", config=config) with make_tempdir() as tmp_path: with (tmp_path / "ner.pkl").open("wb") as file_: pickle.dump(ner, file_) assert ner.cfg["min_action_freq"] == 342 + assert ner.cfg["update_with_oracle_cut_size"] == 111 with (tmp_path / "ner.pkl").open("rb") as file_: ner2 = pickle.load(file_) assert ner2.cfg["min_action_freq"] == 342 + assert ner2.cfg["update_with_oracle_cut_size"] == 111 @pytest.mark.filterwarnings("ignore::UserWarning") def test_issue4725_2(): # ensures that this runs correctly and doesn't hang or crash because of the global vectors - # if it does crash, it's usually because of calling 'spawn' for multiprocessing (e.g. on Windows) + # if it does crash, it's usually because of calling 'spawn' for multiprocessing (e.g. on Windows), + # or because of issues with pickling the NER (cf test_issue4725_1) vocab = Vocab(vectors_name="test_vocab_add_vector") data = numpy.ndarray((5, 3), dtype="f") data[0] = 1.0 @@ -209,8 +212,7 @@ def test_issue4725_2(): vocab.set_vector("cat", data[0]) vocab.set_vector("dog", data[1]) nlp = English(vocab=vocab) - ner = nlp.create_pipe("ner") - nlp.add_pipe(ner) + nlp.add_pipe("ner") nlp.begin_training() docs = ["Kurt is in London."] * 10 for _ in nlp.pipe(docs, batch_size=2, n_process=2): @@ -219,15 +221,12 @@ def test_issue4725_2(): def test_issue4849(): nlp = English() - ruler = EntityRuler( - nlp, - patterns=[ - {"label": "PERSON", "pattern": "joe biden", "id": "joe-biden"}, - {"label": "PERSON", "pattern": "bernie sanders", "id": "bernie-sanders"}, - ], - phrase_matcher_attr="LOWER", - ) - nlp.add_pipe(ruler) + patterns = [ + {"label": "PERSON", "pattern": "joe biden", "id": "joe-biden"}, + {"label": "PERSON", "pattern": "bernie sanders", "id": "bernie-sanders"}, + ] + ruler = nlp.add_pipe("entity_ruler", config={"phrase_matcher_attr": "LOWER"}) + ruler.add_patterns(patterns) text = """ The left is starting to take aim at Democratic front-runner Joe Biden. Sen. Bernie Sanders joined in her criticism: "There is no 'middle ground' when it comes to climate policy." @@ -244,10 +243,10 @@ def test_issue4849(): assert count_ents == 2 +@Language.factory("my_pipe") class CustomPipe: - name = "my_pipe" - - def __init__(self): + def __init__(self, nlp, name="my_pipe"): + self.name = name Span.set_extension("my_ext", getter=self._get_my_ext) Doc.set_extension("my_ext", default=None) @@ -259,7 +258,6 @@ class CustomPipe: gathered_ext.append(sent_ext) doc._.set("my_ext", "\n".join(gathered_ext)) - return doc @staticmethod @@ -271,10 +269,8 @@ def test_issue4903(): """Ensure that this runs correctly and doesn't hang or crash on Windows / macOS.""" nlp = English() - custom_component = CustomPipe() - nlp.add_pipe(nlp.create_pipe("sentencizer")) - nlp.add_pipe(custom_component, after="sentencizer") - + nlp.add_pipe("sentencizer") + nlp.add_pipe("my_pipe", after="sentencizer") text = ["I like bananas.", "Do you like them?", "No, I prefer wasabi."] docs = list(nlp.pipe(text, n_process=2)) assert docs[0].text == "I like bananas." diff --git a/spacy/tests/regression/test_issue5082.py b/spacy/tests/regression/test_issue5082.py index 52a52b177..76f3a552e 100644 --- a/spacy/tests/regression/test_issue5082.py +++ b/spacy/tests/regression/test_issue5082.py @@ -1,6 +1,5 @@ import numpy as np from spacy.lang.en import English -from spacy.pipeline import EntityRuler def test_issue5082(): @@ -19,23 +18,18 @@ def test_issue5082(): vocab.set_vector("Bowie", array4) text = "I like David Bowie" - ruler = EntityRuler(nlp) patterns = [ {"label": "PERSON", "pattern": [{"LOWER": "david"}, {"LOWER": "bowie"}]} ] + ruler = nlp.add_pipe("entity_ruler") ruler.add_patterns(patterns) - nlp.add_pipe(ruler) - parsed_vectors_1 = [t.vector for t in nlp(text)] assert len(parsed_vectors_1) == 4 np.testing.assert_array_equal(parsed_vectors_1[0], array1) np.testing.assert_array_equal(parsed_vectors_1[1], array2) np.testing.assert_array_equal(parsed_vectors_1[2], array3) np.testing.assert_array_equal(parsed_vectors_1[3], array4) - - merge_ents = nlp.create_pipe("merge_entities") - nlp.add_pipe(merge_ents) - + nlp.add_pipe("merge_entities") parsed_vectors_2 = [t.vector for t in nlp(text)] assert len(parsed_vectors_2) == 3 np.testing.assert_array_equal(parsed_vectors_2[0], array1) diff --git a/spacy/tests/regression/test_issue5137.py b/spacy/tests/regression/test_issue5137.py index b621b5faa..095ca8495 100644 --- a/spacy/tests/regression/test_issue5137.py +++ b/spacy/tests/regression/test_issue5137.py @@ -5,12 +5,12 @@ from spacy.tests.util import make_tempdir def test_issue5137(): + @Language.factory("my_component") class MyComponent: - name = "my_component" - - def __init__(self, nlp, **cfg): + def __init__(self, nlp, name="my_component", categories="all_categories"): self.nlp = nlp - self.categories = cfg.get("categories", "all_categories") + self.categories = categories + self.name = name def __call__(self, doc): pass @@ -21,14 +21,12 @@ def test_issue5137(): def from_disk(self, path, **cfg): pass - factory = lambda nlp, model, **cfg: MyComponent(nlp, **cfg) - Language.factories["my_component"] = factory - nlp = English() - nlp.add_pipe(nlp.create_pipe("my_component")) - assert nlp.get_pipe("my_component").categories == "all_categories" + my_component = nlp.add_pipe("my_component") + assert my_component.categories == "all_categories" with make_tempdir() as tmpdir: nlp.to_disk(tmpdir) - nlp2 = spacy.load(tmpdir, categories="my_categories") + overrides = {"my_component": {"categories": "my_categories"}} + nlp2 = spacy.load(tmpdir, component_cfg=overrides) assert nlp2.get_pipe("my_component").categories == "my_categories" diff --git a/spacy/tests/regression/test_issue5230.py b/spacy/tests/regression/test_issue5230.py index d634ee35c..ae9ed1844 100644 --- a/spacy/tests/regression/test_issue5230.py +++ b/spacy/tests/regression/test_issue5230.py @@ -7,7 +7,7 @@ from spacy.kb import KnowledgeBase, Writer from spacy.vectors import Vectors from spacy.language import Language from spacy.pipeline import Pipe - +from spacy.util import registry from ..util import make_tempdir @@ -58,8 +58,7 @@ def custom_pipe(): def tagger(): nlp = Language() - nlp.add_pipe(nlp.create_pipe("tagger")) - tagger = nlp.get_pipe("tagger") + tagger = nlp.add_pipe("tagger") # need to add model for two reasons: # 1. no model leads to error in serialization, # 2. the affected line is the one for model serialization @@ -70,10 +69,15 @@ def tagger(): def entity_linker(): nlp = Language() - kb = KnowledgeBase(nlp.vocab, entity_vector_length=1) - kb.add_entity("test", 0.0, zeros((1, 1), dtype="f")) - nlp.add_pipe(nlp.create_pipe("entity_linker", {"kb": kb})) - entity_linker = nlp.get_pipe("entity_linker") + + @registry.assets.register("TestIssue5230KB.v1") + def dummy_kb() -> KnowledgeBase: + kb = KnowledgeBase(nlp.vocab, entity_vector_length=1) + kb.add_entity("test", 0.0, zeros((1, 1), dtype="f")) + return kb + + config = {"kb": {"@assets": "TestIssue5230KB.v1"}} + entity_linker = nlp.add_pipe("entity_linker", config=config) # need to add model for two reasons: # 1. no model leads to error in serialization, # 2. the affected line is the one for model serialization diff --git a/spacy/tests/regression/test_issue5551.py b/spacy/tests/regression/test_issue5551.py index a8be4cab4..b7139d463 100644 --- a/spacy/tests/regression/test_issue5551.py +++ b/spacy/tests/regression/test_issue5551.py @@ -5,7 +5,14 @@ from spacy.util import fix_random_seed def test_issue5551(): """Test that after fixing the random seed, the results of the pipeline are truly identical""" component = "textcat" - pipe_cfg = {"exclusive_classes": False} + pipe_cfg = { + "model": { + "@architectures": "spacy.TextCatBOW.v1", + "exclusive_classes": True, + "ngram_size": 2, + "no_output_layer": False, + } + } results = [] for i in range(3): @@ -15,11 +22,10 @@ def test_issue5551(): "Once hot, form ping-pong-ball-sized balls of the mixture, each weighing roughly 25 g.", {"cats": {"Labe1": 1.0, "Label2": 0.0, "Label3": 0.0}}, ) - nlp.add_pipe(nlp.create_pipe(component, config=pipe_cfg), last=True) - pipe = nlp.get_pipe(component) + pipe = nlp.add_pipe(component, config=pipe_cfg, last=True) for label in set(example[1]["cats"]): pipe.add_label(label) - nlp.begin_training(component_cfg={component: pipe_cfg}) + nlp.begin_training() # Store the result of each iteration result = pipe.model.predict([nlp.make_doc(example[0])]) diff --git a/spacy/tests/serialize/test_serialize_config.py b/spacy/tests/serialize/test_serialize_config.py index 85346de90..7d149a92e 100644 --- a/spacy/tests/serialize/test_serialize_config.py +++ b/spacy/tests/serialize/test_serialize_config.py @@ -1,23 +1,28 @@ -from thinc.api import Config - +import pytest +from thinc.config import Config, ConfigValidationError import spacy -from spacy import util from spacy.lang.en import English -from spacy.util import registry +from spacy.language import Language +from spacy.util import registry, deep_merge_configs, load_model_from_config +from spacy.ml.models import build_Tok2Vec_model, build_tb_parser_model from ..util import make_tempdir -from ...ml.models import build_Tok2Vec_model, build_tb_parser_model + nlp_config_string = """ +[training] +batch_size = 666 + [nlp] lang = "en" +pipeline = ["tok2vec", "tagger"] -[nlp.pipeline] +[components] -[nlp.pipeline.tok2vec] -factory = "tok2vec" +[components.tok2vec] +@factories = "tok2vec" -[nlp.pipeline.tok2vec.model] +[components.tok2vec.model] @architectures = "spacy.HashEmbedCNN.v1" pretrained_vectors = null width = 342 @@ -28,15 +33,15 @@ maxout_pieces = 3 subword_features = true dropout = null -[nlp.pipeline.tagger] -factory = "tagger" +[components.tagger] +@factories = "tagger" -[nlp.pipeline.tagger.model] +[components.tagger.model] @architectures = "spacy.Tagger.v1" -[nlp.pipeline.tagger.model.tok2vec] +[components.tagger.model.tok2vec] @architectures = "spacy.Tok2VecTensors.v1" -width = ${nlp.pipeline.tok2vec.model:width} +width = ${components.tok2vec.model:width} """ @@ -82,10 +87,52 @@ def my_parser(): return parser +def test_create_nlp_from_config(): + config = Config().from_str(nlp_config_string) + with pytest.raises(ConfigValidationError): + nlp, _ = load_model_from_config(config, auto_fill=False) + nlp, resolved = load_model_from_config(config, auto_fill=True) + assert nlp.config["training"]["batch_size"] == 666 + assert len(nlp.config["training"]) > 1 + assert nlp.pipe_names == ["tok2vec", "tagger"] + assert len(nlp.config["components"]) == 2 + assert len(nlp.config["nlp"]["pipeline"]) == 2 + nlp.remove_pipe("tagger") + assert len(nlp.config["components"]) == 1 + assert len(nlp.config["nlp"]["pipeline"]) == 1 + with pytest.raises(ValueError): + bad_cfg = {"yolo": {}} + load_model_from_config(Config(bad_cfg), auto_fill=True) + with pytest.raises(ValueError): + bad_cfg = {"pipeline": {"foo": "bar"}} + load_model_from_config(Config(bad_cfg), auto_fill=True) + + +def test_create_nlp_from_config_multiple_instances(): + """Test that the nlp object is created correctly for a config with multiple + instances of the same component.""" + config = Config().from_str(nlp_config_string) + config["components"] = { + "t2v": config["components"]["tok2vec"], + "tagger1": config["components"]["tagger"], + "tagger2": config["components"]["tagger"], + } + config["nlp"]["pipeline"] = list(config["components"].keys()) + nlp, _ = load_model_from_config(config, auto_fill=True) + assert nlp.pipe_names == ["t2v", "tagger1", "tagger2"] + assert nlp.get_pipe_meta("t2v").factory == "tok2vec" + assert nlp.get_pipe_meta("tagger1").factory == "tagger" + assert nlp.get_pipe_meta("tagger2").factory == "tagger" + pipeline_config = nlp.config["components"] + assert len(pipeline_config) == 3 + assert list(pipeline_config.keys()) == ["t2v", "tagger1", "tagger2"] + assert nlp.config["nlp"]["pipeline"] == ["t2v", "tagger1", "tagger2"] + + def test_serialize_nlp(): """ Create a custom nlp pipeline from config and ensure it serializes it correctly """ nlp_config = Config().from_str(nlp_config_string) - nlp = util.load_model_from_config(nlp_config["nlp"]) + nlp, _ = load_model_from_config(nlp_config, auto_fill=True) nlp.begin_training() assert "tok2vec" in nlp.pipe_names assert "tagger" in nlp.pipe_names @@ -106,17 +153,15 @@ def test_serialize_custom_nlp(): nlp = English() parser_cfg = dict() parser_cfg["model"] = {"@architectures": "my_test_parser"} - parser = nlp.create_pipe("parser", parser_cfg) - nlp.add_pipe(parser) + nlp.add_pipe("parser", config=parser_cfg) nlp.begin_training() with make_tempdir() as d: nlp.to_disk(d) nlp2 = spacy.load(d) model = nlp2.get_pipe("parser").model - tok2vec = model.get_ref("tok2vec") + model.get_ref("tok2vec") upper = model.get_ref("upper") - # check that we have the correct settings, not the default ones assert upper.get_dim("nI") == 65 @@ -125,17 +170,104 @@ def test_serialize_parser(): """ Create a non-default parser config to check nlp serializes it correctly """ nlp = English() model_config = Config().from_str(parser_config_string) - parser = nlp.create_pipe("parser", config=model_config) + parser = nlp.add_pipe("parser", config=model_config) parser.add_label("nsubj") - nlp.add_pipe(parser) nlp.begin_training() with make_tempdir() as d: nlp.to_disk(d) nlp2 = spacy.load(d) model = nlp2.get_pipe("parser").model - tok2vec = model.get_ref("tok2vec") + model.get_ref("tok2vec") upper = model.get_ref("upper") - # check that we have the correct settings, not the default ones assert upper.get_dim("nI") == 66 + + +def test_deep_merge_configs(): + config = {"a": "hello", "b": {"c": "d"}} + defaults = {"a": "world", "b": {"c": "e", "f": "g"}} + merged = deep_merge_configs(config, defaults) + assert len(merged) == 2 + assert merged["a"] == "hello" + assert merged["b"] == {"c": "d", "f": "g"} + config = {"a": "hello", "b": {"@test": "x", "foo": 1}} + defaults = {"a": "world", "b": {"@test": "x", "foo": 100, "bar": 2}, "c": 100} + merged = deep_merge_configs(config, defaults) + assert len(merged) == 3 + assert merged["a"] == "hello" + assert merged["b"] == {"@test": "x", "foo": 1, "bar": 2} + assert merged["c"] == 100 + config = {"a": "hello", "b": {"@test": "x", "foo": 1}, "c": 100} + defaults = {"a": "world", "b": {"@test": "y", "foo": 100, "bar": 2}} + merged = deep_merge_configs(config, defaults) + assert len(merged) == 3 + assert merged["a"] == "hello" + assert merged["b"] == {"@test": "x", "foo": 1} + assert merged["c"] == 100 + # Test that leaving out the factory just adds to existing + config = {"a": "hello", "b": {"foo": 1}, "c": 100} + defaults = {"a": "world", "b": {"@test": "y", "foo": 100, "bar": 2}} + merged = deep_merge_configs(config, defaults) + assert len(merged) == 3 + assert merged["a"] == "hello" + assert merged["b"] == {"@test": "y", "foo": 1, "bar": 2} + assert merged["c"] == 100 + + +def test_config_nlp_roundtrip(): + """Test that a config prduced by the nlp object passes training config + validation.""" + nlp = English() + nlp.add_pipe("entity_ruler") + nlp.add_pipe("ner") + new_nlp, new_config = load_model_from_config(nlp.config, auto_fill=False) + assert new_nlp.config == nlp.config + assert new_nlp.pipe_names == nlp.pipe_names + assert new_nlp._pipe_configs == nlp._pipe_configs + assert new_nlp._pipe_meta == nlp._pipe_meta + assert new_nlp._factory_meta == nlp._factory_meta + + +def test_serialize_config_language_specific(): + """Test that config serialization works as expected with language-specific + factories.""" + name = "test_serialize_config_language_specific" + + @English.factory(name, default_config={"foo": 20}) + def custom_factory(nlp: Language, name: str, foo: int): + return lambda doc: doc + + nlp = Language() + assert not nlp.has_factory(name) + nlp = English() + assert nlp.has_factory(name) + nlp.add_pipe(name, config={"foo": 100}, name="bar") + pipe_config = nlp.config["components"]["bar"] + assert pipe_config["foo"] == 100 + assert pipe_config["@factories"] == name + + with make_tempdir() as d: + nlp.to_disk(d) + nlp2 = spacy.load(d) + assert nlp2.has_factory(name) + assert nlp2.pipe_names == ["bar"] + assert nlp2.get_pipe_meta("bar").factory == name + pipe_config = nlp2.config["components"]["bar"] + assert pipe_config["foo"] == 100 + assert pipe_config["@factories"] == name + + config = Config().from_str(nlp2.config.to_str()) + config["nlp"]["lang"] = "de" + with pytest.raises(ValueError): + # German doesn't have a factory, only English does + load_model_from_config(config) + + +def test_serialize_config_missing_pipes(): + config = Config().from_str(nlp_config_string) + config["components"].pop("tok2vec") + assert "tok2vec" in config["nlp"]["pipeline"] + assert "tok2vec" not in config["components"] + with pytest.raises(ValueError): + load_model_from_config(config, auto_fill=True) diff --git a/spacy/tests/serialize/test_serialize_pipeline.py b/spacy/tests/serialize/test_serialize_pipeline.py index 0f6a8853c..14a4579be 100644 --- a/spacy/tests/serialize/test_serialize_pipeline.py +++ b/spacy/tests/serialize/test_serialize_pipeline.py @@ -1,8 +1,11 @@ import pytest +from spacy import registry from spacy.pipeline import Tagger, DependencyParser, EntityRecognizer from spacy.pipeline import TextCategorizer, SentenceRecognizer -from spacy.pipeline.defaults import default_parser, default_tagger -from spacy.pipeline.defaults import default_textcat, default_senter +from spacy.pipeline.dep_parser import DEFAULT_PARSER_MODEL +from spacy.pipeline.tagger import DEFAULT_TAGGER_MODEL +from spacy.pipeline.textcat import DEFAULT_TEXTCAT_MODEL +from spacy.pipeline.senter import DEFAULT_SENTER_MODEL from ..util import make_tempdir @@ -15,32 +18,44 @@ def parser(en_vocab): config = { "learn_tokens": False, "min_action_freq": 30, - "beam_width": 1, - "beam_update_prob": 1.0, + "update_with_oracle_cut_size": 100, } - parser = DependencyParser(en_vocab, default_parser(), **config) + model = registry.make_from_config({"model": DEFAULT_PARSER_MODEL}, validate=True)["model"] + parser = DependencyParser(en_vocab, model, **config) parser.add_label("nsubj") return parser @pytest.fixture def blank_parser(en_vocab): - parser = DependencyParser(en_vocab, default_parser()) + config = { + "learn_tokens": False, + "min_action_freq": 30, + "update_with_oracle_cut_size": 100, + } + model = registry.make_from_config({"model": DEFAULT_PARSER_MODEL}, validate=True)["model"] + parser = DependencyParser(en_vocab, model, **config) return parser @pytest.fixture def taggers(en_vocab): - model = default_tagger() - tagger1 = Tagger(en_vocab, model) - tagger2 = Tagger(en_vocab, model) + model = registry.make_from_config({"model": DEFAULT_TAGGER_MODEL}, validate=True)["model"] + tagger1 = Tagger(en_vocab, model, set_morphology=True) + tagger2 = Tagger(en_vocab, model, set_morphology=True) return tagger1, tagger2 @pytest.mark.parametrize("Parser", test_parsers) def test_serialize_parser_roundtrip_bytes(en_vocab, Parser): - parser = Parser(en_vocab, default_parser()) - new_parser = Parser(en_vocab, default_parser()) + config = { + "learn_tokens": False, + "min_action_freq": 0, + "update_with_oracle_cut_size": 100, + } + model = registry.make_from_config({"model": DEFAULT_PARSER_MODEL}, validate=True)["model"] + parser = Parser(en_vocab, model, **config) + new_parser = Parser(en_vocab, model, **config) new_parser = new_parser.from_bytes(parser.to_bytes(exclude=["vocab"])) bytes_2 = new_parser.to_bytes(exclude=["vocab"]) bytes_3 = parser.to_bytes(exclude=["vocab"]) @@ -50,11 +65,17 @@ def test_serialize_parser_roundtrip_bytes(en_vocab, Parser): @pytest.mark.parametrize("Parser", test_parsers) def test_serialize_parser_roundtrip_disk(en_vocab, Parser): - parser = Parser(en_vocab, default_parser()) + config = { + "learn_tokens": False, + "min_action_freq": 0, + "update_with_oracle_cut_size": 100, + } + model = registry.make_from_config({"model": DEFAULT_PARSER_MODEL}, validate=True)["model"] + parser = Parser(en_vocab, model, **config) with make_tempdir() as d: file_path = d / "parser" parser.to_disk(file_path) - parser_d = Parser(en_vocab, default_parser()) + parser_d = Parser(en_vocab, model, **config) parser_d = parser_d.from_disk(file_path) parser_bytes = parser.to_bytes(exclude=["model", "vocab"]) parser_d_bytes = parser_d.to_bytes(exclude=["model", "vocab"]) @@ -83,7 +104,8 @@ def test_serialize_tagger_roundtrip_bytes(en_vocab, taggers): tagger1_b = tagger1.to_bytes() tagger1 = tagger1.from_bytes(tagger1_b) assert tagger1.to_bytes() == tagger1_b - new_tagger1 = Tagger(en_vocab, default_tagger()).from_bytes(tagger1_b) + model = registry.make_from_config({"model": DEFAULT_TAGGER_MODEL}, validate=True)["model"] + new_tagger1 = Tagger(en_vocab, model).from_bytes(tagger1_b) new_tagger1_b = new_tagger1.to_bytes() assert len(new_tagger1_b) == len(tagger1_b) assert new_tagger1_b == tagger1_b @@ -96,26 +118,34 @@ def test_serialize_tagger_roundtrip_disk(en_vocab, taggers): file_path2 = d / "tagger2" tagger1.to_disk(file_path1) tagger2.to_disk(file_path2) - tagger1_d = Tagger(en_vocab, default_tagger()).from_disk(file_path1) - tagger2_d = Tagger(en_vocab, default_tagger()).from_disk(file_path2) + model = registry.make_from_config({"model": DEFAULT_TAGGER_MODEL}, validate=True)["model"] + tagger1_d = Tagger(en_vocab, model, set_morphology=True).from_disk(file_path1) + tagger2_d = Tagger(en_vocab, model, set_morphology=True).from_disk(file_path2) assert tagger1_d.to_bytes() == tagger2_d.to_bytes() def test_serialize_textcat_empty(en_vocab): # See issue #1105 + model = registry.make_from_config({"model": DEFAULT_TEXTCAT_MODEL}, validate=True)["model"] textcat = TextCategorizer( - en_vocab, default_textcat(), labels=["ENTITY", "ACTION", "MODIFIER"] + en_vocab, model, labels=["ENTITY", "ACTION", "MODIFIER"] ) textcat.to_bytes(exclude=["vocab"]) @pytest.mark.parametrize("Parser", test_parsers) def test_serialize_pipe_exclude(en_vocab, Parser): + model = registry.make_from_config({"model": DEFAULT_PARSER_MODEL}, validate=True)["model"] + config = { + "learn_tokens": False, + "min_action_freq": 0, + "update_with_oracle_cut_size": 100, + } def get_new_parser(): - new_parser = Parser(en_vocab, default_parser()) + new_parser = Parser(en_vocab, model, **config) return new_parser - parser = Parser(en_vocab, default_parser()) + parser = Parser(en_vocab, model, **config) parser.cfg["foo"] = "bar" new_parser = get_new_parser().from_bytes(parser.to_bytes(exclude=["vocab"])) assert "foo" in new_parser.cfg @@ -130,7 +160,8 @@ def test_serialize_pipe_exclude(en_vocab, Parser): def test_serialize_sentencerecognizer(en_vocab): - sr = SentenceRecognizer(en_vocab, default_senter()) + model = registry.make_from_config({"model": DEFAULT_SENTER_MODEL}, validate=True)["model"] + sr = SentenceRecognizer(en_vocab, model) sr_b = sr.to_bytes() - sr_d = SentenceRecognizer(en_vocab, default_senter()).from_bytes(sr_b) + sr_d = SentenceRecognizer(en_vocab, model).from_bytes(sr_b) assert sr.to_bytes() == sr_d.to_bytes() diff --git a/spacy/tests/serialize/test_serialize_tokenizer.py b/spacy/tests/serialize/test_serialize_tokenizer.py index a0c36c2a6..00a88ec38 100644 --- a/spacy/tests/serialize/test_serialize_tokenizer.py +++ b/spacy/tests/serialize/test_serialize_tokenizer.py @@ -6,7 +6,7 @@ from ..util import make_tempdir, assert_packed_msg_equal def load_tokenizer(b): - tok = get_lang_class("en").Defaults.create_tokenizer() + tok = get_lang_class("en")().tokenizer tok.from_bytes(b) return tok diff --git a/spacy/tests/test_gold.py b/spacy/tests/test_gold.py index 7fb7a1000..b03765857 100644 --- a/spacy/tests/test_gold.py +++ b/spacy/tests/test_gold.py @@ -5,7 +5,6 @@ from spacy.gold import Corpus, docs_to_json from spacy.gold.example import Example from spacy.gold.converters import json2docs from spacy.lang.en import English -from spacy.pipeline import EntityRuler from spacy.tokens import Doc, DocBin from spacy.util import get_words_and_spaces, minibatch from thinc.api import compounding @@ -18,60 +17,20 @@ from ..gold.augment import make_orth_variants_example @pytest.fixture def doc(): + # fmt: off text = "Sarah's sister flew to Silicon Valley via London." tags = ["NNP", "POS", "NN", "VBD", "IN", "NNP", "NNP", "IN", "NNP", "."] - pos = [ - "PROPN", - "PART", - "NOUN", - "VERB", - "ADP", - "PROPN", - "PROPN", - "ADP", - "PROPN", - "PUNCT", - ] - morphs = [ - "NounType=prop|Number=sing", - "Poss=yes", - "Number=sing", - "Tense=past|VerbForm=fin", - "", - "NounType=prop|Number=sing", - "NounType=prop|Number=sing", - "", - "NounType=prop|Number=sing", - "PunctType=peri", - ] + pos = ["PROPN", "PART", "NOUN", "VERB", "ADP", "PROPN", "PROPN", "ADP", "PROPN", "PUNCT"] + morphs = ["NounType=prop|Number=sing", "Poss=yes", "Number=sing", "Tense=past|VerbForm=fin", + "", "NounType=prop|Number=sing", "NounType=prop|Number=sing", "", + "NounType=prop|Number=sing", "PunctType=peri"] # head of '.' is intentionally nonprojective for testing heads = [2, 0, 3, 3, 3, 6, 4, 3, 7, 5] - deps = [ - "poss", - "case", - "nsubj", - "ROOT", - "prep", - "compound", - "pobj", - "prep", - "pobj", - "punct", - ] - lemmas = [ - "Sarah", - "'s", - "sister", - "fly", - "to", - "Silicon", - "Valley", - "via", - "London", - ".", - ] + deps = ["poss", "case", "nsubj", "ROOT", "prep", "compound", "pobj", "prep", "pobj", "punct"] + lemmas = ["Sarah", "'s", "sister", "fly", "to", "Silicon", "Valley", "via", "London", "."] biluo_tags = ["U-PERSON", "O", "O", "O", "O", "B-LOC", "L-LOC", "O", "U-GPE", "O"] cats = {"TRAVEL": 1.0, "BAKING": 0.0} + # fmt: on nlp = English() doc = nlp(text) for i in range(len(tags)): @@ -308,7 +267,9 @@ def test_gold_biluo_one_to_many(en_vocab, en_tokenizer): (len("Mr and "), len("Mr and Mrs Smith"), "PERSON"), # "Mrs Smith" is a PERSON (len(prefix), len(prefix + "San Francisco Valley"), "LOC"), ] + # fmt: off gold_words = ["Mr and", "Mrs", "Smith", "flew", "to", "San", "Francisco", "Valley", "."] + # fmt: on example = Example.from_dict(doc, {"words": gold_words, "entities": entities}) ner_tags = example.get_aligned_ner() assert ner_tags == ["O", "U-PERSON", "O", "U-LOC", "O"] @@ -317,7 +278,9 @@ def test_gold_biluo_one_to_many(en_vocab, en_tokenizer): (len("Mr and "), len("Mr and Mrs"), "PERSON"), # "Mrs" is a Person (len(prefix), len(prefix + "San Francisco Valley"), "LOC"), ] + # fmt: off gold_words = ["Mr and", "Mrs", "Smith", "flew", "to", "San", "Francisco", "Valley", "."] + # fmt: on example = Example.from_dict(doc, {"words": gold_words, "entities": entities}) ner_tags = example.get_aligned_ner() assert ner_tags == ["O", None, "O", "U-LOC", "O"] @@ -341,7 +304,8 @@ def test_gold_biluo_many_to_one(en_vocab, en_tokenizer): gold_words = ["Mr and", "Mrs Smith", "flew to", "San Francisco Valley", "."] example = Example.from_dict(doc, {"words": gold_words, "entities": entities}) ner_tags = example.get_aligned_ner() - assert ner_tags == ["O", "B-PERSON", "L-PERSON", "O", "O", "B-LOC", "I-LOC", "L-LOC", "O"] + expected = ["O", "B-PERSON", "L-PERSON", "O", "O", "B-LOC", "I-LOC", "L-LOC", "O"] + assert ner_tags == expected def test_gold_biluo_misaligned(en_vocab, en_tokenizer): @@ -438,7 +402,9 @@ def test_aligned_spans_y2x(en_vocab, en_tokenizer): (0, len("Mr and Mrs Smith"), "PERSON"), (len(prefix), len(prefix + "San Francisco Valley"), "LOC"), ] + # fmt: off tokens_ref = ["Mr", "and", "Mrs", "Smith", "flew", "to", "San", "Francisco", "Valley", "."] + # fmt: on example = Example.from_dict(doc, {"words": tokens_ref, "entities": entities}) ents_ref = example.reference.ents assert [(ent.start, ent.end) for ent in ents_ref] == [(0, 4), (6, 9)] @@ -449,11 +415,12 @@ def test_aligned_spans_y2x(en_vocab, en_tokenizer): def test_aligned_spans_x2y(en_vocab, en_tokenizer): text = "Mr and Mrs Smith flew to San Francisco Valley" nlp = English() - ruler = EntityRuler(nlp) - patterns = [{"label": "PERSON", "pattern": "Mr and Mrs Smith"}, - {"label": "LOC", "pattern": "San Francisco Valley"}] + patterns = [ + {"label": "PERSON", "pattern": "Mr and Mrs Smith"}, + {"label": "LOC", "pattern": "San Francisco Valley"}, + ] + ruler = nlp.add_pipe("entity_ruler") ruler.add_patterns(patterns) - nlp.add_pipe(ruler) doc = nlp(text) assert [(ent.start, ent.end) for ent in doc.ents] == [(0, 4), (6, 9)] prefix = "Mr and Mrs Smith flew to " @@ -464,7 +431,6 @@ def test_aligned_spans_x2y(en_vocab, en_tokenizer): tokens_ref = ["Mr and Mrs", "Smith", "flew", "to", "San Francisco", "Valley"] example = Example.from_dict(doc, {"words": tokens_ref, "entities": entities}) assert [(ent.start, ent.end) for ent in example.reference.ents] == [(0, 2), (4, 6)] - # Ensure that 'get_aligned_spans_x2y' has the aligned entities correct ents_pred = example.predicted.ents assert [(ent.start, ent.end) for ent in ents_pred] == [(0, 4), (6, 9)] @@ -652,10 +618,9 @@ def test_tuple_format_implicit_invalid(): def _train_tuples(train_data): nlp = English() - ner = nlp.create_pipe("ner") + ner = nlp.add_pipe("ner") ner.add_label("ORG") ner.add_label("LOC") - nlp.add_pipe(ner) train_examples = [] for t in train_data: diff --git a/spacy/tests/test_language.py b/spacy/tests/test_language.py index 7b4c29c5a..a63a8e24c 100644 --- a/spacy/tests/test_language.py +++ b/spacy/tests/test_language.py @@ -3,6 +3,7 @@ import pytest from spacy.language import Language from spacy.tokens import Doc, Span from spacy.vocab import Vocab +from spacy.lang.en import English from .util import add_vecs_to_vocab, assert_docs_equal from ..gold import Example @@ -11,10 +12,9 @@ from ..gold import Example @pytest.fixture def nlp(): nlp = Language(Vocab()) - textcat = nlp.create_pipe("textcat") + textcat = nlp.add_pipe("textcat") for label in ("POSITIVE", "NEGATIVE"): textcat.add_label(label) - nlp.add_pipe(textcat) nlp.begin_training() return nlp @@ -70,6 +70,7 @@ def test_evaluate_no_pipe(nlp): """Test that docs are processed correctly within Language.pipe if the component doesn't expose a .pipe method.""" + @Language.component("test_evaluate_no_pipe") def pipe(doc): return doc @@ -77,20 +78,23 @@ def test_evaluate_no_pipe(nlp): annots = {"cats": {"POSITIVE": 1.0, "NEGATIVE": 0.0}} nlp = Language(Vocab()) doc = nlp(text) - nlp.add_pipe(pipe) + nlp.add_pipe("test_evaluate_no_pipe") nlp.evaluate([Example.from_dict(doc, annots)]) +@Language.component("test_language_vector_modification_pipe") def vector_modification_pipe(doc): doc.vector += 1 return doc +@Language.component("test_language_userdata_pipe") def userdata_pipe(doc): doc.user_data["foo"] = "bar" return doc +@Language.component("test_language_ner_pipe") def ner_pipe(doc): span = Span(doc, 0, 1, label="FIRST") doc.ents += (span,) @@ -109,9 +113,9 @@ def sample_vectors(): @pytest.fixture def nlp2(nlp, sample_vectors): add_vecs_to_vocab(nlp.vocab, sample_vectors) - nlp.add_pipe(vector_modification_pipe) - nlp.add_pipe(ner_pipe) - nlp.add_pipe(userdata_pipe) + nlp.add_pipe("test_language_vector_modification_pipe") + nlp.add_pipe("test_language_ner_pipe") + nlp.add_pipe("test_language_userdata_pipe") return nlp @@ -147,3 +151,8 @@ def test_language_pipe_stream(nlp2, n_process, texts): n_fetch = 20 for doc, expected_doc in itertools.islice(zip(docs, expecteds), n_fetch): assert_docs_equal(doc, expected_doc) + + +def test_language_from_config(): + English.from_config() + # TODO: add more tests diff --git a/spacy/tests/test_lemmatizer.py b/spacy/tests/test_lemmatizer.py index 19665b627..44f540132 100644 --- a/spacy/tests/test_lemmatizer.py +++ b/spacy/tests/test_lemmatizer.py @@ -35,10 +35,9 @@ def test_tagger_warns_no_lookups(): nlp = Language() nlp.vocab.lookups = Lookups() assert not len(nlp.vocab.lookups) - tagger = nlp.create_pipe("tagger") + tagger = nlp.add_pipe("tagger") with pytest.warns(UserWarning): tagger.begin_training() - nlp.add_pipe(tagger) with pytest.warns(UserWarning): nlp.begin_training() nlp.vocab.lookups.add_table("lemma_lookup") diff --git a/spacy/tests/test_misc.py b/spacy/tests/test_misc.py index f6724f632..e6ef45f90 100644 --- a/spacy/tests/test_misc.py +++ b/spacy/tests/test_misc.py @@ -138,3 +138,22 @@ def test_is_compatible_version(version, constraint, compatible): ) def test_is_unconstrained_version(constraint, expected): assert util.is_unconstrained_version(constraint) is expected + + +@pytest.mark.parametrize( + "dot_notation,expected", + [ + ( + {"token.pos": True, "token._.xyz": True}, + {"token": {"pos": True, "_": {"xyz": True}}}, + ), + ( + {"training.batch_size": 128, "training.optimizer.learn_rate": 0.01}, + {"training": {"batch_size": 128, "optimizer": {"learn_rate": 0.01}}}, + ), + ], +) +def test_dot_to_dict(dot_notation, expected): + result = util.dot_to_dict(dot_notation) + assert result == expected + assert util.dict_to_dot(result) == dot_notation diff --git a/spacy/tests/test_models.py b/spacy/tests/test_models.py index c3270c556..fc1988fcd 100644 --- a/spacy/tests/test_models.py +++ b/spacy/tests/test_models.py @@ -35,7 +35,7 @@ def get_gradient(model, Y): raise ValueError(f"Could not get gradient for type {type(Y)}") -def default_tok2vec(): +def test_tok2vec(): return build_Tok2Vec_model(**TOK2VEC_KWARGS) @@ -63,11 +63,11 @@ TEXTCAT_KWARGS = { "window_size": 1, "conv_depth": 2, "dropout": None, - "nO": 7 + "nO": 7, } TEXTCAT_CNN_KWARGS = { - "tok2vec": default_tok2vec(), + "tok2vec": test_tok2vec(), "exclusive_classes": False, "nO": 13, } @@ -114,7 +114,9 @@ def test_models_predict_consistently(seed, model_func, kwargs, get_X): tok2vec2 = model2.get_ref("tok2vec").predict(get_X()) for i in range(len(tok2vec1)): for j in range(len(tok2vec1[i])): - assert_array_equal(numpy.asarray(tok2vec1[i][j]), numpy.asarray(tok2vec2[i][j])) + assert_array_equal( + numpy.asarray(tok2vec1[i][j]), numpy.asarray(tok2vec2[i][j]) + ) if isinstance(Y1, numpy.ndarray): assert_array_equal(Y1, Y2) @@ -144,7 +146,7 @@ def test_models_update_consistently(seed, dropout, model_func, kwargs, get_X): for _ in range(5): Y, get_dX = model.begin_update(get_X()) dY = get_gradient(model, Y) - _ = get_dX(dY) + get_dX(dY) model.finish_update(optimizer) updated_params = get_all_params(model) with pytest.raises(AssertionError): diff --git a/spacy/tests/tokenizer/test_explain.py b/spacy/tests/tokenizer/test_explain.py index 3e7681234..ea6cf91be 100644 --- a/spacy/tests/tokenizer/test_explain.py +++ b/spacy/tests/tokenizer/test_explain.py @@ -54,7 +54,7 @@ LANGUAGES = [ @pytest.mark.parametrize("lang", LANGUAGES) def test_tokenizer_explain(lang): - tokenizer = get_lang_class(lang).Defaults.create_tokenizer() + tokenizer = get_lang_class(lang)().tokenizer examples = pytest.importorskip(f"spacy.lang.{lang}.examples") for sentence in examples.sentences: tokens = [t.text for t in tokenizer(sentence) if not t.is_space] diff --git a/spacy/tokenizer.pyx b/spacy/tokenizer.pyx index 203488609..114d227c8 100644 --- a/spacy/tokenizer.pyx +++ b/spacy/tokenizer.pyx @@ -1,4 +1,4 @@ -# cython: embedsignature=True, profile=True +# cython: embedsignature=True, profile=True, binding=True from __future__ import unicode_literals from cython.operator cimport dereference as deref @@ -9,6 +9,7 @@ from cymem.cymem cimport Pool from preshed.maps cimport PreshMap cimport cython +from typing import Dict, List, Union, Pattern, Optional import re import warnings @@ -20,10 +21,42 @@ from .attrs import intify_attrs from .symbols import ORTH from .errors import Errors, Warnings from . import util +from .util import registry from .attrs import intify_attrs from .symbols import ORTH +@registry.tokenizers("spacy.Tokenizer.v1") +def create_tokenizer( + # exceptions: Dict[str, List[dict]], + # prefixes: Optional[List[Union[str, Pattern]]], + # suffixes: Optional[List[Union[str, Pattern]]], + # infixes: Optional[List[Union[str, Pattern]]], + # token_match: Optional[Pattern], + # url_match: Optional[Pattern], +) -> "Tokenizer": + def tokenizer_factory(nlp): + exceptions = nlp.Defaults.tokenizer_exceptions + prefixes = nlp.Defaults.prefixes + suffixes = nlp.Defaults.suffixes + infixes = nlp.Defaults.infixes + url_match = nlp.Defaults.url_match + token_match = nlp.Defaults.token_match + prefix_search = util.compile_prefix_regex(prefixes).search if prefixes else None + suffix_search = util.compile_suffix_regex(suffixes).search if suffixes else None + infix_finditer = util.compile_infix_regex(infixes).finditer if infixes else None + return Tokenizer( + nlp.vocab, + rules=exceptions, + prefix_search=prefix_search, + suffix_search=suffix_search, + infix_finditer=infix_finditer, + token_match=token_match, + url_match=url_match, + ) + return tokenizer_factory + + cdef class Tokenizer: """Segment text, and create Doc objects with the discovered segment boundaries. @@ -51,7 +84,6 @@ cdef class Tokenizer: EXAMPLE: >>> tokenizer = Tokenizer(nlp.vocab) - >>> tokenizer = English().Defaults.create_tokenizer(nlp) DOCS: https://spacy.io/api/tokenizer#init """ diff --git a/spacy/util.py b/spacy/util.py index 06027b621..0d732034f 100644 --- a/spacy/util.py +++ b/spacy/util.py @@ -1,4 +1,5 @@ -from typing import List, Union, Type, Dict, Any +from typing import List, Union, Dict, Any, Optional, Iterable, Callable, Tuple +from typing import Iterator, TYPE_CHECKING import os import importlib import importlib.util @@ -6,8 +7,6 @@ import re from pathlib import Path import thinc from thinc.api import NumpyOps, get_current_ops, Adam, Config -from thinc.config import EmptySchema -from pydantic import BaseModel import functools import itertools import numpy.random @@ -23,6 +22,7 @@ from contextlib import contextmanager import tempfile import shutil import shlex +import inspect try: import cupy.random @@ -46,6 +46,10 @@ from .compat import cupy, CudaStream, is_windows from .errors import Errors, Warnings from . import about +if TYPE_CHECKING: + # This lets us add type hints for mypy etc. without causing circular imports + from .language import Language # noqa: F401 + _PRINT_ENV = False OOV_RANK = numpy.iinfo(numpy.uint64).max @@ -54,16 +58,50 @@ OOV_RANK = numpy.iinfo(numpy.uint64).max class registry(thinc.registry): languages = catalogue.create("spacy", "languages", entry_points=True) architectures = catalogue.create("spacy", "architectures", entry_points=True) + tokenizers = catalogue.create("spacy", "tokenizers", entry_points=True) + lemmatizers = catalogue.create("spacy", "lemmatizers", entry_points=True) lookups = catalogue.create("spacy", "lookups", entry_points=True) - factories = catalogue.create("spacy", "factories", entry_points=True) + language_data = catalogue.create("spacy", "language_data", entry_points=True) displacy_colors = catalogue.create("spacy", "displacy_colors", entry_points=True) assets = catalogue.create("spacy", "assets", entry_points=True) + # These are factories registered via third-party packages and the + # spacy_factories entry point. This registry only exists so we can easily + # load them via the entry points. The "true" factories are added via the + # Language.factory decorator (in the spaCy code base and user code) and those + # are the factories used to initialize components via registry.make_from_config. + _entry_point_factories = catalogue.create("spacy", "factories", entry_points=True) + factories = catalogue.create("spacy", "internal_factories") # This is mostly used to get a list of all installed models in the current # environment. spaCy models packaged with `spacy package` will "advertise" # themselves via entry points. models = catalogue.create("spacy", "models", entry_points=True) +class SimpleFrozenDict(dict): + """Simplified implementation of a frozen dict, mainly used as default + function or method argument (for arguments that should default to empty + dictionary). Will raise an error if user or spaCy attempts to add to dict. + """ + + def __init__(self, *args, error: str = Errors.E095, **kwargs) -> None: + """Initialize the frozen dict. Can be initialized with pre-defined + values. + + error (str): The error message when user tries to assign to dict. + """ + super().__init__(*args, **kwargs) + self.error = error + + def __setitem__(self, key, value): + raise NotImplementedError(self.error) + + def pop(self, key, default=None): + raise NotImplementedError(self.error) + + def update(self, other): + raise NotImplementedError(self.error) + + def set_env_log(value): global _PRINT_ENV _PRINT_ENV = value @@ -141,93 +179,95 @@ def get_module_path(module): return Path(sys.modules[module.__module__].__file__).parent -def load_model(name, **overrides): +def load_model( + name: Union[str, Path], + disable: Iterable[str] = tuple(), + component_cfg: Dict[str, Dict[str, Any]] = SimpleFrozenDict(), +): """Load a model from a package or data path. name (str): Package name or model path. **overrides: Specific overrides, like pipeline components to disable. RETURNS (Language): `Language` class with the loaded model. """ + cfg = component_cfg if isinstance(name, str): # name or string path if name.startswith("blank:"): # shortcut for blank model return get_lang_class(name.replace("blank:", ""))() if is_package(name): # installed as package - return load_model_from_package(name, **overrides) + return load_model_from_package(name, disable=disable, component_cfg=cfg) if Path(name).exists(): # path to model data directory - return load_model_from_path(Path(name), **overrides) + return load_model_from_path(Path(name), disable=disable, component_cfg=cfg) elif hasattr(name, "exists"): # Path or Path-like to model data - return load_model_from_path(name, **overrides) + return load_model_from_path(name, disable=disable, component_cfg=cfg) raise IOError(Errors.E050.format(name=name)) -def load_model_from_package(name, **overrides): +def load_model_from_package( + name: str, + disable: Iterable[str] = tuple(), + component_cfg: Dict[str, Dict[str, Any]] = SimpleFrozenDict(), +): """Load a model from an installed package.""" cls = importlib.import_module(name) - return cls.load(**overrides) + return cls.load(disable=disable, component_cfg=component_cfg) -def load_model_from_path(model_path, meta=False, **overrides): +def load_model_from_path( + model_path: Union[str, Path], + meta: Optional[Dict[str, Any]] = None, + disable: Iterable[str] = tuple(), + component_cfg: Dict[str, Dict[str, Any]] = SimpleFrozenDict(), +): """Load a model from a data directory path. Creates Language class with - pipeline from meta.json and then calls from_disk() with path.""" + pipeline from config.cfg and then calls from_disk() with path.""" + if not model_path.exists(): + raise IOError(Errors.E052.format(path=model_path)) if not meta: meta = get_model_meta(model_path) - nlp_config = get_model_config(model_path) - if nlp_config.get("nlp", None): - return load_model_from_config(nlp_config["nlp"]) - - # Support language factories registered via entry points (e.g. custom - # language subclass) while keeping top-level language identifier "lang" - lang = meta.get("lang_factory", meta["lang"]) - cls = get_lang_class(lang) - nlp = cls(meta=meta, **overrides) - pipeline = meta.get("pipeline", []) - factories = meta.get("factories", {}) - disable = overrides.get("disable", []) - if pipeline is True: - pipeline = nlp.Defaults.pipe_names - elif pipeline in (False, None): - pipeline = [] - # skip "vocab" from overrides in component initialization since vocab is - # already configured from overrides when nlp is initialized above - if "vocab" in overrides: - del overrides["vocab"] - for name in pipeline: - if name not in disable: - config = meta.get("pipeline_args", {}).get(name, {}) - config.update(overrides) - factory = factories.get(name, name) - if nlp_config.get(name, None): - model_config = nlp_config[name]["model"] - config["model"] = model_config - component = nlp.create_pipe(factory, config=config) - nlp.add_pipe(component, name=name) + config_path = model_path / "config.cfg" + if not config_path.exists() or not config_path.is_file(): + raise IOError(Errors.E053.format(path=config_path, name="config.cfg")) + config = Config().from_disk(config_path) + override_cfg = {"components": {p: dict_to_dot(c) for p, c in component_cfg.items()}} + overrides = dict_to_dot(override_cfg) + nlp, _ = load_model_from_config(config, disable=disable, overrides=overrides) return nlp.from_disk(model_path, exclude=disable) -def load_model_from_config(nlp_config, replace=False): - if "name" in nlp_config: - nlp = load_model(**nlp_config) - elif "lang" in nlp_config: - lang_class = get_lang_class(nlp_config["lang"]) - nlp = lang_class() - else: - raise ValueError(Errors.E993) - if "pipeline" in nlp_config: - for name, component_cfg in nlp_config["pipeline"].items(): - factory = component_cfg.pop("factory") - if name in nlp.pipe_names: - if replace: - component = nlp.create_pipe(factory, config=component_cfg) - nlp.replace_pipe(name, component) - else: - raise ValueError(Errors.E985.format(component=name)) - else: - component = nlp.create_pipe(factory, config=component_cfg) - nlp.add_pipe(component, name=name) - return nlp +def load_model_from_config( + config: Union[Dict[str, Any], Config], + disable: Iterable[str] = tuple(), + overrides: Dict[str, Any] = {}, + auto_fill: bool = False, + validate: bool = True, +) -> Tuple["Language", Config]: + """Create an nlp object from a config. Expects the full config file including + a section "nlp" containing the settings for the nlp object. + """ + if "nlp" not in config: + raise ValueError(Errors.E985.format(config=config)) + nlp_config = config["nlp"] + if "lang" not in nlp_config: + raise ValueError(Errors.E993.format(config=nlp_config)) + # This will automatically handle all codes registered via the languages + # registry, including custom subclasses provided via entry points + lang_cls = get_lang_class(nlp_config["lang"]) + nlp = lang_cls.from_config( + config, + disable=disable, + overrides=overrides, + auto_fill=auto_fill, + validate=validate, + ) + return nlp, nlp.resolved -def load_model_from_init_py(init_file, **overrides): +def load_model_from_init_py( + init_file: Union[Path, str], + disable: Iterable[str] = tuple(), + component_cfg: Dict[str, Dict[str, Any]] = SimpleFrozenDict(), +): """Helper function to use in the `load()` method of a model package's __init__.py. @@ -241,7 +281,9 @@ def load_model_from_init_py(init_file, **overrides): data_path = model_path / data_dir if not model_path.exists(): raise IOError(Errors.E052.format(path=data_path)) - return load_model_from_path(data_path, meta, **overrides) + return load_model_from_path( + data_path, meta, disable=disable, component_cfg=component_cfg + ) def get_installed_models(): @@ -332,53 +374,6 @@ def get_base_version(version): return Version(version).base_version -def load_config( - path: Union[Path, str], - *, - create_objects: bool = False, - schema: Type[BaseModel] = EmptySchema, - overrides: Dict[str, Any] = {}, - validate: bool = True, -) -> Dict[str, Any]: - """Load a Thinc-formatted config file, optionally filling in objects where - the config references registry entries. See "Thinc config files" for details. - - path (str / Path): Path to the config file - create_objects (bool): Whether to automatically create objects when the config - references registry entries. Defaults to False. - schema (BaseModel): Optional pydantic base schema to use for validation. - overrides (Dict[str, Any]): Optional overrides to substitute in config. - validate (bool): Whether to validate against schema. - RETURNS (dict): The objects from the config file. - """ - config = thinc.config.Config().from_disk(path) - kwargs = {"validate": validate, "schema": schema, "overrides": overrides} - if create_objects: - return registry.make_from_config(config, **kwargs) - else: - # Just fill config here so we can validate and fail early - if validate and schema: - registry.fill_config(config, **kwargs) - return config - - -def load_config_from_str(string, create_objects=False): - """Load a Thinc-formatted config, optionally filling in objects where - the config references registry entries. See "Thinc config files" for details. - - string (str / Path): Text contents of the config file. - create_objects (bool): Whether to automatically create objects when the config - references registry entries. Defaults to False. - - RETURNS (dict): The objects from the config file. - """ - config = thinc.config.Config().from_str(string) - if create_objects: - return registry.make_from_config(config, validate=True) - else: - return config - - def get_model_meta(path): """Get model meta.json from a directory path and validate its contents. @@ -415,23 +410,6 @@ def get_model_meta(path): return meta -def get_model_config(path): - """Get the model's config from a directory path. - - path (str / Path): Path to model directory. - RETURNS (Config): The model's config data. - """ - model_path = ensure_path(path) - if not model_path.exists(): - raise IOError(Errors.E052.format(path=model_path)) - config_path = model_path / "config.cfg" - # model directories are allowed not to have config files ? - if not config_path.is_file(): - return Config({}) - # raise IOError(Errors.E053.format(path=config_path, name="config.cfg")) - return Config().from_disk(config_path) - - def is_package(name): """Check if string maps to a package installed via pip. @@ -553,14 +531,19 @@ def is_in_jupyter(): return False -def get_component_name(component): - if hasattr(component, "name"): - return component.name - if hasattr(component, "__name__"): - return component.__name__ - if hasattr(component, "__class__") and hasattr(component.__class__, "__name__"): - return component.__class__.__name__ - return repr(component) +def get_object_name(obj: Any) -> str: + """Get a human-readable name of a Python object, e.g. a pipeline component. + + obj (Any): The Python object, typically a function or class. + RETURNS (str): A human-readable name. + """ + if hasattr(obj, "name"): + return obj.name + if hasattr(obj, "__name__"): + return obj.__name__ + if hasattr(obj, "__class__") and hasattr(obj.__class__, "__name__"): + return obj.__class__.__name__ + return repr(obj) def get_cuda_stream(require=False, non_blocking=True): @@ -1000,20 +983,106 @@ def get_words_and_spaces(words, text): return (text_words, text_spaces) -class SimpleFrozenDict(dict): - """Simplified implementation of a frozen dict, mainly used as default - function or method argument (for arguments that should default to empty - dictionary). Will raise an error if user or spaCy attempts to add to dict. +def copy_config(config: Union[Dict[str, Any], Config]) -> Config: + """Deep copy a Config. Will raise an error if the config contents are not + JSON-serializable. + + config (Config): The config to copy. + RETURNS (Config): The copied config. """ + try: + return Config(config).copy() + except ValueError: + raise ValueError(Errors.E961.format(config=config)) - def __setitem__(self, key, value): - raise NotImplementedError(Errors.E095) - def pop(self, key, default=None): - raise NotImplementedError(Errors.E095) +def deep_merge_configs( + config: Union[Dict[str, Any], Config], defaults: Union[Dict[str, Any], Config] +) -> Config: + """Deep merge two configs, a base config and its defaults. Ignores + references to registered functions to avoid filling in - def update(self, other): - raise NotImplementedError(Errors.E095) + config (Dict[str, Any]): The config. + destination (Dict[str, Any]): The config defaults. + RETURNS (Dict[str, Any]): The merged config. + """ + config = copy_config(config) + merged = _deep_merge_configs(config, defaults) + return Config(merged) + + +def _deep_merge_configs( + config: Union[Dict[str, Any], Config], defaults: Union[Dict[str, Any], Config] +) -> Union[Dict[str, Any], Config]: + for key, value in defaults.items(): + if isinstance(value, dict): + node = config.setdefault(key, {}) + if not isinstance(node, dict): + continue + promises = [key for key in value if key.startswith("@")] + promise = promises[0] if promises else None + # We only update the block from defaults if it refers to the same + # registered function + if ( + promise + and any(k.startswith("@") for k in node) + and (promise in node and node[promise] != value[promise]) + ): + continue + defaults = _deep_merge_configs(node, value) + elif key not in config: + config[key] = value + return config + + +def dot_to_dict(values: Dict[str, Any]) -> Dict[str, dict]: + """Convert dot notation to a dict. For example: {"token.pos": True, + "token._.xyz": True} becomes {"token": {"pos": True, "_": {"xyz": True }}}. + + values (Dict[str, Any]): The key/value pairs to convert. + RETURNS (Dict[str, dict]): The converted values. + """ + result = {} + for key, value in values.items(): + path = result + parts = key.lower().split(".") + for i, item in enumerate(parts): + is_last = i == len(parts) - 1 + path = path.setdefault(item, value if is_last else {}) + return result + + +def dict_to_dot(obj: Dict[str, dict]) -> Dict[str, Any]: + """Convert dot notation to a dict. For example: {"token": {"pos": True, + "_": {"xyz": True }}} becomes {"token.pos": True, "token._.xyz": True}. + + values (Dict[str, dict]): The dict to convert. + RETURNS (Dict[str, Any]): The key/value pairs. + """ + return {".".join(key): value for key, value in walk_dict(obj)} + + +def walk_dict( + node: Dict[str, Any], parent: List[str] = [] +) -> Iterator[Tuple[List[str], Any]]: + """Walk a dict and yield the path and values of the leaves.""" + for key, value in node.items(): + key_parent = [*parent, key] + if isinstance(value, dict): + yield from walk_dict(value, key_parent) + else: + yield (key_parent, value) + + +def get_arg_names(func: Callable) -> List[str]: + """Get a list of all named arguments of a function (regular, + keyword-only). + + func (Callable): The function + RETURNS (List[str]): The argument names. + """ + argspec = inspect.getfullargspec(func) + return list(set([*argspec.args, *argspec.kwonlyargs])) class DummyTokenizer: diff --git a/spacy/vocab.pxd b/spacy/vocab.pxd index 49f5bf415..f93b6cffe 100644 --- a/spacy/vocab.pxd +++ b/spacy/vocab.pxd @@ -30,6 +30,7 @@ cdef class Vocab: cpdef public object vectors cpdef public object lookups cpdef public object lookups_extra + cpdef public object writing_system cdef readonly int length cdef public object data_dir cdef public object lex_attr_getters diff --git a/spacy/vocab.pyx b/spacy/vocab.pyx index 58c1388fc..3ab90dd2f 100644 --- a/spacy/vocab.pyx +++ b/spacy/vocab.pyx @@ -3,6 +3,7 @@ from libc.string cimport memcpy import srsly from thinc.api import get_array_module +import functools from .lexeme cimport EMPTY_LEXEME, OOV_RANK from .lexeme cimport Lexeme @@ -13,13 +14,13 @@ from .attrs cimport LANG, ORTH, TAG, POS from .compat import copy_reg from .errors import Errors from .lemmatizer import Lemmatizer -from .attrs import intify_attrs, NORM +from .attrs import intify_attrs, NORM, IS_STOP from .vectors import Vectors -from .util import link_vectors_to_models +from .util import link_vectors_to_models, registry from .lookups import Lookups from . import util from .lang.norm_exceptions import BASE_NORMS -from .lang.lex_attrs import LEX_ATTRS +from .lang.lex_attrs import LEX_ATTRS, is_stop, get_lang cdef class Vocab: @@ -31,7 +32,8 @@ cdef class Vocab: """ def __init__(self, lex_attr_getters=None, tag_map=None, lemmatizer=None, strings=tuple(), lookups=None, lookups_extra=None, - oov_prob=-20., vectors_name=None, **deprecated_kwargs): + oov_prob=-20., vectors_name=None, writing_system={}, + **deprecated_kwargs): """Create the vocabulary. lex_attr_getters (dict): A dictionary mapping attribute IDs to @@ -70,6 +72,7 @@ cdef class Vocab: self.vectors = Vectors(name=vectors_name) self.lookups = lookups self.lookups_extra = lookups_extra + self.writing_system = writing_system @property def lang(self): @@ -78,17 +81,6 @@ cdef class Vocab: langfunc = self.lex_attr_getters.get(LANG, None) return langfunc("_") if langfunc else "" - property writing_system: - """A dict with information about the language's writing system. To get - the data, we use the vocab.lang property to fetch the Language class. - If the Language class is not loaded, an empty dict is returned. - """ - def __get__(self): - if not util.lang_class_is_loaded(self.lang): - return {} - lang_class = util.get_lang_class(self.lang) - return dict(lang_class.Defaults.writing_system) - def __len__(self): """The current number of lexemes stored. @@ -426,6 +418,67 @@ cdef class Vocab: orth = self.strings.add(orth) return orth in self.vectors + @classmethod + def from_config( + cls, + config, + lemmatizer=None, + lex_attr_getters=None, + stop_words=None, + vectors_name=None, + tag_map=None, + morph_rules=None + ): + """Create a Vocab from a config and (currently) language defaults, i.e. + nlp.Defaults. + + config (Dict[str, Any]): The full config. + lemmatizer (Callable): Optional lemmatizer. + vectors_name (str): Optional vectors name. + RETURNS (Vocab): The vocab. + """ + # TODO: make this less messy – move lemmatizer out into its own pipeline + # component, move language defaults to config + lang = config["nlp"]["lang"] + writing_system = config["nlp"]["writing_system"] + if not lemmatizer: + lemma_cfg = {"lemmatizer": config["nlp"]["lemmatizer"]} + lemmatizer = registry.make_from_config(lemma_cfg)["lemmatizer"] + lookups = lemmatizer.lookups + if "lexeme_norm" not in lookups: + lookups.add_table("lexeme_norm") + if stop_words is None: + stop_words_cfg = {"stop_words": config["nlp"]["stop_words"]} + stop_words = registry.make_from_config(stop_words_cfg)["stop_words"] + if lex_attr_getters is None: + lex_attrs_cfg = {"lex_attr_getters": config["nlp"]["lex_attr_getters"]} + lex_attr_getters = registry.make_from_config(lex_attrs_cfg)["lex_attr_getters"] + lex_attrs = dict(LEX_ATTRS) + lex_attrs.update(lex_attr_getters) + # This is messy, but it's the minimal working fix to Issue #639. + lex_attrs[IS_STOP] = functools.partial(is_stop, stops=stop_words) + # Ensure that getter can be pickled + lex_attrs[LANG] = functools.partial(get_lang, lang=lang) + lex_attrs[NORM] = util.add_lookups( + lex_attrs.get(NORM, LEX_ATTRS[NORM]), + BASE_NORMS, + # TODO: we need to move the lexeme norms to their own entry + # points so we can specify them separately from the lemma lookups + lookups.get_table("lexeme_norm"), + ) + vocab = cls( + lex_attr_getters=lex_attrs, + lemmatizer=lemmatizer, + lookups=lookups, + writing_system=writing_system, + tag_map=tag_map, + ) + if morph_rules is not None: + vocab.morphology.load_morph_exceptions(morph_rules) + if vocab.vectors.name is None and vectors_name: + vocab.vectors.name = vectors_name + return vocab + def to_disk(self, path, exclude=tuple()): """Save the current state to a directory.