spaCy/spacy/tests/conftest.py
Ines Montani 43b960c01b
Refactor pipeline components, config and language data (#5759)
* Update with WIP

* Update with WIP

* Update with pipeline serialization

* Update types and pipe factories

* Add deep merge, tidy up and add tests

* Fix pipe creation from config

* Don't validate default configs on load

* Update spacy/language.py

Co-authored-by: Ines Montani <ines@ines.io>

* Adjust factory/component meta error

* Clean up factory args and remove defaults

* Add test for failing empty dict defaults

* Update pipeline handling and methods

* provide KB as registry function instead of as object

* small change in test to make functionality more clear

* update example script for EL configuration

* Fix typo

* Simplify test

* Simplify test

* splitting pipes.pyx into separate files

* moving default configs to each component file

* fix batch_size type

* removing default values from component constructors where possible (TODO: test 4725)

* skip instead of xfail

* Add test for config -> nlp with multiple instances

* pipeline.pipes -> pipeline.pipe

* Tidy up, document, remove kwargs

* small cleanup/generalization for Tok2VecListener

* use DEFAULT_UPSTREAM field

* revert to avoid circular imports

* Fix tests

* Replace deprecated arg

* Make model dirs require config

* fix pickling of keyword-only arguments in constructor

* WIP: clean up and integrate full config

* Add helper to handle function args more reliably

Now also includes keyword-only args

* Fix config composition and serialization

* Improve config debugging and add visual diff

* Remove unused defaults and fix type

* Remove pipeline and factories from meta

* Update spacy/default_config.cfg

Co-authored-by: Sofie Van Landeghem <svlandeg@users.noreply.github.com>

* Update spacy/default_config.cfg

* small UX edits

* avoid printing stack trace for debug CLI commands

* Add support for language-specific factories

* specify the section of the config which holds the model to debug

* WIP: add Language.from_config

* Update with language data refactor WIP

* Auto-format

* Add backwards-compat handling for Language.factories

* Update morphologizer.pyx

* Fix morphologizer

* Update and simplify lemmatizers

* Fix Japanese tests

* Port over tagger changes

* Fix Chinese and tests

* Update to latest Thinc

* WIP: xfail first Russian lemmatizer test

* Fix component-specific overrides

* fix nO for output layers in debug_model

* Fix default value

* Fix tests and don't pass objects in config

* Fix deep merging

* Fix lemma lookup data registry

Only load the lookups if an entry is available in the registry (and if spacy-lookups-data is installed)

* Add types

* Add Vocab.from_config

* Fix typo

* Fix tests

* Make config copying more elegant

* Fix pipe analysis

* Fix lemmatizers and is_base_form

* WIP: move language defaults to config

* Fix morphology type

* Fix vocab

* Remove comment

* Update to latest Thinc

* Add morph rules to config

* Tidy up

* Remove set_morphology option from tagger factory

* Hack use_gpu

* Move [pipeline] to top-level block and make [nlp.pipeline] list

Allows separating component blocks from component order – otherwise, ordering the config would mean a changed component order, which is bad. Also allows initial config to define more components and not use all of them

* Fix use_gpu and resume in CLI

* Auto-format

* Remove resume from config

* Fix formatting and error

* [pipeline] -> [components]

* Fix types

* Fix tagger test: requires set_morphology?

Co-authored-by: Sofie Van Landeghem <svlandeg@users.noreply.github.com>
Co-authored-by: svlandeg <sofie.vanlandeghem@gmail.com>
Co-authored-by: Matthew Honnibal <honnibal+gh@gmail.com>
2020-07-22 13:42:59 +02:00

282 lines
6.1 KiB
Python

import pytest
from spacy.util import get_lang_class
def pytest_addoption(parser):
parser.addoption("--slow", action="store_true", help="include slow tests")
def pytest_runtest_setup(item):
def getopt(opt):
# When using 'pytest --pyargs spacy' to test an installed copy of
# spacy, pytest skips running our pytest_addoption() hook. Later, when
# we call getoption(), pytest raises an error, because it doesn't
# recognize the option we're asking about. To avoid this, we need to
# pass a default value. We default to False, i.e., we act like all the
# options weren't given.
return item.config.getoption(f"--{opt}", False)
for opt in ["slow"]:
if opt in item.keywords and not getopt(opt):
pytest.skip(f"need --{opt} option to run")
# Fixtures for language tokenizers (languages sorted alphabetically)
@pytest.fixture(scope="module")
def tokenizer():
return get_lang_class("xx")().tokenizer
@pytest.fixture(scope="session")
def ar_tokenizer():
return get_lang_class("ar")().tokenizer
@pytest.fixture(scope="session")
def bn_tokenizer():
return get_lang_class("bn")().tokenizer
@pytest.fixture(scope="session")
def ca_tokenizer():
return get_lang_class("ca")().tokenizer
@pytest.fixture(scope="session")
def da_tokenizer():
return get_lang_class("da")().tokenizer
@pytest.fixture(scope="session")
def de_tokenizer():
return get_lang_class("de")().tokenizer
@pytest.fixture(scope="session")
def el_tokenizer():
return get_lang_class("el")().tokenizer
@pytest.fixture(scope="session")
def en_tokenizer():
return get_lang_class("en")().tokenizer
@pytest.fixture(scope="session")
def en_vocab():
return get_lang_class("en")().vocab
@pytest.fixture(scope="session")
def en_parser(en_vocab):
nlp = get_lang_class("en")(en_vocab)
return nlp.create_pipe("parser")
@pytest.fixture(scope="session")
def es_tokenizer():
return get_lang_class("es")().tokenizer
@pytest.fixture(scope="session")
def eu_tokenizer():
return get_lang_class("eu")().tokenizer
@pytest.fixture(scope="session")
def fa_tokenizer():
return get_lang_class("fa")().tokenizer
@pytest.fixture(scope="session")
def fi_tokenizer():
return get_lang_class("fi")().tokenizer
@pytest.fixture(scope="session")
def fr_tokenizer():
return get_lang_class("fr")().tokenizer
@pytest.fixture(scope="session")
def ga_tokenizer():
return get_lang_class("ga")().tokenizer
@pytest.fixture(scope="session")
def gu_tokenizer():
return get_lang_class("gu")().tokenizer
@pytest.fixture(scope="session")
def he_tokenizer():
return get_lang_class("he")().tokenizer
@pytest.fixture(scope="session")
def hr_tokenizer():
return get_lang_class("hr")().tokenizer
@pytest.fixture
def hu_tokenizer():
return get_lang_class("hu")().tokenizer
@pytest.fixture(scope="session")
def id_tokenizer():
return get_lang_class("id")().tokenizer
@pytest.fixture(scope="session")
def it_tokenizer():
return get_lang_class("it")().tokenizer
@pytest.fixture(scope="session")
def ja_tokenizer():
pytest.importorskip("sudachipy")
return get_lang_class("ja")().tokenizer
@pytest.fixture(scope="session")
def ko_tokenizer():
pytest.importorskip("natto")
return get_lang_class("ko")().tokenizer
@pytest.fixture(scope="session")
def lb_tokenizer():
return get_lang_class("lb")().tokenizer
@pytest.fixture(scope="session")
def lt_tokenizer():
return get_lang_class("lt")().tokenizer
@pytest.fixture(scope="session")
def ml_tokenizer():
return get_lang_class("ml")().tokenizer
@pytest.fixture(scope="session")
def nb_tokenizer():
return get_lang_class("nb")().tokenizer
@pytest.fixture(scope="session")
def ne_tokenizer():
return get_lang_class("ne")().tokenizer
@pytest.fixture(scope="session")
def nl_tokenizer():
return get_lang_class("nl")().tokenizer
@pytest.fixture(scope="session")
def pl_tokenizer():
return get_lang_class("pl")().tokenizer
@pytest.fixture(scope="session")
def pt_tokenizer():
return get_lang_class("pt")().tokenizer
@pytest.fixture(scope="session")
def ro_tokenizer():
return get_lang_class("ro")().tokenizer
@pytest.fixture(scope="session")
def ru_tokenizer():
pytest.importorskip("pymorphy2")
return get_lang_class("ru")().tokenizer
@pytest.fixture
def ru_lemmatizer():
pytest.importorskip("pymorphy2")
return get_lang_class("ru")().vocab.morphology.lemmatizer
@pytest.fixture(scope="session")
def sr_tokenizer():
return get_lang_class("sr")().tokenizer
@pytest.fixture(scope="session")
def sv_tokenizer():
return get_lang_class("sv")().tokenizer
@pytest.fixture(scope="session")
def th_tokenizer():
pytest.importorskip("pythainlp")
return get_lang_class("th")().tokenizer
@pytest.fixture(scope="session")
def tr_tokenizer():
return get_lang_class("tr")().tokenizer
@pytest.fixture(scope="session")
def tt_tokenizer():
return get_lang_class("tt")().tokenizer
@pytest.fixture(scope="session")
def uk_tokenizer():
pytest.importorskip("pymorphy2")
pytest.importorskip("pymorphy2.lang")
return get_lang_class("uk")().tokenizer
@pytest.fixture(scope="session")
def ur_tokenizer():
return get_lang_class("ur")().tokenizer
@pytest.fixture(scope="session")
def yo_tokenizer():
return get_lang_class("yo")().tokenizer
@pytest.fixture(scope="session")
def zh_tokenizer_char():
nlp = get_lang_class("zh")()
return nlp.tokenizer
@pytest.fixture(scope="session")
def zh_tokenizer_jieba():
pytest.importorskip("jieba")
config = {
"@tokenizers": "spacy.ChineseTokenizer.v1",
"segmenter": "jieba",
}
nlp = get_lang_class("zh").from_config({"nlp": {"tokenizer": config}})
return nlp.tokenizer
@pytest.fixture(scope="session")
def zh_tokenizer_pkuseg():
pytest.importorskip("pkuseg")
config = {
"@tokenizers": "spacy.ChineseTokenizer.v1",
"segmenter": "pkuseg",
"pkuseg_model": "default",
}
nlp = get_lang_class("zh").from_config({"nlp": {"tokenizer": config}})
return nlp.tokenizer
@pytest.fixture(scope="session")
def hy_tokenizer():
return get_lang_class("hy")().tokenizer