mirror of
https://github.com/explosion/spaCy.git
synced 2025-10-31 07:57:35 +03:00
c0f4a1e43b
* verbose and tag_map options * adding init_tok2vec option and only changing the tok2vec that is specified * adding omit_extra_lookups and verifying textcat config * wip * pretrain bugfix * add replace and resume options * train_textcat fix * raw text functionality * improve UX when KeyError or when input data can't be parsed * avoid unnecessary access to goldparse in TextCat pipe * save performance information in nlp.meta * add noise_level to config * move nn_parser's defaults to config file * multitask in config - doesn't work yet * scorer offering both F and AUC options, need to be specified in config * add textcat verification code from old train script * small fixes to config files * clean up * set default config for ner/parser to allow create_pipe to work as before * two more test fixes * small fixes * cleanup * fix NER pickling + additional unit test * create_pipe as before
42 lines
1.3 KiB
Python
42 lines
1.3 KiB
Python
import pickle
|
|
import numpy
|
|
|
|
from spacy.lang.en import English
|
|
from spacy.vocab import Vocab
|
|
|
|
from spacy.tests.util import make_tempdir
|
|
|
|
|
|
def test_pickle_ner():
|
|
""" Ensure the pickling of the NER goes well"""
|
|
vocab = Vocab(vectors_name="test_vocab_add_vector")
|
|
nlp = English(vocab=vocab)
|
|
ner = nlp.create_pipe("ner", config={"min_action_freq": 342})
|
|
with make_tempdir() as tmp_path:
|
|
with (tmp_path / "ner.pkl").open("wb") as file_:
|
|
pickle.dump(ner, file_)
|
|
assert ner.cfg["min_action_freq"] == 342
|
|
|
|
with (tmp_path / "ner.pkl").open("rb") as file_:
|
|
ner2 = pickle.load(file_)
|
|
assert ner2.cfg["min_action_freq"] == 342
|
|
|
|
|
|
def test_issue4725():
|
|
# ensures that this runs correctly and doesn't hang or crash because of the global vectors
|
|
# if it does crash, it's usually because of calling 'spawn' for multiprocessing (e.g. on Windows)
|
|
vocab = Vocab(vectors_name="test_vocab_add_vector")
|
|
data = numpy.ndarray((5, 3), dtype="f")
|
|
data[0] = 1.0
|
|
data[1] = 2.0
|
|
vocab.set_vector("cat", data[0])
|
|
vocab.set_vector("dog", data[1])
|
|
|
|
nlp = English(vocab=vocab)
|
|
ner = nlp.create_pipe("ner")
|
|
nlp.add_pipe(ner)
|
|
nlp.begin_training()
|
|
docs = ["Kurt is in London."] * 10
|
|
for _ in nlp.pipe(docs, batch_size=2, n_process=2):
|
|
pass
|