From 749631ad2850bf4c97473938bdedbaffdd8b59db Mon Sep 17 00:00:00 2001 From: Adriane Boyd Date: Mon, 21 Feb 2022 10:22:36 +0100 Subject: [PATCH 1/3] Fix Tok2Vec for empty batches (#10324) * Add test for tok2vec with vectors and empty docs * Add shortcut for empty batch in Tok2Vec.predict * Avoid types --- spacy/pipeline/tok2vec.py | 4 ++++ spacy/tests/pipeline/test_tok2vec.py | 23 +++++++++++++++++++++-- 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/spacy/pipeline/tok2vec.py b/spacy/pipeline/tok2vec.py index 00d9548a4..2f6d2b60f 100644 --- a/spacy/pipeline/tok2vec.py +++ b/spacy/pipeline/tok2vec.py @@ -118,6 +118,10 @@ class Tok2Vec(TrainablePipe): DOCS: https://spacy.io/api/tok2vec#predict """ + if not any(len(doc) for doc in docs): + # Handle cases where there are no tokens in any docs. + width = self.model.get_dim("nO") + return [self.model.ops.alloc((0, width)) for doc in docs] tokvecs = self.model.predict(docs) batch_id = Tok2VecListener.get_batch_id(docs) for listener in self.listeners: diff --git a/spacy/tests/pipeline/test_tok2vec.py b/spacy/tests/pipeline/test_tok2vec.py index 809a79dd6..8b3541cec 100644 --- a/spacy/tests/pipeline/test_tok2vec.py +++ b/spacy/tests/pipeline/test_tok2vec.py @@ -11,7 +11,7 @@ from spacy.lang.en import English from thinc.api import Config, get_current_ops from numpy.testing import assert_array_equal -from ..util import get_batch, make_tempdir +from ..util import get_batch, make_tempdir, add_vecs_to_vocab def test_empty_doc(): @@ -134,9 +134,25 @@ TRAIN_DATA = [ ] -def test_tok2vec_listener(): +@pytest.mark.parametrize("with_vectors", (False, True)) +def test_tok2vec_listener(with_vectors): orig_config = Config().from_str(cfg_string) + orig_config["components"]["tok2vec"]["model"]["embed"][ + "include_static_vectors" + ] = with_vectors nlp = util.load_model_from_config(orig_config, auto_fill=True, validate=True) + + if with_vectors: + ops = get_current_ops() + vectors = [ + ("apple", ops.asarray([1, 2, 3])), + ("orange", ops.asarray([-1, -2, -3])), + ("and", ops.asarray([-1, -1, -1])), + ("juice", ops.asarray([5, 5, 10])), + ("pie", ops.asarray([7, 6.3, 8.9])), + ] + add_vecs_to_vocab(nlp.vocab, vectors) + assert nlp.pipe_names == ["tok2vec", "tagger"] tagger = nlp.get_pipe("tagger") tok2vec = nlp.get_pipe("tok2vec") @@ -163,6 +179,9 @@ def test_tok2vec_listener(): ops = get_current_ops() assert_array_equal(ops.to_numpy(doc.tensor), ops.to_numpy(doc_tensor)) + # test with empty doc + doc = nlp("") + # TODO: should this warn or error? nlp.select_pipes(disable="tok2vec") assert nlp.pipe_names == ["tagger"] From 6a4a00c44745633a61fbbbb5d75f1394081cc7ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Danie=CC=88l=20de=20Kok?= Date: Thu, 16 Dec 2021 09:31:45 +0100 Subject: [PATCH 2/3] Pin mypy to 0.910 until there is a compatible pydantic version --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 46337389c..524edf69d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -28,3 +28,4 @@ pytest-timeout>=1.3.0,<2.0.0 mock>=2.0.0,<3.0.0 flake8>=3.5.0,<3.6.0 hypothesis>=3.27.0,<7.0.0 +mypy==0.910 From 5caccbd19e00479c3b2179244c6f537cce87876c Mon Sep 17 00:00:00 2001 From: Adriane Boyd Date: Tue, 30 Nov 2021 10:08:51 +0100 Subject: [PATCH 3/3] Switch to latest CI images (#9773) --- azure-pipelines.yml | 43 ++++++++++++++++++++++++++----------------- 1 file changed, 26 insertions(+), 17 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 7ad1b27b3..704285c2b 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -22,7 +22,7 @@ jobs: # defined in .flake8 and overwrites the selected codes. - job: "Validate" pool: - vmImage: "ubuntu-18.04" + vmImage: "ubuntu-latest" steps: - task: UsePythonVersion@0 inputs: @@ -38,41 +38,50 @@ jobs: matrix: # We're only running one platform per Python version to speed up builds Python36Linux: - imageName: "ubuntu-18.04" + imageName: "ubuntu-latest" python.version: "3.6" # Python36Windows: - # imageName: "vs2017-win2016" + # imageName: "windows-latest" # python.version: "3.6" # Python36Mac: - # imageName: "macos-10.14" + # imageName: "macos-latest" # python.version: "3.6" # Python37Linux: - # imageName: "ubuntu-18.04" + # imageName: "ubuntu-latest" # python.version: "3.7" Python37Windows: - imageName: "vs2017-win2016" + imageName: "windows-latest" python.version: "3.7" # Python37Mac: - # imageName: "macos-10.14" + # imageName: "macos-latest" # python.version: "3.7" # Python38Linux: - # imageName: "ubuntu-18.04" + # imageName: "ubuntu-latest" # python.version: "3.8" # Python38Windows: - # imageName: "vs2017-win2016" + # imageName: "windows-latest" # python.version: "3.8" Python38Mac: - imageName: "macos-10.14" + imageName: "macos-latest" python.version: "3.8" Python39Linux: - imageName: "ubuntu-18.04" - python.version: "3.9" - Python39Windows: - imageName: "vs2017-win2016" - python.version: "3.9" - Python39Mac: - imageName: "macos-10.14" + imageName: "ubuntu-latest" python.version: "3.9" + # Python39Windows: + # imageName: "windows-latest" + # python.version: "3.9" + # Python39Mac: + # imageName: "macos-latest" + # python.version: "3.9" + Python310Linux: + imageName: "ubuntu-latest" + python.version: "3.10" + Python310Windows: + imageName: "windows-latest" + python.version: "3.10" + Python310Mac: + imageName: "macos-latest" + python.version: "3.10" maxParallel: 4 pool: vmImage: $(imageName)