diff --git a/spacy/_ml.py b/spacy/_ml.py
index eb751ca6c..964b1fa7a 100644
--- a/spacy/_ml.py
+++ b/spacy/_ml.py
@@ -306,8 +306,9 @@ def Tok2Vec(width, embed_size, **kwargs):
                 embed
                 >> convolution ** conv_depth, pad=conv_depth
             )
-            >> PyTorchBiLSTM(width, width, bilstm_depth)
         )
+        if bilstm_depth >= 1:
+            tok2vec = tok2vec >> PyTorchBiLSTM(width, width, bilstm_depth)
         # Work around thinc API limitations :(. TODO: Revise in Thinc 7
         tok2vec.nO = width
         tok2vec.embed = embed