diff --git a/spacy/pipeline.pyx b/spacy/pipeline.pyx
index e826ee0d6..b233f2071 100644
--- a/spacy/pipeline.pyx
+++ b/spacy/pipeline.pyx
@@ -892,12 +892,10 @@ cdef class DependencyParser(Parser):
         self._multitasks.append(labeller)
 
     def init_multitask_objectives(self, gold_tuples, pipeline, sgd=None, **cfg):
-        self.add_multitask_objective('tag')
         for labeller in self._multitasks:
             tok2vec = self.model[0]
             labeller.begin_training(gold_tuples, pipeline=pipeline,
                                     tok2vec=tok2vec, sgd=sgd)
-            pipeline.append((labeller.name, labeller))
 
     def __reduce__(self):
         return (DependencyParser, (self.vocab, self.moves, self.model),
@@ -919,7 +917,6 @@ cdef class EntityRecognizer(Parser):
             tok2vec = self.model[0]
             labeller.begin_training(gold_tuples, pipeline=pipeline,
                                     tok2vec=tok2vec)
-            pipeline.append((labeller.name, labeller))
 
     def __reduce__(self):
         return (EntityRecognizer, (self.vocab, self.moves, self.model),
diff --git a/spacy/syntax/nn_parser.pyx b/spacy/syntax/nn_parser.pyx
index a4647c159..804838b66 100644
--- a/spacy/syntax/nn_parser.pyx
+++ b/spacy/syntax/nn_parser.pyx
@@ -605,6 +605,8 @@ cdef class Parser:
                 break
         self._make_updates(d_tokvecs,
             bp_tokvecs, backprops, sgd, cuda_stream)
+        for multitask in self._multitasks:
+            multitask.update(docs, golds, drop=drop, sgd=sgd)
 
     def update_beam(self, docs, golds, width=None, density=None,
             drop=0., sgd=None, losses=None):