diff --git a/bin/parser/train_ud.py b/bin/parser/train_ud.py
index 4efc063d7..62256cc14 100644
--- a/bin/parser/train_ud.py
+++ b/bin/parser/train_ud.py
@@ -71,7 +71,7 @@ def main(train_loc, dev_loc, model_dir, tag_map_loc):
     features = get_templates('basic')
     
     model_dir = pathlib.Path(model_dir)
-    with (model_dir / 'deps' / 'config.json').open('wb') as file_:
+    with (model_dir / 'deps' / 'config.json').open('w') as file_:
         json.dump({'pseudoprojective': True, 'labels': actions, 'features': features}, file_)
 
     vocab = Vocab(lex_attr_getters=Language.Defaults.lex_attr_getters, tag_map=tag_map)
diff --git a/spacy/language.py b/spacy/language.py
index a1aad54b7..99ba46889 100644
--- a/spacy/language.py
+++ b/spacy/language.py
@@ -5,10 +5,7 @@ import pathlib
 from contextlib import contextmanager
 import shutil
 
-try:
-    import ujson as json
-except ImportError:
-    import json
+import ujson as json
 
 
 try:
@@ -222,11 +219,11 @@ class Language(object):
         parser_cfg['actions'] = ArcEager.get_actions(gold_parses=gold_tuples)
         entity_cfg['actions'] = BiluoPushDown.get_actions(gold_parses=gold_tuples)
 
-        with (dep_model_dir / 'config.json').open('wb') as file_:
+        with (dep_model_dir / 'config.json').open('w') as file_:
             json.dump(parser_cfg, file_)
-        with (ner_model_dir / 'config.json').open('wb') as file_:
+        with (ner_model_dir / 'config.json').open('w') as file_:
             json.dump(entity_cfg, file_)
-        with (pos_model_dir / 'config.json').open('wb') as file_:
+        with (pos_model_dir / 'config.json').open('w') as file_:
             json.dump(tagger_cfg, file_)
 
         self = cls(
@@ -387,7 +384,7 @@ class Language(object):
         else:
             entity_iob_freqs = []
             entity_type_freqs = []
-        with (path / 'vocab' / 'serializer.json').open('wb') as file_:
+        with (path / 'vocab' / 'serializer.json').open('w') as file_:
             file_.write(
                 json.dumps([
                     (TAG, tagger_freqs),