diff --git a/spacy/scorer.py b/spacy/scorer.py
index 40a819e7c..61fe85a62 100644
--- a/spacy/scorer.py
+++ b/spacy/scorer.py
@@ -242,7 +242,8 @@ class Scorer:
                 per_feat[field].score_set(
                     pred_per_feat.get(field, set()), gold_per_feat.get(field, set()),
                 )
-        return {f"{attr}_per_feat": per_feat}
+        result = {k: v.to_dict() for k, v in per_feat.items()}
+        return {f"{attr}_per_feat": result}
 
     @staticmethod
     def score_spans(
diff --git a/spacy/tests/test_scorer.py b/spacy/tests/test_scorer.py
index 422b4e328..6dae14210 100644
--- a/spacy/tests/test_scorer.py
+++ b/spacy/tests/test_scorer.py
@@ -259,7 +259,7 @@ def test_tag_score(tagged_doc):
     assert results["tag_acc"] == 1.0
     assert results["pos_acc"] == 1.0
     assert results["morph_acc"] == 1.0
-    assert results["morph_per_feat"]["NounType"].fscore == 1.0
+    assert results["morph_per_feat"]["NounType"]["f"] == 1.0
 
     # Gold annotation is modified
     scorer = Scorer()
@@ -282,9 +282,9 @@ def test_tag_score(tagged_doc):
     assert results["tag_acc"] == 0.9
     assert results["pos_acc"] == 0.9
     assert results["morph_acc"] == approx(0.8)
-    assert results["morph_per_feat"]["NounType"].fscore == 1.0
-    assert results["morph_per_feat"]["Poss"].fscore == 0.0
-    assert results["morph_per_feat"]["Number"].fscore == approx(0.72727272)
+    assert results["morph_per_feat"]["NounType"]["f"] == 1.0
+    assert results["morph_per_feat"]["Poss"]["f"] == 0.0
+    assert results["morph_per_feat"]["Number"]["f"] == approx(0.72727272)
 
 
 def test_roc_auc_score():