From 4890ee1732a2bdf7417363476799cc024f68a3e8 Mon Sep 17 00:00:00 2001 From: Matthew Honnibal Date: Sat, 24 Feb 2018 10:32:32 +0100 Subject: [PATCH] Fix scoring of tokenization for punct --- spacy/scorer.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/spacy/scorer.py b/spacy/scorer.py index 673df132c..fa69e03e8 100644 --- a/spacy/scorer.py +++ b/spacy/scorer.py @@ -100,8 +100,7 @@ class Scorer(object): continue gold_i = gold.cand_to_gold[token.i] if gold_i is None: - if token.dep_.lower() not in punct_labels: - self.tokens.fp += 1 + self.tokens.fp += 1 else: self.tokens.tp += 1 cand_tags.add((gold_i, token.tag_))