explosion/spaCy
1
1
Fork 0
mirror of https://github.com/explosion/spaCy.git synced 2025-10-31 07:57:35 +03:00
Code Issues Packages Projects Releases Wiki Activity

replace Counter with normal dict

Browse Source
This commit is contained in:
Wolfgang Seeker 2016-03-03 21:36:27 +01:00
parent 1ae487a4f6
commit 7adbd7a785
1 changed files with 4 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
spacy/syntax/nonproj.pyx
View File

@ -1,8 +1,4 @@
from future.standard_library import install_aliases
install_aliases()
from copy import copy from copy import copy
from collections import Counter
from ..tokens.doc cimport Doc from ..tokens.doc cimport Doc
from spacy.attrs import DEP, HEAD from spacy.attrs import DEP, HEAD
@ -80,7 +76,7 @@ class PseudoProjectivity:
@classmethod @classmethod
def preprocess_training_data(cls, gold_tuples, label_freq_cutoff=30): def preprocess_training_data(cls, gold_tuples, label_freq_cutoff=30):
preprocessed = [] preprocessed = []
freqs = Counter() freqs = {}
for raw_text, sents in gold_tuples: for raw_text, sents in gold_tuples:
prepro_sents = [] prepro_sents = []
for (ids, words, tags, heads, labels, iob), ctnts in sents: for (ids, words, tags, heads, labels, iob), ctnts in sents:
@ -89,7 +85,9 @@ class PseudoProjectivity:
deco_labels = [ 'ROOT' if head == i else deco_labels[i] for i,head in enumerate(proj_heads) ] deco_labels = [ 'ROOT' if head == i else deco_labels[i] for i,head in enumerate(proj_heads) ]
# count label frequencies # count label frequencies
if label_freq_cutoff > 0: if label_freq_cutoff > 0:
freqs.update( label for label in deco_labels if cls.is_decorated(label) ) for label in deco_labels:
if cls.is_decorated(label):
freqs[label] = freqs.get(label,0) + 1
prepro_sents.append(((ids,words,tags,proj_heads,deco_labels,iob), ctnts)) prepro_sents.append(((ids,words,tags,proj_heads,deco_labels,iob), ctnts))
preprocessed.append((raw_text, prepro_sents)) preprocessed.append((raw_text, prepro_sents))