mirror of
https://github.com/explosion/spaCy.git
synced 2025-07-04 20:03:13 +03:00
Update spacy/cli/debug_data.py
This commit is contained in:
parent
382c296f7d
commit
fd00de42aa
|
@ -1073,7 +1073,8 @@ def _get_distribution(docs, normalize: bool = True) -> Counter:
|
||||||
word_counts: Counter = Counter()
|
word_counts: Counter = Counter()
|
||||||
for doc in docs:
|
for doc in docs:
|
||||||
for token in doc:
|
for token in doc:
|
||||||
word_counts[token.text] += 1
|
t = token.text.lower()
|
||||||
|
word_counts[t] += 1
|
||||||
if normalize:
|
if normalize:
|
||||||
total = sum(word_counts.values(), 0.0)
|
total = sum(word_counts.values(), 0.0)
|
||||||
word_counts = Counter({k: v / total for k, v in word_counts.items()})
|
word_counts = Counter({k: v / total for k, v in word_counts.items()})
|
||||||
|
|
Loading…
Reference in New Issue
Block a user