explosion/spaCy
1
1
Fork 0
mirror of https://github.com/explosion/spaCy.git synced 2025-07-14 18:22:27 +03:00
Code Issues Packages Projects Releases Wiki Activity

Update spacy/cli/debug_data.py

Browse Source
This commit is contained in:
Adriane Boyd 2023-12-20 17:39:45 +01:00 committed by GitHub
parent 382c296f7d
commit fd00de42aa
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 2 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
spacy/cli/debug_data.py
View File

@ -1073,7 +1073,8 @@ def _get_distribution(docs, normalize: bool = True) -> Counter:
word_counts: Counter = Counter() word_counts: Counter = Counter()
for doc in docs: for doc in docs:
for token in doc: for token in doc:
word_counts[token.text] += 1 t = token.text.lower()
word_counts[t] += 1
if normalize: if normalize:
total = sum(word_counts.values(), 0.0) total = sum(word_counts.values(), 0.0)
word_counts = Counter({k: v / total for k, v in word_counts.items()}) word_counts = Counter({k: v / total for k, v in word_counts.items()})