explosion/spaCy
1
1
Fork 0
mirror of https://github.com/explosion/spaCy.git synced 2025-04-05 09:44:12 +03:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1055 from recognai/master

Browse Source 
Enable pruning out rare words from clusters data
This commit is contained in:
Ines Montani 2017-05-13 03:22:56 +02:00 committed by GitHub
commit 8d742ac8ff
1 changed files with 0 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
spacy/cli/model.py
View File

@ -98,10 +98,6 @@ def read_clusters(clusters_path):
def populate_vocab(vocab, clusters, probs, oov_prob):
# Ensure probs has entries for all words seen during clustering.
for word in clusters:
if word not in probs:
probs[word] = oov_prob
for word, prob in reversed(sorted(list(probs.items()), key=lambda item: item[1])):
lexeme = vocab[word]
lexeme.prob = prob