explosion/spaCy
1
1
Fork 0
mirror of https://github.com/explosion/spaCy.git synced 2024-09-21 11:29:13 +03:00
Code Issues Packages Projects Releases Wiki Activity

* Remove probability cap on lexicon

Browse Source
This commit is contained in:
Matthew Honnibal 2015-07-25 23:05:51 +02:00
parent c62eb110c0
commit 5b6bf4d4a6
1 changed files with 1 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
bin/init_model.py
View File

@ -158,14 +158,12 @@ def setup_vocab(src_dir, dst_dir):
lexicon = [] lexicon = []
for word, prob in reversed(sorted(list(probs.items()), key=lambda item: item[1])): for word, prob in reversed(sorted(list(probs.items()), key=lambda item: item[1])):
entry = get_lex_props(word) entry = get_lex_props(word)
if word in clusters or float(prob) >= -17: if word in clusters:
entry['prob'] = float(prob) entry['prob'] = float(prob)
cluster = clusters.get(word, '0') cluster = clusters.get(word, '0')
# Decode as a little-endian string, so that we can do & 15 to get # Decode as a little-endian string, so that we can do & 15 to get
# the first 4 bits. See _parse_features.pyx # the first 4 bits. See _parse_features.pyx
entry['cluster'] = int(cluster[::-1], 2) entry['cluster'] = int(cluster[::-1], 2)
orth_senses = set()
lemmas = []
vocab[word] = entry vocab[word] = entry
vocab.dump(str(dst_dir / 'lexemes.bin')) vocab.dump(str(dst_dir / 'lexemes.bin'))
vocab.strings.dump(str(dst_dir / 'strings.txt')) vocab.strings.dump(str(dst_dir / 'strings.txt'))