explosion/spaCy
1
1
Fork 0
mirror of https://github.com/explosion/spaCy.git synced 2025-06-05 21:53:05 +03:00
Code Issues Packages Projects Releases Wiki Activity

* Use a gzipped frequencies file in init_model

Browse Source
This commit is contained in:
Matthew Honnibal 2015-10-11 06:59:44 +02:00
parent 0090f79fbd
commit 92f750cf8b
1 changed files with 10 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
bin/init_model.py
View File

@ -42,7 +42,10 @@ import spacy.de
import spacy.fi import spacy.fi
import spacy.it import spacy.it
try:
unicode
except NameError:
unicode = str
def setup_tokenizer(lang_data_dir, tok_dir): def setup_tokenizer(lang_data_dir, tok_dir):
@ -112,8 +115,12 @@ def _read_freqs(loc, max_length=100, min_doc_freq=5, min_freq=200):
total += freq total += freq
counts.smooth() counts.smooth()
log_total = math.log(total) log_total = math.log(total)
if str(loc).endswith('gz'):
file_ = gzip.open(str(loc))
else:
file_ = loc.open()
probs = {} probs = {}
for line in loc.open(): for line in file_:
freq, doc_freq, key = line.split('\t', 2) freq, doc_freq, key = line.split('\t', 2)
doc_freq = int(doc_freq) doc_freq = int(doc_freq)
freq = int(freq) freq = int(freq)
@ -158,7 +165,7 @@ def setup_vocab(get_lex_attr, tag_map, src_dir, dst_dir):
clusters = _read_clusters(src_dir / 'clusters.txt') clusters = _read_clusters(src_dir / 'clusters.txt')
probs, oov_prob = _read_probs(src_dir / 'words.sgt.prob') probs, oov_prob = _read_probs(src_dir / 'words.sgt.prob')
if not probs: if not probs:
probs, oov_prob = _read_freqs(src_dir / 'freqs.txt') probs, oov_prob = _read_freqs(src_dir / 'freqs.txt.gz')
if not probs: if not probs:
oov_prob = -20 oov_prob = -20
else: else: