explosion/spaCy
1
1
Fork 0
mirror of https://github.com/explosion/spaCy.git synced 2025-07-15 02:32:37 +03:00
Code Issues Packages Projects Releases Wiki Activity

* Fix init_model

Browse Source
This commit is contained in:
Matthew Honnibal 2015-07-25 22:56:35 +02:00
parent 1a99eb69da
commit 0301472d15
1 changed files with 4 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
bin/init_model.py
View File

@ -18,6 +18,7 @@ Requires:
from __future__ import unicode_literals from __future__ import unicode_literals
from ast import literal_eval from ast import literal_eval
import math
import plac import plac
from pathlib import Path from pathlib import Path
@ -91,15 +92,15 @@ def _read_probs(loc):
def _read_freqs(loc): def _read_freqs(loc):
counts = PreshCounter() counts = PreshCounter()
total = 0 total = 0
for line in open(loc): for i, line in enumerate(loc.open()):
freq, doc_freq, key = line.split('\t', 2) freq, doc_freq, key = line.split('\t', 2)
freq = int(freq) freq = int(freq)
counts[hash_string(key)] = freq counts.inc(i+1, freq)
total += freq total += freq
counts.smooth() counts.smooth()
log_total = math.log(total) log_total = math.log(total)
probs = {} probs = {}
for line in open(loc): for line in loc.open():
freq, doc_freq, key = line.split('\t', 2) freq, doc_freq, key = line.split('\t', 2)
if int(doc_freq) >= 2 and int(freq) >= 5 and len(key) < 200: if int(doc_freq) >= 2 and int(freq) >= 5 and len(key) < 200:
word = literal_eval(key) word = literal_eval(key)