mirror of
https://github.com/explosion/spaCy.git
synced 2025-07-10 16:22:29 +03:00
* Fix init_model
This commit is contained in:
parent
1a99eb69da
commit
0301472d15
|
@ -18,6 +18,7 @@ Requires:
|
||||||
from __future__ import unicode_literals
|
from __future__ import unicode_literals
|
||||||
|
|
||||||
from ast import literal_eval
|
from ast import literal_eval
|
||||||
|
import math
|
||||||
|
|
||||||
import plac
|
import plac
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
@ -91,15 +92,15 @@ def _read_probs(loc):
|
||||||
def _read_freqs(loc):
|
def _read_freqs(loc):
|
||||||
counts = PreshCounter()
|
counts = PreshCounter()
|
||||||
total = 0
|
total = 0
|
||||||
for line in open(loc):
|
for i, line in enumerate(loc.open()):
|
||||||
freq, doc_freq, key = line.split('\t', 2)
|
freq, doc_freq, key = line.split('\t', 2)
|
||||||
freq = int(freq)
|
freq = int(freq)
|
||||||
counts[hash_string(key)] = freq
|
counts.inc(i+1, freq)
|
||||||
total += freq
|
total += freq
|
||||||
counts.smooth()
|
counts.smooth()
|
||||||
log_total = math.log(total)
|
log_total = math.log(total)
|
||||||
probs = {}
|
probs = {}
|
||||||
for line in open(loc):
|
for line in loc.open():
|
||||||
freq, doc_freq, key = line.split('\t', 2)
|
freq, doc_freq, key = line.split('\t', 2)
|
||||||
if int(doc_freq) >= 2 and int(freq) >= 5 and len(key) < 200:
|
if int(doc_freq) >= 2 and int(freq) >= 5 and len(key) < 200:
|
||||||
word = literal_eval(key)
|
word = literal_eval(key)
|
||||||
|
|
Loading…
Reference in New Issue
Block a user