explosion/spaCy
1
1
Fork 0
mirror of https://github.com/explosion/spaCy.git synced 2024-12-26 01:46:28 +03:00
Code Issues Packages Projects Releases Wiki Activity

Fix init_model if there's no vocab (closes #4048) (#4049)

Browse Source
This commit is contained in:
Ines Montani 2019-08-01 17:26:09 +02:00 committed by GitHub
parent 925a852bb6
commit 8718ca8b1f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 28 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
spacy/cli/init_model.py
View File

@ -24,6 +24,7 @@ except ImportError:
ftfy = None ftfy = None
DEFAULT_OOV_PROB = -20
msg = Printer() msg = Printer()
@ -108,14 +109,21 @@ def open_file(loc):
def read_attrs_from_deprecated(freqs_loc, clusters_loc): def read_attrs_from_deprecated(freqs_loc, clusters_loc):
if freqs_loc is not None:
with msg.loading("Counting frequencies..."): with msg.loading("Counting frequencies..."):
probs, oov_prob = read_freqs(freqs_loc) if freqs_loc is not None else ({}, -20) probs, _ = read_freqs(freqs_loc)
msg.good("Counted frequencies") msg.good("Counted frequencies")
else:
probs, _ = ({}, DEFAULT_OOV_PROB)
if clusters_loc:
with msg.loading("Reading clusters..."): with msg.loading("Reading clusters..."):
clusters = read_clusters(clusters_loc) if clusters_loc else {} clusters = read_clusters(clusters_loc)
msg.good("Read clusters") msg.good("Read clusters")
else:
clusters = {}
lex_attrs = [] lex_attrs = []
sorted_probs = sorted(probs.items(), key=lambda item: item[1], reverse=True) sorted_probs = sorted(probs.items(), key=lambda item: item[1], reverse=True)
if len(sorted_probs):
for i, (word, prob) in tqdm(enumerate(sorted_probs)): for i, (word, prob) in tqdm(enumerate(sorted_probs)):
attrs = {"orth": word, "id": i, "prob": prob} attrs = {"orth": word, "id": i, "prob": prob}
# Decode as a little-endian string, so that we can do & 15 to get # Decode as a little-endian string, so that we can do & 15 to get
@ -142,8 +150,11 @@ def create_model(lang, lex_attrs):
lexeme.is_oov = False lexeme.is_oov = False
lex_added += 1 lex_added += 1
lex_added += 1 lex_added += 1
oov_prob = min(lex.prob for lex in nlp.vocab) if len(nlp.vocab):
nlp.vocab.cfg.update({"oov_prob": oov_prob - 1}) oov_prob = min(lex.prob for lex in nlp.vocab) - 1
else:
oov_prob = DEFAULT_OOV_PROB
nlp.vocab.cfg.update({"oov_prob": oov_prob})
return nlp return nlp