spaCy/spacy/util.py

76 lines
2.8 KiB
Python

def utf8open(loc, mode='r'):
return codecs.open(loc, mode, 'utf8')
def load_case_stats(data_dir):
case_loc = path.join(data_dir, 'english.case')
case_stats = {}
with utf8open(case_loc) as cases_file:
for line in cases_file:
word, upper, title = line.split()
case_stats[word] = (float(upper), float(title))
return case_stats
def load_clitics(data_dir):
clitics_loc = path.join(data_dir, 'clitics.txt')
entries = []
seen = set()
with utf8open(clitics_loc) as clitics_file:
for line in clitics_file:
line = line.strip()
if line.startswith('#'):
continue
if not line:
continue
clitics = line.split()
word = clitics.pop(0)
norm_form = clitics.pop(0)
assert word not in seen, word
seen.add(word)
entries.append((word, norm_form, clitics))
return entries
"""
def load_browns(self, data_dir):
cdef Lexeme* w
case_stats = load_case_stats(data_dir)
brown_loc = path.join(data_dir, 'bllip-clusters')
assert path.exists(brown_loc)
cdef size_t start
cdef int end
with utf8open(brown_loc) as browns_file:
for i, line in enumerate(browns_file):
cluster_str, word, freq_str = line.split()
# Decode as a little-endian string, so that we can do & 15 to get
# the first 4 bits. See redshift._parse_features.pyx
cluster = int(cluster_str[::-1], 2)
upper_pc, title_pc = case_stats.get(word.lower(), (0.0, 0.0))
start = 0
end = -1
find_slice(&start, &end, word)
print "Load", repr(word), start, end
w = <Lexeme*>init_word(word, start, end, cluster,
upper_pc, title_pc, int(freq_str))
self.words[_hash_str(word)] = <size_t>w
self.strings[<size_t>w] = word
def load_clitics(self, data_dir):
cdef unicode orig_str
cdef unicode clitic
for orig_str, norm_form, clitic_strs in util.load_clitics(data_dir):
w = init_clitic(orig_str, <Lexeme*>self.lookup_slice(norm_form, 0, -1))
self.words[w.orig] = <size_t>w
self.strings[<size_t>w] = orig_str
assert len(clitic_strs) < MAX_CLITICS
assert clitic_strs
for i, clitic in enumerate(clitic_strs):
# If we write punctuation here, assume we want to keep it,
# so tell it the slice boundaries (the full string)
w.clitics[i] = self.lookup_slice(clitic, 0, -1)
# Ensure we null terminate
w.clitics[i+1] = 0
"""