explosion/spaCy
1
1
Fork 0
mirror of https://github.com/explosion/spaCy.git synced 2024-09-21 11:29:13 +03:00
Code Issues Packages Projects Releases Wiki Activity

* Fix gather_freqs.py

Browse Source
This commit is contained in:
Matthew Honnibal 2016-02-04 20:21:58 +01:00
parent 48ce09687d
commit a66e2f2f53
1 changed files with 19 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
bin/gather_freqs.py
View File

@ -1,26 +1,28 @@
from __future__ import unicode_literals
import plac
import io
def main(in_loc, out_loc):
out_file = open(out_loc, 'w')
this_key = None
this_freq = 0
df = 0
for line in open(in_loc):
line = line.strip()
if not line:
continue
freq, key = line.split('\t', 1)
freq = int(freq)
if this_key is not None and key != this_key:
out_file.write('%d\t%d\t%s\n' % (this_freq, df, this_key))
this_key = key
this_freq = freq
df = 1
else:
this_freq += freq
df += 1
out_file.write('%d\t%d\t%s\n' % (this_freq, df, this_key))
out_file.close()
with io.open(out_loc, 'w', encoding='utf8') as out_file:
for line in io.open(in_loc, encoding='utf8'):
line = line.strip()
if not line:
continue
freq, key = line.split('\t', 1)
freq = int(freq)
if this_key is not None and key != this_key:
out_file.write('%d\t%d\t%s\n' % (this_freq, df, this_key))
this_key = key
this_freq = freq
df = 1
else:
this_freq += freq
df += 1
this_key = key
out_file.write('%d\t%d\t%s\n' % (this_freq, df, this_key))
if __name__ == '__main__':