Rooba
/
spaCy
mirror of https://github.com/explosion/spaCy.git
1
1
Fork 0
Code Issues Packages Projects Releases Wiki Activity

* Update get_freqs.py script

This commit is contained in:
Matthew Honnibal 2015-10-16 04:20:35 +11:00
parent 710e8fb168
commit e08a4b46a2
1 changed files with 10 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
bin/get_freqs.py
View File

@ -1,6 +1,6 @@
#!/usr/bin/env python
from __future__ import unicode_literals
from __future__ import unicode_literals, print_function
import plac
import joblib
@ -14,7 +14,7 @@ from joblib import Parallel, delayed
import spacy.en
from spacy.strings import StringStore
from spacy.en.attrs import ORTH
from spacy.attrs import ORTH
def iter_comments(loc):
@ -40,20 +40,19 @@ def null_props(string):
def count_freqs(input_loc, output_loc):
print output_loc
nlp = spacy.en.English(Parser=None, Tagger=None, Entity=None, load_vectors=False)
nlp.vocab.lexeme_props_getter = null_props
print(output_loc)
tokenizer = Tokenizer.from_dir(Vocab(), English.default_data_dir())
counts = PreshCounter()
tokenizer = nlp.tokenizer
for json_comment in iter_comments(input_loc):
doc = tokenizer(json_comment['body'])
doc.count_by(ORTH, counts=counts)
with codecs.open(output_loc, 'w', 'utf8') as file_:
for orth, freq in counts:
string = nlp.vocab.strings[orth]
file_.write('%d\t%s\n' % (freq, repr(string)))
string = tokenizer.vocab.strings[orth]
if not string.isspace():
file_.write('%d\t%s\n' % (freq, string))
def parallelize(func, iterator, n_jobs):
@ -64,12 +63,12 @@ def merge_counts(locs, out_loc):
string_map = StringStore()
counts = PreshCounter()
for loc in locs:
with codecs.open(loc, 'r', 'utf8') as file_:
with io.open(loc, 'r', encoding='utf8') as file_:
for line in file_:
freq, word = line.strip().split('\t', 1)
orth = string_map[word]
counts.inc(orth, int(freq))
with codecs.open(out_loc, 'w', 'utf8') as file_:
with io.open(out_loc, 'w', encoding='utf8') as file_:
for orth, count in counts:
string = string_map[orth]
file_.write('%d\t%s\n' % (count, string))
@ -98,7 +97,7 @@ def main(input_loc, freqs_dir, output_loc, n_jobs=2, skip_existing=False):
if tasks:
parallelize(count_freqs, tasks, n_jobs)
print "Merge"
print("Merge")
merge_counts(outputs, output_loc)