diff --git a/bin/get_freqs.py b/bin/get_freqs.py index c9f858b22..2633e6865 100755 --- a/bin/get_freqs.py +++ b/bin/get_freqs.py @@ -1,6 +1,6 @@ #!/usr/bin/env python -from __future__ import unicode_literals +from __future__ import unicode_literals, print_function import plac import joblib @@ -14,7 +14,7 @@ from joblib import Parallel, delayed import spacy.en from spacy.strings import StringStore -from spacy.en.attrs import ORTH +from spacy.attrs import ORTH def iter_comments(loc): @@ -40,20 +40,19 @@ def null_props(string): def count_freqs(input_loc, output_loc): - print output_loc - nlp = spacy.en.English(Parser=None, Tagger=None, Entity=None, load_vectors=False) - nlp.vocab.lexeme_props_getter = null_props + print(output_loc) + tokenizer = Tokenizer.from_dir(Vocab(), English.default_data_dir()) counts = PreshCounter() - tokenizer = nlp.tokenizer for json_comment in iter_comments(input_loc): doc = tokenizer(json_comment['body']) doc.count_by(ORTH, counts=counts) with codecs.open(output_loc, 'w', 'utf8') as file_: for orth, freq in counts: - string = nlp.vocab.strings[orth] - file_.write('%d\t%s\n' % (freq, repr(string))) + string = tokenizer.vocab.strings[orth] + if not string.isspace(): + file_.write('%d\t%s\n' % (freq, string)) def parallelize(func, iterator, n_jobs): @@ -64,12 +63,12 @@ def merge_counts(locs, out_loc): string_map = StringStore() counts = PreshCounter() for loc in locs: - with codecs.open(loc, 'r', 'utf8') as file_: + with io.open(loc, 'r', encoding='utf8') as file_: for line in file_: freq, word = line.strip().split('\t', 1) orth = string_map[word] counts.inc(orth, int(freq)) - with codecs.open(out_loc, 'w', 'utf8') as file_: + with io.open(out_loc, 'w', encoding='utf8') as file_: for orth, count in counts: string = string_map[orth] file_.write('%d\t%s\n' % (count, string)) @@ -98,7 +97,7 @@ def main(input_loc, freqs_dir, output_loc, n_jobs=2, skip_existing=False): if tasks: parallelize(count_freqs, tasks, n_jobs) - print "Merge" + print("Merge") merge_counts(outputs, output_loc)