2017-11-26 22:21:47 +00:00
|
|
|
# coding: utf8
|
|
|
|
from __future__ import unicode_literals
|
|
|
|
|
|
|
|
import plac
|
|
|
|
import math
|
|
|
|
from tqdm import tqdm
|
|
|
|
import numpy
|
|
|
|
from ast import literal_eval
|
|
|
|
from pathlib import Path
|
|
|
|
from preshed.counter import PreshCounter
|
|
|
|
|
2017-12-07 09:03:07 +00:00
|
|
|
from ..compat import fix_text
|
|
|
|
from ..vectors import Vectors
|
|
|
|
from ..util import prints, ensure_path, get_lang_class
|
2017-11-26 22:21:47 +00:00
|
|
|
|
|
|
|
|
|
|
|
@plac.annotations(
|
|
|
|
lang=("model language", "positional", None, str),
|
|
|
|
output_dir=("model output directory", "positional", None, Path),
|
2017-12-11 10:08:29 +00:00
|
|
|
freqs_loc=("location of words frequencies file", "positional", None, Path),
|
2017-11-26 22:21:47 +00:00
|
|
|
clusters_loc=("optional: location of brown clusters data",
|
2017-12-07 09:03:07 +00:00
|
|
|
"option", "c", str),
|
2017-11-26 22:21:47 +00:00
|
|
|
vectors_loc=("optional: location of vectors file in GenSim text format",
|
2017-12-07 09:03:07 +00:00
|
|
|
"option", "v", str),
|
2017-11-26 22:21:47 +00:00
|
|
|
prune_vectors=("optional: number of vectors to prune to",
|
|
|
|
"option", "V", int)
|
|
|
|
)
|
2018-01-04 20:33:47 +00:00
|
|
|
def init_model(lang, output_dir, freqs_loc, clusters_loc=None, vectors_loc=None, prune_vectors=-1):
|
2017-12-07 09:23:09 +00:00
|
|
|
"""
|
|
|
|
Create a new model from raw data, like word frequencies, Brown clusters
|
|
|
|
and word vectors.
|
|
|
|
"""
|
2017-11-26 22:21:47 +00:00
|
|
|
if not freqs_loc.exists():
|
|
|
|
prints(freqs_loc, title="Can't find words frequencies file", exits=1)
|
|
|
|
clusters_loc = ensure_path(clusters_loc)
|
|
|
|
vectors_loc = ensure_path(vectors_loc)
|
|
|
|
|
|
|
|
probs, oov_prob = read_freqs(freqs_loc)
|
2018-01-03 11:20:49 +00:00
|
|
|
vectors_data, vector_keys = read_vectors(vectors_loc) if vectors_loc else None, None
|
2017-11-26 22:21:47 +00:00
|
|
|
clusters = read_clusters(clusters_loc) if clusters_loc else {}
|
|
|
|
|
|
|
|
nlp = create_model(lang, probs, oov_prob, clusters, vectors_data, vector_keys, prune_vectors)
|
|
|
|
|
|
|
|
if not output_dir.exists():
|
|
|
|
output_dir.mkdir()
|
|
|
|
nlp.to_disk(output_dir)
|
|
|
|
return nlp
|
|
|
|
|
|
|
|
|
|
|
|
def create_model(lang, probs, oov_prob, clusters, vectors_data, vector_keys, prune_vectors):
|
2017-12-07 08:59:23 +00:00
|
|
|
print("Creating model...")
|
|
|
|
lang_class = get_lang_class(lang)
|
|
|
|
nlp = lang_class()
|
2017-11-26 22:21:47 +00:00
|
|
|
for lexeme in nlp.vocab:
|
|
|
|
lexeme.rank = 0
|
|
|
|
|
|
|
|
lex_added = 0
|
|
|
|
for i, (word, prob) in enumerate(tqdm(sorted(probs.items(), key=lambda item: item[1], reverse=True))):
|
|
|
|
lexeme = nlp.vocab[word]
|
|
|
|
lexeme.rank = i
|
|
|
|
lexeme.prob = prob
|
|
|
|
lexeme.is_oov = False
|
|
|
|
# Decode as a little-endian string, so that we can do & 15 to get
|
|
|
|
# the first 4 bits. See _parse_features.pyx
|
|
|
|
if word in clusters:
|
|
|
|
lexeme.cluster = int(clusters[word][::-1], 2)
|
|
|
|
else:
|
|
|
|
lexeme.cluster = 0
|
|
|
|
lex_added += 1
|
|
|
|
nlp.vocab.cfg.update({'oov_prob': oov_prob})
|
|
|
|
|
2018-01-03 11:20:49 +00:00
|
|
|
if vectors_data:
|
2017-11-26 22:21:47 +00:00
|
|
|
nlp.vocab.vectors = Vectors(data=vectors_data, keys=vector_keys)
|
|
|
|
if prune_vectors >= 1:
|
|
|
|
nlp.vocab.prune_vectors(prune_vectors)
|
|
|
|
vec_added = len(nlp.vocab.vectors)
|
|
|
|
|
|
|
|
prints("{} entries, {} vectors".format(lex_added, vec_added),
|
|
|
|
title="Sucessfully compiled vocab")
|
|
|
|
return nlp
|
|
|
|
|
|
|
|
|
|
|
|
def read_vectors(vectors_loc):
|
2017-12-07 08:59:23 +00:00
|
|
|
print("Reading vectors...")
|
2017-11-26 22:21:47 +00:00
|
|
|
with vectors_loc.open() as f:
|
|
|
|
shape = tuple(int(size) for size in f.readline().split())
|
|
|
|
vectors_data = numpy.zeros(shape=shape, dtype='f')
|
|
|
|
vectors_keys = []
|
|
|
|
for i, line in enumerate(tqdm(f)):
|
|
|
|
pieces = line.split()
|
|
|
|
word = pieces.pop(0)
|
|
|
|
vectors_data[i] = numpy.array([float(val_str) for val_str in pieces], dtype='f')
|
|
|
|
vectors_keys.append(word)
|
|
|
|
return vectors_data, vectors_keys
|
|
|
|
|
|
|
|
|
|
|
|
def read_freqs(freqs_loc, max_length=100, min_doc_freq=5, min_freq=50):
|
2017-12-07 08:59:23 +00:00
|
|
|
print("Counting frequencies...")
|
2017-11-26 22:21:47 +00:00
|
|
|
counts = PreshCounter()
|
|
|
|
total = 0
|
|
|
|
with freqs_loc.open() as f:
|
|
|
|
for i, line in enumerate(f):
|
|
|
|
freq, doc_freq, key = line.rstrip().split('\t', 2)
|
|
|
|
freq = int(freq)
|
|
|
|
counts.inc(i + 1, freq)
|
|
|
|
total += freq
|
|
|
|
counts.smooth()
|
|
|
|
log_total = math.log(total)
|
|
|
|
probs = {}
|
|
|
|
with freqs_loc.open() as f:
|
|
|
|
for line in tqdm(f):
|
|
|
|
freq, doc_freq, key = line.rstrip().split('\t', 2)
|
|
|
|
doc_freq = int(doc_freq)
|
|
|
|
freq = int(freq)
|
|
|
|
if doc_freq >= min_doc_freq and freq >= min_freq and len(key) < max_length:
|
|
|
|
word = literal_eval(key)
|
|
|
|
smooth_count = counts.smoother(int(freq))
|
|
|
|
probs[word] = math.log(smooth_count) - log_total
|
|
|
|
oov_prob = math.log(counts.smoother(0)) - log_total
|
|
|
|
return probs, oov_prob
|
|
|
|
|
|
|
|
|
|
|
|
def read_clusters(clusters_loc):
|
2017-12-07 08:59:23 +00:00
|
|
|
print("Reading clusters...")
|
2017-11-26 22:21:47 +00:00
|
|
|
clusters = {}
|
|
|
|
with clusters_loc.open() as f:
|
|
|
|
for line in tqdm(f):
|
|
|
|
try:
|
|
|
|
cluster, word, freq = line.split()
|
|
|
|
word = fix_text(word)
|
|
|
|
except ValueError:
|
|
|
|
continue
|
|
|
|
# If the clusterer has only seen the word a few times, its
|
|
|
|
# cluster is unreliable.
|
|
|
|
if int(freq) >= 3:
|
|
|
|
clusters[word] = cluster
|
|
|
|
else:
|
|
|
|
clusters[word] = '0'
|
|
|
|
# Expand clusters with re-casing
|
|
|
|
for word, cluster in list(clusters.items()):
|
|
|
|
if word.lower() not in clusters:
|
|
|
|
clusters[word.lower()] = cluster
|
|
|
|
if word.title() not in clusters:
|
|
|
|
clusters[word.title()] = cluster
|
|
|
|
if word.upper() not in clusters:
|
|
|
|
clusters[word.upper()] = cluster
|
|
|
|
return clusters
|