2017-10-27 12:39:30 +00:00
|
|
|
# coding: utf8
|
|
|
|
from __future__ import unicode_literals
|
|
|
|
|
|
|
|
import numpy
|
|
|
|
from thinc.v2v import Model, Maxout, Softmax, Affine, ReLu
|
2017-10-03 18:07:17 +00:00
|
|
|
from thinc.i2v import HashEmbed, StaticVectors
|
|
|
|
from thinc.t2t import ExtractWindow, ParametricAttention
|
2017-10-27 12:39:30 +00:00
|
|
|
from thinc.t2v import Pooling, sum_pool
|
2017-10-03 18:07:17 +00:00
|
|
|
from thinc.misc import Residual
|
|
|
|
from thinc.misc import LayerNorm as LN
|
2017-05-06 18:38:12 +00:00
|
|
|
from thinc.api import add, layerize, chain, clone, concatenate, with_flatten
|
2017-10-27 12:39:30 +00:00
|
|
|
from thinc.api import FeatureExtracter, with_getitem, flatten_add_lengths
|
|
|
|
from thinc.api import uniqued, wrap, noop
|
2018-09-13 17:28:35 +00:00
|
|
|
from thinc.api import with_square_sequences
|
2017-10-03 18:07:17 +00:00
|
|
|
from thinc.linear.linear import LinearModel
|
Update draft of parser neural network model
Model is good, but code is messy. Currently requires Chainer, which may cause the build to fail on machines without a GPU.
Outline of the model:
We first predict context-sensitive vectors for each word in the input:
(embed_lower | embed_prefix | embed_suffix | embed_shape)
>> Maxout(token_width)
>> convolution ** 4
This convolutional layer is shared between the tagger and the parser. This prevents the parser from needing tag features.
To boost the representation, we make a "super tag" with POS, morphology and dependency label. The tagger predicts this
by adding a softmax layer onto the convolutional layer --- so, we're teaching the convolutional layer to give us a
representation that's one affine transform from this informative lexical information. This is obviously good for the
parser (which backprops to the convolutions too).
The parser model makes a state vector by concatenating the vector representations for its context tokens. Current
results suggest few context tokens works well. Maybe this is a bug.
The current context tokens:
* S0, S1, S2: Top three words on the stack
* B0, B1: First two words of the buffer
* S0L1, S0L2: Leftmost and second leftmost children of S0
* S0R1, S0R2: Rightmost and second rightmost children of S0
* S1L1, S1L2, S1R2, S1R, B0L1, B0L2: Likewise for S1 and B0
This makes the state vector quite long: 13*T, where T is the token vector width (128 is working well). Fortunately,
there's a way to structure the computation to save some expense (and make it more GPU friendly).
The parser typically visits 2*N states for a sentence of length N (although it may visit more, if it back-tracks
with a non-monotonic transition). A naive implementation would require 2*N (B, 13*T) @ (13*T, H) matrix multiplications
for a batch of size B. We can instead perform one (B*N, T) @ (T, 13*H) multiplication, to pre-compute the hidden
weights for each positional feature wrt the words in the batch. (Note that our token vectors come from the CNN
-- so we can't play this trick over the vocabulary. That's how Stanford's NN parser works --- and why its model
is so big.)
This pre-computation strategy allows a nice compromise between GPU-friendliness and implementation simplicity.
The CNN and the wide lower layer are computed on the GPU, and then the precomputed hidden weights are moved
to the CPU, before we start the transition-based parsing process. This makes a lot of things much easier.
We don't have to worry about variable-length batch sizes, and we don't have to implement the dynamic oracle
in CUDA to train.
Currently the parser's loss function is multilabel log loss, as the dynamic oracle allows multiple states to
be 0 cost. This is defined as:
(exp(score) / Z) - (exp(score) / gZ)
Where gZ is the sum of the scores assigned to gold classes. I'm very interested in regressing on the cost directly,
but so far this isn't working well.
Machinery is in place for beam-search, which has been working well for the linear model. Beam search should benefit
greatly from the pre-computation trick.
2017-05-12 21:09:15 +00:00
|
|
|
from thinc.neural.ops import NumpyOps, CupyOps
|
2017-10-20 01:07:45 +00:00
|
|
|
from thinc.neural.util import get_array_module, copy_array
|
|
|
|
from thinc.neural._lsuv import svd_orthonormal
|
2017-11-06 13:25:37 +00:00
|
|
|
from thinc.neural.optimizers import Adam
|
2017-10-03 18:07:17 +00:00
|
|
|
|
2017-05-08 09:36:37 +00:00
|
|
|
from thinc import describe
|
|
|
|
from thinc.describe import Dimension, Synapses, Biases, Gradient
|
|
|
|
from thinc.neural._classes.affine import _set_dimensions_if_needed
|
2017-10-03 16:39:57 +00:00
|
|
|
import thinc.extra.load_nlp
|
2017-08-12 10:45:20 +00:00
|
|
|
|
2017-10-27 12:39:30 +00:00
|
|
|
from .attrs import ID, ORTH, LOWER, NORM, PREFIX, SUFFIX, SHAPE
|
2018-04-03 13:50:31 +00:00
|
|
|
from .errors import Errors
|
2017-08-18 19:55:23 +00:00
|
|
|
from . import util
|
2017-05-04 11:31:40 +00:00
|
|
|
|
2018-09-13 17:28:35 +00:00
|
|
|
try:
|
|
|
|
import torch.nn
|
2018-09-13 22:54:34 +00:00
|
|
|
from thinc.extra.wrappers import PyTorchWrapperRNN
|
2018-09-13 17:28:35 +00:00
|
|
|
except:
|
|
|
|
torch = None
|
2017-10-03 18:29:58 +00:00
|
|
|
|
2017-09-22 14:38:22 +00:00
|
|
|
VECTORS_KEY = 'spacy_pretrained_vectors'
|
2017-05-08 09:36:37 +00:00
|
|
|
|
2017-10-27 12:39:30 +00:00
|
|
|
|
2017-10-31 01:00:26 +00:00
|
|
|
def cosine(vec1, vec2):
|
2017-10-31 10:40:46 +00:00
|
|
|
xp = get_array_module(vec1)
|
|
|
|
norm1 = xp.linalg.norm(vec1)
|
|
|
|
norm2 = xp.linalg.norm(vec2)
|
|
|
|
if norm1 == 0. or norm2 == 0.:
|
|
|
|
return 0
|
|
|
|
else:
|
|
|
|
return vec1.dot(vec2) / (norm1 * norm2)
|
2017-10-31 01:00:26 +00:00
|
|
|
|
|
|
|
|
2017-11-06 13:11:59 +00:00
|
|
|
def create_default_optimizer(ops, **cfg):
|
|
|
|
learn_rate = util.env_opt('learn_rate', 0.001)
|
|
|
|
beta1 = util.env_opt('optimizer_B1', 0.9)
|
2018-05-01 13:18:20 +00:00
|
|
|
beta2 = util.env_opt('optimizer_B2', 0.9)
|
|
|
|
eps = util.env_opt('optimizer_eps', 1e-12)
|
2017-11-06 13:11:59 +00:00
|
|
|
L2 = util.env_opt('L2_penalty', 1e-6)
|
|
|
|
max_grad_norm = util.env_opt('grad_norm_clip', 1.)
|
|
|
|
optimizer = Adam(ops, learn_rate, L2=L2, beta1=beta1,
|
|
|
|
beta2=beta2, eps=eps)
|
|
|
|
optimizer.max_grad_norm = max_grad_norm
|
2017-11-06 13:25:37 +00:00
|
|
|
optimizer.device = ops.device
|
2017-11-06 13:11:59 +00:00
|
|
|
return optimizer
|
|
|
|
|
2017-07-22 18:03:40 +00:00
|
|
|
@layerize
|
|
|
|
def _flatten_add_lengths(seqs, pad=0, drop=0.):
|
|
|
|
ops = Model.ops
|
|
|
|
lengths = ops.asarray([len(seq) for seq in seqs], dtype='i')
|
2017-10-27 12:39:30 +00:00
|
|
|
|
2017-07-22 18:03:40 +00:00
|
|
|
def finish_update(d_X, sgd=None):
|
|
|
|
return ops.unflatten(d_X, lengths, pad=pad)
|
2017-10-27 12:39:30 +00:00
|
|
|
|
2017-07-22 18:03:40 +00:00
|
|
|
X = ops.flatten(seqs, pad=pad)
|
|
|
|
return (X, lengths), finish_update
|
|
|
|
|
|
|
|
|
|
|
|
def _zero_init(model):
|
|
|
|
def _zero_init_impl(self, X, y):
|
|
|
|
self.W.fill(0)
|
|
|
|
model.on_data_hooks.append(_zero_init_impl)
|
|
|
|
if model.W is not None:
|
|
|
|
model.W.fill(0.)
|
|
|
|
return model
|
|
|
|
|
2017-08-18 19:55:23 +00:00
|
|
|
|
2017-07-22 18:03:40 +00:00
|
|
|
@layerize
|
|
|
|
def _preprocess_doc(docs, drop=0.):
|
2017-11-11 23:43:41 +00:00
|
|
|
keys = [doc.to_array(LOWER) for doc in docs]
|
2017-07-22 18:03:40 +00:00
|
|
|
ops = Model.ops
|
2017-11-08 12:24:47 +00:00
|
|
|
# The dtype here matches what thinc is expecting -- which differs per
|
|
|
|
# platform (by int definition). This should be fixed once the problem
|
|
|
|
# is fixed on Thinc's side.
|
|
|
|
lengths = ops.asarray([arr.shape[0] for arr in keys], dtype=numpy.int_)
|
2017-07-22 18:03:40 +00:00
|
|
|
keys = ops.xp.concatenate(keys)
|
2017-11-11 23:43:41 +00:00
|
|
|
vals = ops.allocate(keys.shape) + 1.
|
|
|
|
return (keys, vals, lengths), None
|
|
|
|
|
|
|
|
@layerize
|
|
|
|
def _preprocess_doc_bigrams(docs, drop=0.):
|
|
|
|
unigrams = [doc.to_array(LOWER) for doc in docs]
|
|
|
|
ops = Model.ops
|
|
|
|
bigrams = [ops.ngrams(2, doc_unis) for doc_unis in unigrams]
|
|
|
|
keys = [ops.xp.concatenate(feats) for feats in zip(unigrams, bigrams)]
|
|
|
|
keys, vals = zip(*[ops.xp.unique(k, return_counts=True) for k in keys])
|
|
|
|
# The dtype here matches what thinc is expecting -- which differs per
|
|
|
|
# platform (by int definition). This should be fixed once the problem
|
|
|
|
# is fixed on Thinc's side.
|
|
|
|
lengths = ops.asarray([arr.shape[0] for arr in keys], dtype=numpy.int_)
|
|
|
|
keys = ops.xp.concatenate(keys)
|
|
|
|
vals = ops.asarray(ops.xp.concatenate(vals), dtype='f')
|
2017-07-22 18:03:40 +00:00
|
|
|
return (keys, vals, lengths), None
|
|
|
|
|
|
|
|
|
2017-10-20 01:07:45 +00:00
|
|
|
@describe.on_data(_set_dimensions_if_needed,
|
|
|
|
lambda model, X, y: model.init_weights(model))
|
2017-05-08 09:36:37 +00:00
|
|
|
@describe.attributes(
|
|
|
|
nI=Dimension("Input size"),
|
|
|
|
nF=Dimension("Number of features"),
|
|
|
|
nO=Dimension("Output size"),
|
2017-10-20 14:23:31 +00:00
|
|
|
nP=Dimension("Maxout pieces"),
|
2017-05-08 09:36:37 +00:00
|
|
|
W=Synapses("Weights matrix",
|
2017-10-28 11:16:06 +00:00
|
|
|
lambda obj: (obj.nF, obj.nO, obj.nP, obj.nI)),
|
2017-05-08 09:36:37 +00:00
|
|
|
b=Biases("Bias vector",
|
2017-10-28 11:16:06 +00:00
|
|
|
lambda obj: (obj.nO, obj.nP)),
|
2017-10-28 16:45:14 +00:00
|
|
|
pad=Synapses("Pad",
|
|
|
|
lambda obj: (1, obj.nF, obj.nO, obj.nP),
|
|
|
|
lambda M, ops: ops.normal_init(M, 1.)),
|
2017-05-08 09:36:37 +00:00
|
|
|
d_W=Gradient("W"),
|
2017-10-28 16:45:14 +00:00
|
|
|
d_pad=Gradient("pad"),
|
2017-10-27 12:39:30 +00:00
|
|
|
d_b=Gradient("b"))
|
2017-05-08 09:36:37 +00:00
|
|
|
class PrecomputableAffine(Model):
|
2017-10-20 14:23:31 +00:00
|
|
|
def __init__(self, nO=None, nI=None, nF=None, nP=None, **kwargs):
|
2017-05-08 09:36:37 +00:00
|
|
|
Model.__init__(self, **kwargs)
|
|
|
|
self.nO = nO
|
2017-10-20 14:23:31 +00:00
|
|
|
self.nP = nP
|
2017-05-08 09:36:37 +00:00
|
|
|
self.nI = nI
|
|
|
|
self.nF = nF
|
|
|
|
|
2017-10-20 10:14:52 +00:00
|
|
|
def begin_update(self, X, drop=0.):
|
2018-03-27 17:23:02 +00:00
|
|
|
Yf = self.ops.gemm(X,
|
|
|
|
self.W.reshape((self.nF*self.nO*self.nP, self.nI)), trans2=True)
|
2017-10-28 16:45:14 +00:00
|
|
|
Yf = Yf.reshape((Yf.shape[0], self.nF, self.nO, self.nP))
|
|
|
|
Yf = self._add_padding(Yf)
|
2017-10-19 11:44:49 +00:00
|
|
|
|
2017-05-08 09:36:37 +00:00
|
|
|
def backward(dY_ids, sgd=None):
|
|
|
|
dY, ids = dY_ids
|
2017-10-28 16:45:14 +00:00
|
|
|
dY, ids = self._backprop_padding(dY, ids)
|
2017-05-08 09:36:37 +00:00
|
|
|
Xf = X[ids]
|
2017-10-27 10:18:36 +00:00
|
|
|
Xf = Xf.reshape((Xf.shape[0], self.nF * self.nI))
|
2017-10-20 10:14:52 +00:00
|
|
|
|
2017-10-19 16:42:11 +00:00
|
|
|
self.d_b += dY.sum(axis=0)
|
2017-10-27 10:18:36 +00:00
|
|
|
dY = dY.reshape((dY.shape[0], self.nO*self.nP))
|
|
|
|
|
|
|
|
Wopfi = self.W.transpose((1, 2, 0, 3))
|
|
|
|
Wopfi = self.ops.xp.ascontiguousarray(Wopfi)
|
|
|
|
Wopfi = Wopfi.reshape((self.nO*self.nP, self.nF * self.nI))
|
2018-03-27 17:23:02 +00:00
|
|
|
dXf = self.ops.gemm(dY.reshape((dY.shape[0], self.nO*self.nP)), Wopfi)
|
2017-10-28 16:45:14 +00:00
|
|
|
|
2017-10-27 10:18:36 +00:00
|
|
|
# Reuse the buffer
|
|
|
|
dWopfi = Wopfi; dWopfi.fill(0.)
|
2018-03-27 17:23:02 +00:00
|
|
|
self.ops.gemm(dY, Xf, out=dWopfi, trans1=True)
|
2017-10-27 10:18:36 +00:00
|
|
|
dWopfi = dWopfi.reshape((self.nO, self.nP, self.nF, self.nI))
|
|
|
|
# (o, p, f, i) --> (f, o, p, i)
|
|
|
|
self.d_W += dWopfi.transpose((2, 0, 1, 3))
|
2017-05-08 09:36:37 +00:00
|
|
|
|
|
|
|
if sgd is not None:
|
|
|
|
sgd(self._mem.weights, self._mem.gradient, key=self.id)
|
2017-10-27 10:18:36 +00:00
|
|
|
return dXf.reshape((dXf.shape[0], self.nF, self.nI))
|
2017-05-08 09:36:37 +00:00
|
|
|
return Yf, backward
|
2018-04-03 13:50:31 +00:00
|
|
|
|
2017-10-28 16:45:14 +00:00
|
|
|
def _add_padding(self, Yf):
|
|
|
|
Yf_padded = self.ops.xp.vstack((self.pad, Yf))
|
2017-10-31 01:33:34 +00:00
|
|
|
return Yf_padded
|
2017-10-28 16:45:14 +00:00
|
|
|
|
|
|
|
def _backprop_padding(self, dY, ids):
|
2017-10-31 01:33:34 +00:00
|
|
|
# (1, nF, nO, nP) += (nN, nF, nO, nP) where IDs (nN, nF) < 0
|
2017-11-03 00:54:34 +00:00
|
|
|
mask = ids < 0.
|
|
|
|
mask = mask.sum(axis=1)
|
|
|
|
d_pad = dY * mask.reshape((ids.shape[0], 1, 1))
|
2017-11-02 23:49:11 +00:00
|
|
|
self.d_pad += d_pad.sum(axis=0)
|
2017-10-28 16:45:14 +00:00
|
|
|
return dY, ids
|
2017-05-08 09:36:37 +00:00
|
|
|
|
2017-10-20 01:07:45 +00:00
|
|
|
@staticmethod
|
|
|
|
def init_weights(model):
|
|
|
|
'''This is like the 'layer sequential unit variance', but instead
|
|
|
|
of taking the actual inputs, we randomly generate whitened data.
|
|
|
|
|
|
|
|
Why's this all so complicated? We have a huge number of inputs,
|
|
|
|
and the maxout unit makes guessing the dynamics tricky. Instead
|
|
|
|
we set the maxout weights to values that empirically result in
|
|
|
|
whitened outputs given whitened inputs.
|
|
|
|
'''
|
|
|
|
if (model.W**2).sum() != 0.:
|
|
|
|
return
|
2017-10-31 01:33:34 +00:00
|
|
|
ops = model.ops
|
|
|
|
xp = ops.xp
|
|
|
|
ops.normal_init(model.W, model.nF * model.nI, inplace=True)
|
|
|
|
|
|
|
|
ids = ops.allocate((5000, model.nF), dtype='f')
|
|
|
|
ids += xp.random.uniform(0, 1000, ids.shape)
|
|
|
|
ids = ops.asarray(ids, dtype='i')
|
|
|
|
tokvecs = ops.allocate((5000, model.nI), dtype='f')
|
|
|
|
tokvecs += xp.random.normal(loc=0., scale=1.,
|
2017-10-20 01:07:45 +00:00
|
|
|
size=tokvecs.size).reshape(tokvecs.shape)
|
|
|
|
|
|
|
|
def predict(ids, tokvecs):
|
2018-05-02 01:35:59 +00:00
|
|
|
# nS ids. nW tokvecs. Exclude the padding array.
|
|
|
|
hiddens = model(tokvecs[:-1]) # (nW, f, o, p)
|
|
|
|
vectors = model.ops.allocate((ids.shape[0], model.nO * model.nP), dtype='f')
|
2017-10-28 17:05:01 +00:00
|
|
|
# need nS vectors
|
2018-05-02 01:35:59 +00:00
|
|
|
hiddens = hiddens.reshape((hiddens.shape[0] * model.nF, model.nO * model.nP))
|
|
|
|
model.ops.scatter_add(vectors, ids.flatten(), hiddens)
|
|
|
|
vectors = vectors.reshape((vectors.shape[0], model.nO, model.nP))
|
2017-10-28 17:05:01 +00:00
|
|
|
vectors += model.b
|
2018-05-02 01:35:59 +00:00
|
|
|
vectors = model.ops.asarray(vectors)
|
2017-10-20 01:07:45 +00:00
|
|
|
if model.nP >= 2:
|
2017-10-28 17:05:01 +00:00
|
|
|
return model.ops.maxout(vectors)[0]
|
2017-10-20 01:07:45 +00:00
|
|
|
else:
|
2017-10-28 17:05:01 +00:00
|
|
|
return vectors * (vectors >= 0)
|
2017-10-20 01:07:45 +00:00
|
|
|
|
|
|
|
tol_var = 0.01
|
|
|
|
tol_mean = 0.01
|
|
|
|
t_max = 10
|
|
|
|
t_i = 0
|
|
|
|
for t_i in range(t_max):
|
|
|
|
acts1 = predict(ids, tokvecs)
|
2017-10-31 01:33:34 +00:00
|
|
|
var = model.ops.xp.var(acts1)
|
|
|
|
mean = model.ops.xp.mean(acts1)
|
2017-10-20 01:07:45 +00:00
|
|
|
if abs(var - 1.0) >= tol_var:
|
2017-10-31 01:33:34 +00:00
|
|
|
model.W /= model.ops.xp.sqrt(var)
|
2017-10-20 01:07:45 +00:00
|
|
|
elif abs(mean) >= tol_mean:
|
|
|
|
model.b -= mean
|
|
|
|
else:
|
|
|
|
break
|
|
|
|
|
2017-05-08 12:24:43 +00:00
|
|
|
|
2017-09-22 14:38:36 +00:00
|
|
|
def link_vectors_to_models(vocab):
|
|
|
|
vectors = vocab.vectors
|
2018-03-28 14:02:59 +00:00
|
|
|
if vectors.name is None:
|
2018-03-28 16:24:53 +00:00
|
|
|
vectors.name = VECTORS_KEY
|
2018-05-19 16:51:55 +00:00
|
|
|
if vectors.data.size != 0:
|
|
|
|
print(
|
|
|
|
"Warning: Unnamed vectors -- this won't allow multiple vectors "
|
|
|
|
"models to be loaded. (Shape: (%d, %d))" % vectors.data.shape)
|
2017-09-22 14:38:36 +00:00
|
|
|
ops = Model.ops
|
|
|
|
for word in vocab:
|
|
|
|
if word.orth in vectors.key2row:
|
|
|
|
word.rank = vectors.key2row[word.orth]
|
|
|
|
else:
|
|
|
|
word.rank = 0
|
|
|
|
data = ops.asarray(vectors.data)
|
|
|
|
# Set an entry here, so that vectors are accessed by StaticVectors
|
|
|
|
# (unideal, I know)
|
2018-03-28 14:02:59 +00:00
|
|
|
thinc.extra.load_nlp.VECTORS[(ops.device, vectors.name)] = data
|
2017-08-18 19:55:23 +00:00
|
|
|
|
2017-10-27 12:39:30 +00:00
|
|
|
|
2018-09-13 17:28:35 +00:00
|
|
|
def PyTorchBiLSTM(nO, nI, depth, dropout=0.2):
|
|
|
|
if depth == 0:
|
2018-09-13 22:54:34 +00:00
|
|
|
return layerize(noop())
|
2018-09-13 17:28:35 +00:00
|
|
|
model = torch.nn.LSTM(nI, nO//2, depth, bidirectional=True, dropout=dropout)
|
|
|
|
return with_square_sequences(PyTorchWrapperRNN(model))
|
|
|
|
|
|
|
|
|
2017-09-21 12:59:48 +00:00
|
|
|
def Tok2Vec(width, embed_size, **kwargs):
|
2018-03-28 14:02:59 +00:00
|
|
|
pretrained_vectors = kwargs.get('pretrained_vectors', None)
|
2017-10-11 07:44:17 +00:00
|
|
|
cnn_maxout_pieces = kwargs.get('cnn_maxout_pieces', 2)
|
2018-08-26 23:50:48 +00:00
|
|
|
subword_features = kwargs.get('subword_features', True)
|
|
|
|
conv_depth = kwargs.get('conv_depth', 4)
|
2018-09-13 17:28:35 +00:00
|
|
|
bilstm_depth = kwargs.get('bilstm_depth', 0)
|
2017-08-18 19:55:23 +00:00
|
|
|
cols = [ID, NORM, PREFIX, SUFFIX, SHAPE, ORTH]
|
2017-10-27 12:39:30 +00:00
|
|
|
with Model.define_operators({'>>': chain, '|': concatenate, '**': clone,
|
|
|
|
'+': add, '*': reapply}):
|
|
|
|
norm = HashEmbed(width, embed_size, column=cols.index(NORM),
|
|
|
|
name='embed_norm')
|
2018-08-26 23:50:48 +00:00
|
|
|
if subword_features:
|
|
|
|
prefix = HashEmbed(width, embed_size//2, column=cols.index(PREFIX),
|
|
|
|
name='embed_prefix')
|
|
|
|
suffix = HashEmbed(width, embed_size//2, column=cols.index(SUFFIX),
|
|
|
|
name='embed_suffix')
|
|
|
|
shape = HashEmbed(width, embed_size//2, column=cols.index(SHAPE),
|
|
|
|
name='embed_shape')
|
|
|
|
else:
|
2018-08-27 00:12:39 +00:00
|
|
|
prefix, suffix, shape = (None, None, None)
|
2018-03-28 14:02:59 +00:00
|
|
|
if pretrained_vectors is not None:
|
|
|
|
glove = StaticVectors(pretrained_vectors, width, column=cols.index(ID))
|
2017-09-22 14:38:36 +00:00
|
|
|
|
2018-08-26 23:50:48 +00:00
|
|
|
if subword_features:
|
|
|
|
embed = uniqued(
|
|
|
|
(glove | norm | prefix | suffix | shape)
|
|
|
|
>> LN(Maxout(width, width*5, pieces=3)), column=cols.index(ORTH))
|
|
|
|
else:
|
|
|
|
embed = uniqued(
|
|
|
|
(glove | norm)
|
|
|
|
>> LN(Maxout(width, width*2, pieces=3)), column=cols.index(ORTH))
|
|
|
|
elif subword_features:
|
2017-09-22 14:38:36 +00:00
|
|
|
embed = uniqued(
|
|
|
|
(norm | prefix | suffix | shape)
|
2018-03-28 14:02:59 +00:00
|
|
|
>> LN(Maxout(width, width*4, pieces=3)), column=cols.index(ORTH))
|
2018-08-26 23:50:48 +00:00
|
|
|
else:
|
|
|
|
embed = norm
|
2017-09-22 14:38:36 +00:00
|
|
|
|
2017-09-21 00:14:41 +00:00
|
|
|
convolution = Residual(
|
|
|
|
ExtractWindow(nW=1)
|
|
|
|
>> LN(Maxout(width, width*3, pieces=cnn_maxout_pieces))
|
|
|
|
)
|
2017-09-22 14:38:36 +00:00
|
|
|
tok2vec = (
|
|
|
|
FeatureExtracter(cols)
|
2017-10-28 17:05:11 +00:00
|
|
|
>> with_flatten(
|
|
|
|
embed
|
2018-08-26 23:50:48 +00:00
|
|
|
>> convolution ** conv_depth, pad=conv_depth
|
2017-10-28 17:05:11 +00:00
|
|
|
)
|
2017-09-22 14:38:36 +00:00
|
|
|
)
|
2018-09-14 20:51:58 +00:00
|
|
|
if bilstm_depth >= 1:
|
|
|
|
tok2vec = tok2vec >> PyTorchBiLSTM(width, width, bilstm_depth)
|
2017-05-15 19:46:08 +00:00
|
|
|
# Work around thinc API limitations :(. TODO: Revise in Thinc 7
|
|
|
|
tok2vec.nO = width
|
2017-05-29 22:53:29 +00:00
|
|
|
tok2vec.embed = embed
|
2017-05-15 19:46:08 +00:00
|
|
|
return tok2vec
|
|
|
|
|
2017-05-04 11:31:40 +00:00
|
|
|
|
2017-09-22 14:37:03 +00:00
|
|
|
def reapply(layer, n_times):
|
|
|
|
def reapply_fwd(X, drop=0.):
|
|
|
|
backprops = []
|
|
|
|
for i in range(n_times):
|
|
|
|
Y, backprop = layer.begin_update(X, drop=drop)
|
|
|
|
X = Y
|
|
|
|
backprops.append(backprop)
|
2017-10-27 12:39:30 +00:00
|
|
|
|
2017-09-22 14:37:03 +00:00
|
|
|
def reapply_bwd(dY, sgd=None):
|
|
|
|
dX = None
|
|
|
|
for backprop in reversed(backprops):
|
|
|
|
dY = backprop(dY, sgd=sgd)
|
|
|
|
if dX is None:
|
|
|
|
dX = dY
|
|
|
|
else:
|
|
|
|
dX += dY
|
|
|
|
return dX
|
2017-10-27 12:39:30 +00:00
|
|
|
|
2017-09-22 14:37:03 +00:00
|
|
|
return Y, reapply_bwd
|
|
|
|
return wrap(reapply_fwd, layer)
|
|
|
|
|
|
|
|
|
2017-05-22 09:47:47 +00:00
|
|
|
def asarray(ops, dtype):
|
|
|
|
def forward(X, drop=0.):
|
|
|
|
return ops.asarray(X, dtype=dtype), None
|
|
|
|
return layerize(forward)
|
|
|
|
|
|
|
|
|
2017-05-20 11:40:10 +00:00
|
|
|
def _divide_array(X, size):
|
|
|
|
parts = []
|
|
|
|
index = 0
|
|
|
|
while index < len(X):
|
2017-10-27 12:39:30 +00:00
|
|
|
parts.append(X[index:index + size])
|
2017-05-20 11:40:10 +00:00
|
|
|
index += size
|
|
|
|
return parts
|
|
|
|
|
|
|
|
|
2017-05-04 11:31:40 +00:00
|
|
|
def get_col(idx):
|
2018-04-03 13:50:31 +00:00
|
|
|
if idx < 0:
|
|
|
|
raise IndexError(Errors.E066.format(value=idx))
|
2017-10-27 12:39:30 +00:00
|
|
|
|
2017-05-04 11:31:40 +00:00
|
|
|
def forward(X, drop=0.):
|
Update draft of parser neural network model
Model is good, but code is messy. Currently requires Chainer, which may cause the build to fail on machines without a GPU.
Outline of the model:
We first predict context-sensitive vectors for each word in the input:
(embed_lower | embed_prefix | embed_suffix | embed_shape)
>> Maxout(token_width)
>> convolution ** 4
This convolutional layer is shared between the tagger and the parser. This prevents the parser from needing tag features.
To boost the representation, we make a "super tag" with POS, morphology and dependency label. The tagger predicts this
by adding a softmax layer onto the convolutional layer --- so, we're teaching the convolutional layer to give us a
representation that's one affine transform from this informative lexical information. This is obviously good for the
parser (which backprops to the convolutions too).
The parser model makes a state vector by concatenating the vector representations for its context tokens. Current
results suggest few context tokens works well. Maybe this is a bug.
The current context tokens:
* S0, S1, S2: Top three words on the stack
* B0, B1: First two words of the buffer
* S0L1, S0L2: Leftmost and second leftmost children of S0
* S0R1, S0R2: Rightmost and second rightmost children of S0
* S1L1, S1L2, S1R2, S1R, B0L1, B0L2: Likewise for S1 and B0
This makes the state vector quite long: 13*T, where T is the token vector width (128 is working well). Fortunately,
there's a way to structure the computation to save some expense (and make it more GPU friendly).
The parser typically visits 2*N states for a sentence of length N (although it may visit more, if it back-tracks
with a non-monotonic transition). A naive implementation would require 2*N (B, 13*T) @ (13*T, H) matrix multiplications
for a batch of size B. We can instead perform one (B*N, T) @ (T, 13*H) multiplication, to pre-compute the hidden
weights for each positional feature wrt the words in the batch. (Note that our token vectors come from the CNN
-- so we can't play this trick over the vocabulary. That's how Stanford's NN parser works --- and why its model
is so big.)
This pre-computation strategy allows a nice compromise between GPU-friendliness and implementation simplicity.
The CNN and the wide lower layer are computed on the GPU, and then the precomputed hidden weights are moved
to the CPU, before we start the transition-based parsing process. This makes a lot of things much easier.
We don't have to worry about variable-length batch sizes, and we don't have to implement the dynamic oracle
in CUDA to train.
Currently the parser's loss function is multilabel log loss, as the dynamic oracle allows multiple states to
be 0 cost. This is defined as:
(exp(score) / Z) - (exp(score) / gZ)
Where gZ is the sum of the scores assigned to gold classes. I'm very interested in regressing on the cost directly,
but so far this isn't working well.
Machinery is in place for beam-search, which has been working well for the linear model. Beam search should benefit
greatly from the pre-computation trick.
2017-05-12 21:09:15 +00:00
|
|
|
if isinstance(X, numpy.ndarray):
|
|
|
|
ops = NumpyOps()
|
|
|
|
else:
|
|
|
|
ops = CupyOps()
|
2017-05-18 11:21:32 +00:00
|
|
|
output = ops.xp.ascontiguousarray(X[:, idx], dtype=X.dtype)
|
2017-10-27 12:39:30 +00:00
|
|
|
|
2017-05-06 18:38:12 +00:00
|
|
|
def backward(y, sgd=None):
|
Update draft of parser neural network model
Model is good, but code is messy. Currently requires Chainer, which may cause the build to fail on machines without a GPU.
Outline of the model:
We first predict context-sensitive vectors for each word in the input:
(embed_lower | embed_prefix | embed_suffix | embed_shape)
>> Maxout(token_width)
>> convolution ** 4
This convolutional layer is shared between the tagger and the parser. This prevents the parser from needing tag features.
To boost the representation, we make a "super tag" with POS, morphology and dependency label. The tagger predicts this
by adding a softmax layer onto the convolutional layer --- so, we're teaching the convolutional layer to give us a
representation that's one affine transform from this informative lexical information. This is obviously good for the
parser (which backprops to the convolutions too).
The parser model makes a state vector by concatenating the vector representations for its context tokens. Current
results suggest few context tokens works well. Maybe this is a bug.
The current context tokens:
* S0, S1, S2: Top three words on the stack
* B0, B1: First two words of the buffer
* S0L1, S0L2: Leftmost and second leftmost children of S0
* S0R1, S0R2: Rightmost and second rightmost children of S0
* S1L1, S1L2, S1R2, S1R, B0L1, B0L2: Likewise for S1 and B0
This makes the state vector quite long: 13*T, where T is the token vector width (128 is working well). Fortunately,
there's a way to structure the computation to save some expense (and make it more GPU friendly).
The parser typically visits 2*N states for a sentence of length N (although it may visit more, if it back-tracks
with a non-monotonic transition). A naive implementation would require 2*N (B, 13*T) @ (13*T, H) matrix multiplications
for a batch of size B. We can instead perform one (B*N, T) @ (T, 13*H) multiplication, to pre-compute the hidden
weights for each positional feature wrt the words in the batch. (Note that our token vectors come from the CNN
-- so we can't play this trick over the vocabulary. That's how Stanford's NN parser works --- and why its model
is so big.)
This pre-computation strategy allows a nice compromise between GPU-friendliness and implementation simplicity.
The CNN and the wide lower layer are computed on the GPU, and then the precomputed hidden weights are moved
to the CPU, before we start the transition-based parsing process. This makes a lot of things much easier.
We don't have to worry about variable-length batch sizes, and we don't have to implement the dynamic oracle
in CUDA to train.
Currently the parser's loss function is multilabel log loss, as the dynamic oracle allows multiple states to
be 0 cost. This is defined as:
(exp(score) / Z) - (exp(score) / gZ)
Where gZ is the sum of the scores assigned to gold classes. I'm very interested in regressing on the cost directly,
but so far this isn't working well.
Machinery is in place for beam-search, which has been working well for the linear model. Beam search should benefit
greatly from the pre-computation trick.
2017-05-12 21:09:15 +00:00
|
|
|
dX = ops.allocate(X.shape)
|
2017-05-06 18:38:12 +00:00
|
|
|
dX[:, idx] += y
|
|
|
|
return dX
|
2017-05-04 11:31:40 +00:00
|
|
|
|
2017-10-27 12:39:30 +00:00
|
|
|
return output, backward
|
2017-05-04 11:31:40 +00:00
|
|
|
|
2017-10-27 12:39:30 +00:00
|
|
|
return layerize(forward)
|
Update draft of parser neural network model
Model is good, but code is messy. Currently requires Chainer, which may cause the build to fail on machines without a GPU.
Outline of the model:
We first predict context-sensitive vectors for each word in the input:
(embed_lower | embed_prefix | embed_suffix | embed_shape)
>> Maxout(token_width)
>> convolution ** 4
This convolutional layer is shared between the tagger and the parser. This prevents the parser from needing tag features.
To boost the representation, we make a "super tag" with POS, morphology and dependency label. The tagger predicts this
by adding a softmax layer onto the convolutional layer --- so, we're teaching the convolutional layer to give us a
representation that's one affine transform from this informative lexical information. This is obviously good for the
parser (which backprops to the convolutions too).
The parser model makes a state vector by concatenating the vector representations for its context tokens. Current
results suggest few context tokens works well. Maybe this is a bug.
The current context tokens:
* S0, S1, S2: Top three words on the stack
* B0, B1: First two words of the buffer
* S0L1, S0L2: Leftmost and second leftmost children of S0
* S0R1, S0R2: Rightmost and second rightmost children of S0
* S1L1, S1L2, S1R2, S1R, B0L1, B0L2: Likewise for S1 and B0
This makes the state vector quite long: 13*T, where T is the token vector width (128 is working well). Fortunately,
there's a way to structure the computation to save some expense (and make it more GPU friendly).
The parser typically visits 2*N states for a sentence of length N (although it may visit more, if it back-tracks
with a non-monotonic transition). A naive implementation would require 2*N (B, 13*T) @ (13*T, H) matrix multiplications
for a batch of size B. We can instead perform one (B*N, T) @ (T, 13*H) multiplication, to pre-compute the hidden
weights for each positional feature wrt the words in the batch. (Note that our token vectors come from the CNN
-- so we can't play this trick over the vocabulary. That's how Stanford's NN parser works --- and why its model
is so big.)
This pre-computation strategy allows a nice compromise between GPU-friendliness and implementation simplicity.
The CNN and the wide lower layer are computed on the GPU, and then the precomputed hidden weights are moved
to the CPU, before we start the transition-based parsing process. This makes a lot of things much easier.
We don't have to worry about variable-length batch sizes, and we don't have to implement the dynamic oracle
in CUDA to train.
Currently the parser's loss function is multilabel log loss, as the dynamic oracle allows multiple states to
be 0 cost. This is defined as:
(exp(score) / Z) - (exp(score) / gZ)
Where gZ is the sum of the scores assigned to gold classes. I'm very interested in regressing on the cost directly,
but so far this isn't working well.
Machinery is in place for beam-search, which has been working well for the linear model. Beam search should benefit
greatly from the pre-computation trick.
2017-05-12 21:09:15 +00:00
|
|
|
|
|
|
|
|
|
|
|
def doc2feats(cols=None):
|
2017-08-18 19:55:23 +00:00
|
|
|
if cols is None:
|
|
|
|
cols = [ID, NORM, PREFIX, SUFFIX, SHAPE, ORTH]
|
2017-10-27 12:39:30 +00:00
|
|
|
|
2017-05-07 00:02:43 +00:00
|
|
|
def forward(docs, drop=0.):
|
2017-05-18 09:22:20 +00:00
|
|
|
feats = []
|
|
|
|
for doc in docs:
|
2017-05-22 09:47:47 +00:00
|
|
|
feats.append(doc.to_array(cols))
|
2017-05-07 00:02:43 +00:00
|
|
|
return feats, None
|
2017-10-27 12:39:30 +00:00
|
|
|
|
2017-05-06 14:47:15 +00:00
|
|
|
model = layerize(forward)
|
Update draft of parser neural network model
Model is good, but code is messy. Currently requires Chainer, which may cause the build to fail on machines without a GPU.
Outline of the model:
We first predict context-sensitive vectors for each word in the input:
(embed_lower | embed_prefix | embed_suffix | embed_shape)
>> Maxout(token_width)
>> convolution ** 4
This convolutional layer is shared between the tagger and the parser. This prevents the parser from needing tag features.
To boost the representation, we make a "super tag" with POS, morphology and dependency label. The tagger predicts this
by adding a softmax layer onto the convolutional layer --- so, we're teaching the convolutional layer to give us a
representation that's one affine transform from this informative lexical information. This is obviously good for the
parser (which backprops to the convolutions too).
The parser model makes a state vector by concatenating the vector representations for its context tokens. Current
results suggest few context tokens works well. Maybe this is a bug.
The current context tokens:
* S0, S1, S2: Top three words on the stack
* B0, B1: First two words of the buffer
* S0L1, S0L2: Leftmost and second leftmost children of S0
* S0R1, S0R2: Rightmost and second rightmost children of S0
* S1L1, S1L2, S1R2, S1R, B0L1, B0L2: Likewise for S1 and B0
This makes the state vector quite long: 13*T, where T is the token vector width (128 is working well). Fortunately,
there's a way to structure the computation to save some expense (and make it more GPU friendly).
The parser typically visits 2*N states for a sentence of length N (although it may visit more, if it back-tracks
with a non-monotonic transition). A naive implementation would require 2*N (B, 13*T) @ (13*T, H) matrix multiplications
for a batch of size B. We can instead perform one (B*N, T) @ (T, 13*H) multiplication, to pre-compute the hidden
weights for each positional feature wrt the words in the batch. (Note that our token vectors come from the CNN
-- so we can't play this trick over the vocabulary. That's how Stanford's NN parser works --- and why its model
is so big.)
This pre-computation strategy allows a nice compromise between GPU-friendliness and implementation simplicity.
The CNN and the wide lower layer are computed on the GPU, and then the precomputed hidden weights are moved
to the CPU, before we start the transition-based parsing process. This makes a lot of things much easier.
We don't have to worry about variable-length batch sizes, and we don't have to implement the dynamic oracle
in CUDA to train.
Currently the parser's loss function is multilabel log loss, as the dynamic oracle allows multiple states to
be 0 cost. This is defined as:
(exp(score) / Z) - (exp(score) / gZ)
Where gZ is the sum of the scores assigned to gold classes. I'm very interested in regressing on the cost directly,
but so far this isn't working well.
Machinery is in place for beam-search, which has been working well for the linear model. Beam search should benefit
greatly from the pre-computation trick.
2017-05-12 21:09:15 +00:00
|
|
|
model.cols = cols
|
2017-05-06 14:47:15 +00:00
|
|
|
return model
|
|
|
|
|
2017-05-27 20:50:40 +00:00
|
|
|
|
2017-05-06 18:38:12 +00:00
|
|
|
def print_shape(prefix):
|
|
|
|
def forward(X, drop=0.):
|
|
|
|
return X, lambda dX, **kwargs: dX
|
|
|
|
return layerize(forward)
|
2017-05-07 01:57:26 +00:00
|
|
|
|
2017-05-06 18:38:12 +00:00
|
|
|
|
|
|
|
@layerize
|
|
|
|
def get_token_vectors(tokens_attrs_vectors, drop=0.):
|
|
|
|
tokens, attrs, vectors = tokens_attrs_vectors
|
2017-10-27 12:39:30 +00:00
|
|
|
|
2017-05-06 18:38:12 +00:00
|
|
|
def backward(d_output, sgd=None):
|
|
|
|
return (tokens, d_output)
|
2017-09-04 14:26:38 +00:00
|
|
|
|
2017-10-27 12:39:30 +00:00
|
|
|
return vectors, backward
|
2017-07-19 22:17:17 +00:00
|
|
|
|
|
|
|
|
|
|
|
@layerize
|
|
|
|
def logistic(X, drop=0.):
|
|
|
|
xp = get_array_module(X)
|
|
|
|
if not isinstance(X, xp.ndarray):
|
|
|
|
X = xp.asarray(X)
|
|
|
|
# Clip to range (-10, 10)
|
|
|
|
X = xp.minimum(X, 10., X)
|
|
|
|
X = xp.maximum(X, -10., X)
|
|
|
|
Y = 1. / (1. + xp.exp(-X))
|
2017-10-27 12:39:30 +00:00
|
|
|
|
2017-07-19 22:17:17 +00:00
|
|
|
def logistic_bwd(dY, sgd=None):
|
|
|
|
dX = dY * (Y * (1-Y))
|
|
|
|
return dX
|
2017-10-27 12:39:30 +00:00
|
|
|
|
2017-07-19 22:17:17 +00:00
|
|
|
return Y, logistic_bwd
|
|
|
|
|
|
|
|
|
|
|
|
def zero_init(model):
|
|
|
|
def _zero_init_impl(self, X, y):
|
|
|
|
self.W.fill(0)
|
|
|
|
model.on_data_hooks.append(_zero_init_impl)
|
|
|
|
return model
|
|
|
|
|
2017-10-27 12:39:30 +00:00
|
|
|
|
2017-07-19 22:17:17 +00:00
|
|
|
@layerize
|
|
|
|
def preprocess_doc(docs, drop=0.):
|
|
|
|
keys = [doc.to_array([LOWER]) for doc in docs]
|
|
|
|
ops = Model.ops
|
|
|
|
lengths = ops.asarray([arr.shape[0] for arr in keys])
|
|
|
|
keys = ops.xp.concatenate(keys)
|
|
|
|
vals = ops.allocate(keys.shape[0]) + 1
|
|
|
|
return (keys, vals, lengths), None
|
|
|
|
|
2017-10-27 10:16:41 +00:00
|
|
|
|
2017-08-18 19:55:23 +00:00
|
|
|
def getitem(i):
|
|
|
|
def getitem_fwd(X, drop=0.):
|
|
|
|
return X[i], None
|
|
|
|
return layerize(getitem_fwd)
|
|
|
|
|
2017-10-27 10:16:41 +00:00
|
|
|
|
2018-09-24 15:35:28 +00:00
|
|
|
@describe.attributes(
|
|
|
|
W=Synapses("Weights matrix",
|
|
|
|
lambda obj: (obj.nO, obj.nI),
|
|
|
|
lambda W, ops: None)
|
|
|
|
)
|
|
|
|
class MultiSoftmax(Affine):
|
|
|
|
'''Neural network layer that predicts several multi-class attributes at once.
|
|
|
|
For instance, we might predict one class with 6 variables, and another with 5.
|
|
|
|
We predict the 11 neurons required for this, and then softmax them such
|
|
|
|
that columns 0-6 make a probability distribution and coumns 6-11 make another.
|
|
|
|
'''
|
|
|
|
name = 'multisoftmax'
|
|
|
|
|
|
|
|
def __init__(self, out_sizes, nI=None, **kwargs):
|
|
|
|
Model.__init__(self, **kwargs)
|
|
|
|
self.out_sizes = out_sizes
|
|
|
|
self.nO = sum(out_sizes)
|
|
|
|
self.nI = nI
|
|
|
|
|
|
|
|
def predict(self, input__BI):
|
|
|
|
output__BO = self.ops.affine(self.W, self.b, input__BI)
|
|
|
|
i = 0
|
|
|
|
for out_size in self.out_sizes:
|
|
|
|
self.ops.softmax(output__BO[:, i : i+out_size], inplace=True)
|
|
|
|
i += out_size
|
|
|
|
return output__BO
|
|
|
|
|
|
|
|
def begin_update(self, input__BI, drop=0.):
|
|
|
|
output__BO = self.predict(input__BI)
|
|
|
|
def finish_update(grad__BO, sgd=None):
|
|
|
|
self.d_W += self.ops.gemm(grad__BO, input__BI, trans1=True)
|
|
|
|
self.d_b += grad__BO.sum(axis=0)
|
|
|
|
grad__BI = self.ops.gemm(grad__BO, self.W)
|
|
|
|
if sgd is not None:
|
|
|
|
sgd(self._mem.weights, self._mem.gradient, key=self.id)
|
|
|
|
return grad__BI
|
|
|
|
return output__BO, finish_update
|
|
|
|
|
|
|
|
|
2018-09-25 08:57:59 +00:00
|
|
|
def build_tagger_model(nr_class, **cfg):
|
|
|
|
embed_size = util.env_opt('embed_size', 7000)
|
|
|
|
if 'token_vector_width' in cfg:
|
|
|
|
token_vector_width = cfg['token_vector_width']
|
|
|
|
else:
|
|
|
|
token_vector_width = util.env_opt('token_vector_width', 128)
|
|
|
|
pretrained_vectors = cfg.get('pretrained_vectors')
|
|
|
|
subword_features = cfg.get('subword_features', True)
|
|
|
|
with Model.define_operators({'>>': chain, '+': add}):
|
|
|
|
if 'tok2vec' in cfg:
|
|
|
|
tok2vec = cfg['tok2vec']
|
|
|
|
else:
|
|
|
|
tok2vec = Tok2Vec(token_vector_width, embed_size,
|
|
|
|
subword_features=subword_features,
|
|
|
|
pretrained_vectors=pretrained_vectors)
|
|
|
|
softmax = with_flatten(
|
|
|
|
Softmax(nr_class, token_vector_width))
|
|
|
|
model = (
|
|
|
|
tok2vec
|
|
|
|
>> softmax
|
|
|
|
)
|
|
|
|
model.nI = None
|
|
|
|
model.tok2vec = tok2vec
|
|
|
|
model.softmax = softmax
|
|
|
|
return model
|
|
|
|
|
|
|
|
def build_morphologizer_model(class_nums, **cfg):
|
2017-09-28 13:07:41 +00:00
|
|
|
embed_size = util.env_opt('embed_size', 7000)
|
2017-09-21 18:07:26 +00:00
|
|
|
if 'token_vector_width' in cfg:
|
|
|
|
token_vector_width = cfg['token_vector_width']
|
|
|
|
else:
|
2017-09-23 00:58:54 +00:00
|
|
|
token_vector_width = util.env_opt('token_vector_width', 128)
|
2018-03-28 18:12:45 +00:00
|
|
|
pretrained_vectors = cfg.get('pretrained_vectors')
|
2018-08-26 23:50:48 +00:00
|
|
|
subword_features = cfg.get('subword_features', True)
|
2017-08-18 19:55:23 +00:00
|
|
|
with Model.define_operators({'>>': chain, '+': add}):
|
2017-09-26 10:51:52 +00:00
|
|
|
if 'tok2vec' in cfg:
|
|
|
|
tok2vec = cfg['tok2vec']
|
|
|
|
else:
|
|
|
|
tok2vec = Tok2Vec(token_vector_width, embed_size,
|
2018-08-26 23:50:48 +00:00
|
|
|
subword_features=subword_features,
|
2018-03-28 14:02:59 +00:00
|
|
|
pretrained_vectors=pretrained_vectors)
|
2018-09-24 15:35:28 +00:00
|
|
|
softmax = with_flatten(
|
|
|
|
MultiSoftmax(class_nums, token_vector_width))
|
2017-09-23 00:58:54 +00:00
|
|
|
model = (
|
2017-09-21 12:59:48 +00:00
|
|
|
tok2vec
|
2017-11-03 10:19:51 +00:00
|
|
|
>> softmax
|
2017-08-18 19:55:23 +00:00
|
|
|
)
|
|
|
|
model.nI = None
|
2017-09-21 12:59:48 +00:00
|
|
|
model.tok2vec = tok2vec
|
2017-11-03 10:22:01 +00:00
|
|
|
model.softmax = softmax
|
2017-08-18 19:55:23 +00:00
|
|
|
return model
|
|
|
|
|
2017-08-05 23:13:23 +00:00
|
|
|
|
2017-09-01 14:39:55 +00:00
|
|
|
@layerize
|
|
|
|
def SpacyVectors(docs, drop=0.):
|
|
|
|
batch = []
|
|
|
|
for doc in docs:
|
|
|
|
indices = numpy.zeros((len(doc),), dtype='i')
|
|
|
|
for i, word in enumerate(doc):
|
|
|
|
if word.orth in doc.vocab.vectors.key2row:
|
|
|
|
indices[i] = doc.vocab.vectors.key2row[word.orth]
|
|
|
|
else:
|
|
|
|
indices[i] = 0
|
|
|
|
vectors = doc.vocab.vectors.data[indices]
|
|
|
|
batch.append(vectors)
|
|
|
|
return batch, None
|
|
|
|
|
|
|
|
|
2017-07-19 22:17:17 +00:00
|
|
|
def build_text_classifier(nr_class, width=64, **cfg):
|
2018-03-27 17:23:02 +00:00
|
|
|
depth = cfg.get('depth', 2)
|
2017-09-02 12:56:30 +00:00
|
|
|
nr_vector = cfg.get('nr_vector', 5000)
|
2017-10-04 12:55:15 +00:00
|
|
|
pretrained_dims = cfg.get('pretrained_dims', 0)
|
2017-09-01 14:39:55 +00:00
|
|
|
with Model.define_operators({'>>': chain, '+': add, '|': concatenate,
|
|
|
|
'**': clone}):
|
2017-11-01 15:34:09 +00:00
|
|
|
if cfg.get('low_data') and pretrained_dims:
|
2017-09-02 12:56:30 +00:00
|
|
|
model = (
|
|
|
|
SpacyVectors
|
|
|
|
>> flatten_add_lengths
|
2017-10-27 12:39:30 +00:00
|
|
|
>> with_getitem(0, Affine(width, pretrained_dims))
|
2017-09-02 12:56:30 +00:00
|
|
|
>> ParametricAttention(width)
|
|
|
|
>> Pooling(sum_pool)
|
|
|
|
>> Residual(ReLu(width, width)) ** 2
|
|
|
|
>> zero_init(Affine(nr_class, width, drop_factor=0.0))
|
|
|
|
>> logistic
|
|
|
|
)
|
|
|
|
return model
|
2017-07-19 22:17:17 +00:00
|
|
|
|
2017-09-01 14:39:55 +00:00
|
|
|
lower = HashEmbed(width, nr_vector, column=1)
|
|
|
|
prefix = HashEmbed(width//2, nr_vector, column=2)
|
|
|
|
suffix = HashEmbed(width//2, nr_vector, column=3)
|
|
|
|
shape = HashEmbed(width//2, nr_vector, column=4)
|
|
|
|
|
|
|
|
trained_vectors = (
|
|
|
|
FeatureExtracter([ORTH, LOWER, PREFIX, SUFFIX, SHAPE, ID])
|
|
|
|
>> with_flatten(
|
2017-07-25 16:57:59 +00:00
|
|
|
uniqued(
|
2017-09-02 09:41:00 +00:00
|
|
|
(lower | prefix | suffix | shape)
|
2017-09-02 12:56:30 +00:00
|
|
|
>> LN(Maxout(width, width+(width//2)*3)),
|
2017-09-02 09:41:00 +00:00
|
|
|
column=0
|
|
|
|
)
|
2017-07-19 22:17:17 +00:00
|
|
|
)
|
2017-09-01 14:39:55 +00:00
|
|
|
)
|
|
|
|
|
2017-10-04 12:55:15 +00:00
|
|
|
if pretrained_dims:
|
|
|
|
static_vectors = (
|
|
|
|
SpacyVectors
|
|
|
|
>> with_flatten(Affine(width, pretrained_dims))
|
|
|
|
)
|
2017-09-01 14:39:55 +00:00
|
|
|
# TODO Make concatenate support lists
|
2017-10-04 12:55:15 +00:00
|
|
|
vectors = concatenate_lists(trained_vectors, static_vectors)
|
|
|
|
vectors_width = width*2
|
|
|
|
else:
|
|
|
|
vectors = trained_vectors
|
|
|
|
vectors_width = width
|
|
|
|
static_vectors = None
|
|
|
|
cnn_model = (
|
|
|
|
vectors
|
2017-09-02 12:56:30 +00:00
|
|
|
>> with_flatten(
|
2017-10-04 12:55:15 +00:00
|
|
|
LN(Maxout(width, vectors_width))
|
2017-09-02 12:56:30 +00:00
|
|
|
>> Residual(
|
2017-10-04 13:15:53 +00:00
|
|
|
(ExtractWindow(nW=1) >> LN(Maxout(width, width*3)))
|
2018-03-27 17:23:02 +00:00
|
|
|
) ** depth, pad=depth
|
2017-09-02 09:41:00 +00:00
|
|
|
)
|
2017-09-02 12:56:30 +00:00
|
|
|
>> flatten_add_lengths
|
2017-09-01 14:39:55 +00:00
|
|
|
>> ParametricAttention(width)
|
2017-07-25 16:57:59 +00:00
|
|
|
>> Pooling(sum_pool)
|
2017-09-02 12:56:30 +00:00
|
|
|
>> Residual(zero_init(Maxout(width, width)))
|
2017-07-25 16:57:59 +00:00
|
|
|
>> zero_init(Affine(nr_class, width, drop_factor=0.0))
|
2017-07-23 12:10:51 +00:00
|
|
|
)
|
2017-09-01 14:39:55 +00:00
|
|
|
|
2017-07-23 12:10:51 +00:00
|
|
|
linear_model = (
|
|
|
|
_preprocess_doc
|
2017-11-11 23:43:41 +00:00
|
|
|
>> LinearModel(nr_class)
|
2017-07-23 12:10:51 +00:00
|
|
|
)
|
|
|
|
model = (
|
2017-07-25 16:57:59 +00:00
|
|
|
(linear_model | cnn_model)
|
|
|
|
>> zero_init(Affine(nr_class, nr_class*2, drop_factor=0.0))
|
2017-07-19 22:17:17 +00:00
|
|
|
>> logistic
|
|
|
|
)
|
2017-10-04 14:07:30 +00:00
|
|
|
model.nO = nr_class
|
2017-07-19 22:17:17 +00:00
|
|
|
model.lsuv = False
|
|
|
|
return model
|
|
|
|
|
2017-10-27 12:39:30 +00:00
|
|
|
|
2017-09-01 14:39:55 +00:00
|
|
|
@layerize
|
|
|
|
def flatten(seqs, drop=0.):
|
|
|
|
ops = Model.ops
|
|
|
|
lengths = ops.asarray([len(seq) for seq in seqs], dtype='i')
|
2017-10-27 12:39:30 +00:00
|
|
|
|
2017-09-01 14:39:55 +00:00
|
|
|
def finish_update(d_X, sgd=None):
|
|
|
|
return ops.unflatten(d_X, lengths, pad=0)
|
2017-10-27 12:39:30 +00:00
|
|
|
|
2017-09-01 14:39:55 +00:00
|
|
|
X = ops.flatten(seqs, pad=0)
|
|
|
|
return X, finish_update
|
|
|
|
|
|
|
|
|
2017-10-27 12:39:30 +00:00
|
|
|
def concatenate_lists(*layers, **kwargs): # pragma: no cover
|
|
|
|
"""Compose two or more models `f`, `g`, etc, such that their outputs are
|
2017-09-01 14:39:55 +00:00
|
|
|
concatenated, i.e. `concatenate(f, g)(x)` computes `hstack(f(x), g(x))`
|
2017-10-27 12:39:30 +00:00
|
|
|
"""
|
2017-09-01 14:39:55 +00:00
|
|
|
if not layers:
|
|
|
|
return noop()
|
|
|
|
drop_factor = kwargs.get('drop_factor', 1.0)
|
|
|
|
ops = layers[0].ops
|
|
|
|
layers = [chain(layer, flatten) for layer in layers]
|
|
|
|
concat = concatenate(*layers)
|
2017-10-27 12:39:30 +00:00
|
|
|
|
2017-09-01 14:39:55 +00:00
|
|
|
def concatenate_lists_fwd(Xs, drop=0.):
|
|
|
|
drop *= drop_factor
|
|
|
|
lengths = ops.asarray([len(X) for X in Xs], dtype='i')
|
|
|
|
flat_y, bp_flat_y = concat.begin_update(Xs, drop=drop)
|
|
|
|
ys = ops.unflatten(flat_y, lengths)
|
2017-10-27 12:39:30 +00:00
|
|
|
|
2017-09-01 14:39:55 +00:00
|
|
|
def concatenate_lists_bwd(d_ys, sgd=None):
|
|
|
|
return bp_flat_y(ops.flatten(d_ys), sgd=sgd)
|
2017-10-27 12:39:30 +00:00
|
|
|
|
2017-09-01 14:39:55 +00:00
|
|
|
return ys, concatenate_lists_bwd
|
2017-10-27 12:39:30 +00:00
|
|
|
|
2017-09-01 14:39:55 +00:00
|
|
|
model = wrap(concatenate_lists_fwd, concat)
|
|
|
|
return model
|