spaCy/examples/training/pretrain_textcat.py

217 lines
7.3 KiB
Python
Raw Normal View History

2018-12-02 03:26:26 +00:00
"""This script is experimental.
2018-11-03 17:44:12 +00:00
Try pre-training the CNN component of the text categorizer using a cheap
2019-10-02 08:37:39 +00:00
language modelling-like objective. Specifically, we load pretrained vectors
2018-11-03 17:44:12 +00:00
(from something like word2vec, GloVe, FastText etc), and use the CNN to
2019-10-02 08:37:39 +00:00
predict the tokens' pretrained vectors. This isn't as easy as it sounds:
2018-11-03 17:44:12 +00:00
we're not merely doing compression here, because heavy dropout is applied,
including over the input words. This means the model must often (50% of the time)
use the context in order to predict the word.
To evaluate the technique, we're pre-training with the 50k texts from the IMDB
corpus, and then training with only 100 labels. Note that it's a bit dirty to
pre-train with the development data, but also not *so* terrible: we're not using
the development labels, after all --- only the unlabelled text.
2018-12-02 03:26:26 +00:00
"""
2018-11-02 22:52:12 +00:00
import plac
import tqdm
2018-11-03 12:53:25 +00:00
import random
2018-11-02 22:52:12 +00:00
import spacy
import thinc.extra.datasets
2018-11-03 12:53:25 +00:00
from spacy.util import minibatch, use_gpu, compounding
from spacy._ml import Tok2Vec
from spacy.pipeline import TextCategorizer
import numpy
2018-11-02 22:52:12 +00:00
2018-11-03 12:53:25 +00:00
def load_texts(limit=0):
2018-11-02 22:52:12 +00:00
train, dev = thinc.extra.datasets.imdb()
2018-11-03 12:53:25 +00:00
train_texts, train_labels = zip(*train)
2018-11-03 17:44:12 +00:00
dev_texts, dev_labels = zip(*train)
train_texts = list(train_texts)
dev_texts = list(dev_texts)
random.shuffle(train_texts)
random.shuffle(dev_texts)
2018-11-03 12:53:25 +00:00
if limit >= 1:
return train_texts[:limit]
else:
2018-11-03 17:44:12 +00:00
return list(train_texts) + list(dev_texts)
2018-11-03 12:53:25 +00:00
2018-11-02 22:52:12 +00:00
2018-11-03 17:44:12 +00:00
def load_textcat_data(limit=0):
2018-11-03 12:53:25 +00:00
"""Load data from the IMDB dataset."""
# Partition off part of the train data for evaluation
2018-11-03 17:44:12 +00:00
train_data, eval_data = thinc.extra.datasets.imdb()
2018-11-03 12:53:25 +00:00
random.shuffle(train_data)
train_data = train_data[-limit:]
texts, labels = zip(*train_data)
2018-11-03 17:44:12 +00:00
eval_texts, eval_labels = zip(*eval_data)
2018-12-02 03:26:26 +00:00
cats = [{"POSITIVE": bool(y), "NEGATIVE": not bool(y)} for y in labels]
eval_cats = [{"POSITIVE": bool(y), "NEGATIVE": not bool(y)} for y in eval_labels]
2018-11-03 17:44:12 +00:00
return (texts, cats), (eval_texts, eval_cats)
2018-11-02 22:52:12 +00:00
2018-11-03 10:54:20 +00:00
def prefer_gpu():
used = spacy.util.use_gpu(0)
if used is None:
return False
else:
import cupy.random
2018-12-02 03:26:26 +00:00
cupy.random.seed(0)
2018-11-03 10:54:20 +00:00
return True
2018-11-03 12:53:25 +00:00
def build_textcat_model(tok2vec, nr_class, width):
2018-11-04 00:17:09 +00:00
from thinc.v2v import Model, Softmax, Maxout
2018-11-03 12:53:25 +00:00
from thinc.api import flatten_add_lengths, chain
2018-11-04 00:17:09 +00:00
from thinc.t2v import Pooling, sum_pool, mean_pool, max_pool
2018-11-03 12:53:25 +00:00
from thinc.misc import Residual, LayerNorm
from spacy._ml import logistic, zero_init
2018-12-02 03:26:26 +00:00
with Model.define_operators({">>": chain}):
2018-11-03 12:53:25 +00:00
model = (
2018-11-03 17:44:12 +00:00
tok2vec
2018-11-03 12:53:25 +00:00
>> flatten_add_lengths
2018-11-04 00:17:09 +00:00
>> Pooling(mean_pool)
>> Softmax(nr_class, width)
2018-11-03 12:53:25 +00:00
)
model.tok2vec = tok2vec
return model
2018-12-02 03:26:26 +00:00
2018-11-03 12:53:25 +00:00
def block_gradients(model):
from thinc.api import wrap
2018-12-02 03:26:26 +00:00
def forward(X, drop=0.0):
2018-11-03 12:53:25 +00:00
Y, _ = model.begin_update(X, drop=drop)
return Y, None
2018-12-02 03:26:26 +00:00
2018-11-03 12:53:25 +00:00
return wrap(forward, model)
2018-12-02 03:26:26 +00:00
2018-11-03 12:53:25 +00:00
def create_pipeline(width, embed_size, vectors_model):
2018-11-02 22:52:12 +00:00
print("Load vectors")
2018-11-03 10:54:20 +00:00
nlp = spacy.load(vectors_model)
2018-11-02 22:52:12 +00:00
print("Start training")
2018-12-02 03:26:26 +00:00
textcat = TextCategorizer(
nlp.vocab,
labels=["POSITIVE", "NEGATIVE"],
2018-11-03 12:53:25 +00:00
model=build_textcat_model(
2018-12-02 03:26:26 +00:00
Tok2Vec(width=width, embed_size=embed_size), 2, width
),
)
2018-11-03 12:53:25 +00:00
nlp.add_pipe(textcat)
return nlp
2018-12-02 03:26:26 +00:00
2018-11-03 12:53:25 +00:00
def train_tensorizer(nlp, texts, dropout, n_iter):
2018-12-02 03:26:26 +00:00
tensorizer = nlp.create_pipe("tensorizer")
2018-11-02 22:52:12 +00:00
nlp.add_pipe(tensorizer)
optimizer = nlp.begin_training()
2018-11-03 12:53:25 +00:00
for i in range(n_iter):
2018-11-02 22:52:12 +00:00
losses = {}
2018-11-03 12:53:25 +00:00
for i, batch in enumerate(minibatch(tqdm.tqdm(texts))):
2018-11-02 22:52:12 +00:00
docs = [nlp.make_doc(text) for text in batch]
2018-11-03 12:53:25 +00:00
tensorizer.update(docs, None, losses=losses, sgd=optimizer, drop=dropout)
2018-11-03 10:54:20 +00:00
print(losses)
2018-11-03 12:53:25 +00:00
return optimizer
2018-12-02 03:26:26 +00:00
2018-11-03 17:44:12 +00:00
def train_textcat(nlp, n_texts, n_iter=10):
2018-12-02 03:26:26 +00:00
textcat = nlp.get_pipe("textcat")
2018-11-03 17:44:12 +00:00
tok2vec_weights = textcat.model.tok2vec.to_bytes()
2018-11-03 12:53:25 +00:00
(train_texts, train_cats), (dev_texts, dev_cats) = load_textcat_data(limit=n_texts)
2018-12-02 03:26:26 +00:00
print(
"Using {} examples ({} training, {} evaluation)".format(
n_texts, len(train_texts), len(dev_texts)
)
)
train_data = list(zip(train_texts, [{"cats": cats} for cats in train_cats]))
2018-11-03 12:53:25 +00:00
# get names of other pipes to disable them during training
2018-12-02 03:26:26 +00:00
other_pipes = [pipe for pipe in nlp.pipe_names if pipe != "textcat"]
2018-11-03 12:53:25 +00:00
with nlp.disable_pipes(*other_pipes): # only train textcat
2018-11-03 17:44:12 +00:00
optimizer = nlp.begin_training()
textcat.model.tok2vec.from_bytes(tok2vec_weights)
2018-11-03 12:53:25 +00:00
print("Training the model...")
2018-12-02 03:26:26 +00:00
print("{:^5}\t{:^5}\t{:^5}\t{:^5}".format("LOSS", "P", "R", "F"))
2018-11-03 12:53:25 +00:00
for i in range(n_iter):
2018-12-02 03:26:26 +00:00
losses = {"textcat": 0.0}
2018-11-03 12:53:25 +00:00
# batch up the examples using spaCy's minibatch
batches = minibatch(tqdm.tqdm(train_data), size=2)
for batch in batches:
texts, annotations = zip(*batch)
2018-12-02 03:26:26 +00:00
nlp.update(texts, annotations, sgd=optimizer, drop=0.2, losses=losses)
2018-11-03 12:53:25 +00:00
with textcat.model.use_params(optimizer.averages):
# evaluate on the dev data split off in load_data()
scores = evaluate_textcat(nlp.tokenizer, textcat, dev_texts, dev_cats)
2018-12-02 03:26:26 +00:00
print(
"{0:.3f}\t{1:.3f}\t{2:.3f}\t{3:.3f}".format( # print a simple table
losses["textcat"],
scores["textcat_p"],
scores["textcat_r"],
scores["textcat_f"],
)
)
2018-11-03 12:53:25 +00:00
def evaluate_textcat(tokenizer, textcat, texts, cats):
docs = (tokenizer(text) for text in texts)
2018-11-03 17:44:12 +00:00
tp = 1e-8
fp = 1e-8
tn = 1e-8
fn = 1e-8
2018-11-03 12:53:25 +00:00
for i, doc in enumerate(textcat.pipe(docs)):
gold = cats[i]
for label, score in doc.cats.items():
if label not in gold:
continue
if score >= 0.5 and gold[label] >= 0.5:
2018-12-02 03:26:26 +00:00
tp += 1.0
2018-11-03 12:53:25 +00:00
elif score >= 0.5 and gold[label] < 0.5:
2018-12-02 03:26:26 +00:00
fp += 1.0
2018-11-03 12:53:25 +00:00
elif score < 0.5 and gold[label] < 0.5:
tn += 1
elif score < 0.5 and gold[label] >= 0.5:
fn += 1
precision = tp / (tp + fp)
recall = tp / (tp + fn)
f_score = 2 * (precision * recall) / (precision + recall)
2018-12-02 03:26:26 +00:00
return {"textcat_p": precision, "textcat_r": recall, "textcat_f": f_score}
2018-11-03 12:53:25 +00:00
@plac.annotations(
width=("Width of CNN layers", "positional", None, int),
embed_size=("Embedding rows", "positional", None, int),
pretrain_iters=("Number of iterations to pretrain", "option", "pn", int),
train_iters=("Number of iterations to pretrain", "option", "tn", int),
train_examples=("Number of labelled examples", "option", "eg", int),
2018-12-02 03:26:26 +00:00
vectors_model=("Name or path to vectors model to learn from"),
2018-11-03 12:53:25 +00:00
)
2018-12-02 03:26:26 +00:00
def main(
width,
embed_size,
vectors_model,
pretrain_iters=30,
train_iters=30,
train_examples=1000,
):
2018-11-03 12:53:25 +00:00
random.seed(0)
numpy.random.seed(0)
use_gpu = prefer_gpu()
print("Using GPU?", use_gpu)
nlp = create_pipeline(width, embed_size, vectors_model)
print("Load data")
texts = load_texts(limit=0)
print("Train tensorizer")
2018-11-03 17:44:12 +00:00
optimizer = train_tensorizer(nlp, texts, dropout=0.2, n_iter=pretrain_iters)
2018-11-03 12:53:25 +00:00
print("Train textcat")
2018-11-03 17:44:12 +00:00
train_textcat(nlp, train_examples, n_iter=train_iters)
2018-11-02 22:52:12 +00:00
2018-12-02 03:26:26 +00:00
if __name__ == "__main__":
2018-11-02 22:52:12 +00:00
plac.call(main)