2017-05-06 14:47:15 +00:00
|
|
|
from __future__ import unicode_literals, print_function
|
2015-10-08 01:00:11 +00:00
|
|
|
import plac
|
|
|
|
import json
|
|
|
|
import random
|
2016-11-25 23:45:45 +00:00
|
|
|
import pathlib
|
2015-10-08 01:00:11 +00:00
|
|
|
|
2016-11-25 17:19:33 +00:00
|
|
|
from spacy.tokens import Doc
|
|
|
|
from spacy.syntax.nonproj import PseudoProjectivity
|
|
|
|
from spacy.language import Language
|
2015-10-08 01:00:11 +00:00
|
|
|
from spacy.gold import GoldParse
|
|
|
|
from spacy.tagger import Tagger
|
2017-05-06 14:47:15 +00:00
|
|
|
from spacy.pipeline import DependencyParser, TokenVectorEncoder
|
2015-10-08 01:00:11 +00:00
|
|
|
from spacy.syntax.parser import get_templates
|
2016-11-25 17:19:33 +00:00
|
|
|
from spacy.syntax.arc_eager import ArcEager
|
2015-10-08 01:00:11 +00:00
|
|
|
from spacy.scorer import Scorer
|
2017-01-09 15:53:46 +00:00
|
|
|
from spacy.language_data.tag_map import TAG_MAP as DEFAULT_TAG_MAP
|
2016-05-23 10:53:00 +00:00
|
|
|
import spacy.attrs
|
2016-11-25 23:45:45 +00:00
|
|
|
import io
|
2017-05-07 12:31:09 +00:00
|
|
|
from thinc.neural.ops import CupyOps
|
|
|
|
from thinc.neural import Model
|
|
|
|
|
|
|
|
try:
|
|
|
|
import cupy
|
|
|
|
print("Using GPU")
|
|
|
|
Model.ops = CupyOps()
|
|
|
|
except ImportError:
|
|
|
|
pass
|
2015-10-08 01:00:11 +00:00
|
|
|
|
2017-05-07 16:04:24 +00:00
|
|
|
from thinc.neural import Model
|
|
|
|
|
|
|
|
|
|
|
|
try:
|
|
|
|
import cupy
|
|
|
|
from thinc.neural.ops import CupyOps
|
|
|
|
except:
|
|
|
|
cupy = None
|
|
|
|
|
2015-10-08 01:00:11 +00:00
|
|
|
|
2017-03-11 17:11:05 +00:00
|
|
|
def read_conllx(loc, n=0):
|
2016-11-25 23:45:45 +00:00
|
|
|
with io.open(loc, 'r', encoding='utf8') as file_:
|
2015-10-08 01:00:11 +00:00
|
|
|
text = file_.read()
|
2017-03-11 17:11:05 +00:00
|
|
|
i = 0
|
2015-10-08 01:00:11 +00:00
|
|
|
for sent in text.strip().split('\n\n'):
|
|
|
|
lines = sent.strip().split('\n')
|
|
|
|
if lines:
|
2016-05-23 10:53:00 +00:00
|
|
|
while lines[0].startswith('#'):
|
2015-10-08 01:00:11 +00:00
|
|
|
lines.pop(0)
|
|
|
|
tokens = []
|
|
|
|
for line in lines:
|
2017-03-21 22:00:13 +00:00
|
|
|
id_, word, lemma, pos, tag, morph, head, dep, _1, \
|
|
|
|
_2 = line.split('\t')
|
2017-03-16 22:08:15 +00:00
|
|
|
if '-' in id_ or '.' in id_:
|
2015-10-08 01:00:11 +00:00
|
|
|
continue
|
2016-11-25 17:19:33 +00:00
|
|
|
try:
|
|
|
|
id_ = int(id_) - 1
|
|
|
|
head = (int(head) - 1) if head != '0' else id_
|
2017-05-07 01:57:26 +00:00
|
|
|
dep = 'ROOT' if dep == 'root' else dep #'unlabelled'
|
2017-05-06 14:47:15 +00:00
|
|
|
tokens.append((id_, word, pos+'__'+morph, head, dep, 'O'))
|
2016-11-25 17:19:33 +00:00
|
|
|
except:
|
|
|
|
raise
|
|
|
|
tuples = [list(t) for t in zip(*tokens)]
|
|
|
|
yield (None, [[tuples, []]])
|
2017-03-11 17:11:05 +00:00
|
|
|
i += 1
|
|
|
|
if n >= 1 and i >= n:
|
|
|
|
break
|
2016-11-25 17:19:33 +00:00
|
|
|
|
|
|
|
|
2017-05-06 14:47:15 +00:00
|
|
|
def score_model(vocab, encoder, tagger, parser, Xs, ys, verbose=False):
|
2015-10-08 01:00:11 +00:00
|
|
|
scorer = Scorer()
|
2017-05-06 14:47:15 +00:00
|
|
|
correct = 0.
|
|
|
|
total = 0.
|
|
|
|
for doc, gold in zip(Xs, ys):
|
|
|
|
doc = Doc(vocab, words=[w.text for w in doc])
|
|
|
|
encoder(doc)
|
|
|
|
tagger(doc)
|
|
|
|
parser(doc)
|
|
|
|
PseudoProjectivity.deprojectivize(doc)
|
|
|
|
scorer.score(doc, gold, verbose=verbose)
|
|
|
|
for token, tag in zip(doc, gold.tags):
|
|
|
|
univ_guess, _ = token.tag_.split('_', 1)
|
|
|
|
univ_truth, _ = tag.split('_', 1)
|
|
|
|
correct += univ_guess == univ_truth
|
|
|
|
total += 1
|
|
|
|
return scorer
|
|
|
|
|
|
|
|
|
|
|
|
def organize_data(vocab, train_sents):
|
|
|
|
Xs = []
|
|
|
|
ys = []
|
|
|
|
for _, doc_sents in train_sents:
|
|
|
|
for (ids, words, tags, heads, deps, ner), _ in doc_sents:
|
2016-11-25 17:19:33 +00:00
|
|
|
doc = Doc(vocab, words=words)
|
|
|
|
gold = GoldParse(doc, tags=tags, heads=heads, deps=deps)
|
2017-05-06 14:47:15 +00:00
|
|
|
Xs.append(doc)
|
|
|
|
ys.append(gold)
|
|
|
|
return Xs, ys
|
2015-10-08 01:00:11 +00:00
|
|
|
|
|
|
|
|
2017-03-16 22:08:15 +00:00
|
|
|
def main(lang_name, train_loc, dev_loc, model_dir, clusters_loc=None):
|
2017-05-07 16:04:24 +00:00
|
|
|
if cupy is not None:
|
|
|
|
Model.ops = CupyOps()
|
2017-03-16 22:08:15 +00:00
|
|
|
LangClass = spacy.util.get_lang_class(lang_name)
|
2015-10-08 01:00:11 +00:00
|
|
|
train_sents = list(read_conllx(train_loc))
|
2017-05-07 01:57:26 +00:00
|
|
|
dev_sents = list(read_conllx(dev_loc))
|
2016-11-25 17:19:33 +00:00
|
|
|
train_sents = PseudoProjectivity.preprocess_training_data(train_sents)
|
2016-11-25 23:45:45 +00:00
|
|
|
|
2016-11-25 17:19:33 +00:00
|
|
|
actions = ArcEager.get_actions(gold_parses=train_sents)
|
|
|
|
features = get_templates('basic')
|
2017-03-11 17:11:05 +00:00
|
|
|
|
2016-11-25 23:45:45 +00:00
|
|
|
model_dir = pathlib.Path(model_dir)
|
2017-03-16 22:08:15 +00:00
|
|
|
if not model_dir.exists():
|
|
|
|
model_dir.mkdir()
|
2017-01-08 23:55:44 +00:00
|
|
|
if not (model_dir / 'deps').exists():
|
|
|
|
(model_dir / 'deps').mkdir()
|
2017-03-16 22:08:15 +00:00
|
|
|
if not (model_dir / 'pos').exists():
|
|
|
|
(model_dir / 'pos').mkdir()
|
2017-01-09 15:53:46 +00:00
|
|
|
with (model_dir / 'deps' / 'config.json').open('wb') as file_:
|
|
|
|
file_.write(
|
|
|
|
json.dumps(
|
|
|
|
{'pseudoprojective': True, 'labels': actions, 'features': features}).encode('utf8'))
|
2017-03-16 22:08:15 +00:00
|
|
|
|
|
|
|
vocab = LangClass.Defaults.create_vocab()
|
|
|
|
if not (model_dir / 'vocab').exists():
|
|
|
|
(model_dir / 'vocab').mkdir()
|
|
|
|
else:
|
|
|
|
if (model_dir / 'vocab' / 'strings.json').exists():
|
|
|
|
with (model_dir / 'vocab' / 'strings.json').open() as file_:
|
|
|
|
vocab.strings.load(file_)
|
|
|
|
if (model_dir / 'vocab' / 'lexemes.bin').exists():
|
|
|
|
vocab.load_lexemes(model_dir / 'vocab' / 'lexemes.bin')
|
|
|
|
|
|
|
|
if clusters_loc is not None:
|
|
|
|
clusters_loc = pathlib.Path(clusters_loc)
|
|
|
|
with clusters_loc.open() as file_:
|
|
|
|
for line in file_:
|
|
|
|
try:
|
|
|
|
cluster, word, freq = line.split()
|
|
|
|
except ValueError:
|
|
|
|
continue
|
|
|
|
lex = vocab[word]
|
|
|
|
lex.cluster = int(cluster[::-1], 2)
|
2016-11-25 17:19:33 +00:00
|
|
|
# Populate vocab
|
|
|
|
for _, doc_sents in train_sents:
|
|
|
|
for (ids, words, tags, heads, deps, ner), _ in doc_sents:
|
|
|
|
for word in words:
|
|
|
|
_ = vocab[word]
|
2016-11-25 23:45:45 +00:00
|
|
|
for dep in deps:
|
|
|
|
_ = vocab[dep]
|
|
|
|
for tag in tags:
|
|
|
|
_ = vocab[tag]
|
2017-03-16 22:08:15 +00:00
|
|
|
if vocab.morphology.tag_map:
|
2017-01-09 15:53:46 +00:00
|
|
|
for tag in tags:
|
2017-03-16 22:08:15 +00:00
|
|
|
assert tag in vocab.morphology.tag_map, repr(tag)
|
|
|
|
tagger = Tagger(vocab)
|
2017-05-06 14:47:15 +00:00
|
|
|
encoder = TokenVectorEncoder(vocab)
|
2017-03-11 17:11:05 +00:00
|
|
|
parser = DependencyParser(vocab, actions=actions, features=features, L1=0.0)
|
|
|
|
|
2017-05-06 14:47:15 +00:00
|
|
|
Xs, ys = organize_data(vocab, train_sents)
|
2017-05-07 01:57:26 +00:00
|
|
|
dev_Xs, dev_ys = organize_data(vocab, dev_sents)
|
2017-05-07 12:31:09 +00:00
|
|
|
Xs = Xs
|
|
|
|
ys = ys
|
|
|
|
dev_Xs = dev_Xs[:1000]
|
|
|
|
dev_ys = dev_ys[:1000]
|
2017-05-06 14:47:15 +00:00
|
|
|
with encoder.model.begin_training(Xs[:100], ys[:100]) as (trainer, optimizer):
|
|
|
|
docs = list(Xs)
|
|
|
|
for doc in docs:
|
|
|
|
encoder(doc)
|
|
|
|
parser.begin_training(docs, ys)
|
|
|
|
nn_loss = [0.]
|
|
|
|
def track_progress():
|
2017-05-07 01:57:26 +00:00
|
|
|
with encoder.tagger.use_params(optimizer.averages):
|
|
|
|
scorer = score_model(vocab, encoder, tagger, parser, dev_Xs, dev_ys)
|
2017-05-06 14:47:15 +00:00
|
|
|
itn = len(nn_loss)
|
|
|
|
print('%d:\t%.3f\t%.3f\t%.3f' % (itn, nn_loss[-1], scorer.uas, scorer.tags_acc))
|
|
|
|
nn_loss.append(0.)
|
|
|
|
trainer.each_epoch.append(track_progress)
|
2017-05-07 16:04:24 +00:00
|
|
|
trainer.batch_size = 24
|
2017-05-07 12:31:09 +00:00
|
|
|
trainer.nb_epoch = 10
|
|
|
|
for docs, golds in trainer.iterate(Xs, ys):
|
2017-05-06 14:47:15 +00:00
|
|
|
docs = [Doc(vocab, words=[w.text for w in doc]) for doc in docs]
|
|
|
|
tokvecs, upd_tokvecs = encoder.begin_update(docs)
|
|
|
|
for doc, tokvec in zip(docs, tokvecs):
|
|
|
|
doc.tensor = tokvec
|
|
|
|
for doc, gold in zip(docs, golds):
|
2016-11-25 17:19:33 +00:00
|
|
|
tagger.update(doc, gold)
|
2017-05-06 14:47:15 +00:00
|
|
|
d_tokvecs, loss = parser.update(docs, golds, sgd=optimizer)
|
|
|
|
upd_tokvecs(d_tokvecs, sgd=optimizer)
|
2017-05-07 12:31:09 +00:00
|
|
|
encoder.update(docs, golds, sgd=optimizer)
|
2017-05-06 14:47:15 +00:00
|
|
|
nn_loss[-1] += loss
|
2017-03-21 22:18:54 +00:00
|
|
|
nlp = LangClass(vocab=vocab, tagger=tagger, parser=parser)
|
2015-10-08 01:00:11 +00:00
|
|
|
nlp.end_training(model_dir)
|
2016-11-25 17:19:33 +00:00
|
|
|
scorer = score_model(vocab, tagger, parser, read_conllx(dev_loc))
|
2015-10-08 01:00:11 +00:00
|
|
|
print('%d:\t%.3f\t%.3f\t%.3f' % (itn, scorer.uas, scorer.las, scorer.tags_acc))
|
2017-03-11 17:11:05 +00:00
|
|
|
|
2015-10-08 01:00:11 +00:00
|
|
|
|
|
|
|
if __name__ == '__main__':
|
|
|
|
plac.call(main)
|