mirror of https://github.com/explosion/spaCy.git
* Add more options to bin/parser/train
This commit is contained in:
parent
1fee7ade61
commit
1736fc5a67
|
@ -141,8 +141,11 @@ def train(Language, gold_tuples, model_dir, n_iter=15, feat_set=u'basic',
|
||||||
nlp.vocab.strings.dump(path.join(model_dir, 'vocab', 'strings.txt'))
|
nlp.vocab.strings.dump(path.join(model_dir, 'vocab', 'strings.txt'))
|
||||||
|
|
||||||
|
|
||||||
def evaluate(Language, gold_tuples, model_dir, gold_preproc=False, verbose=False):
|
def evaluate(Language, gold_tuples, model_dir, gold_preproc=False, verbose=False,
|
||||||
|
beam_width=None):
|
||||||
nlp = Language(data_dir=model_dir)
|
nlp = Language(data_dir=model_dir)
|
||||||
|
if beam_width is not None:
|
||||||
|
nlp.parser.cfg.beam_width = beam_width
|
||||||
scorer = Scorer()
|
scorer = Scorer()
|
||||||
for raw_text, sents in gold_tuples:
|
for raw_text, sents in gold_tuples:
|
||||||
if gold_preproc:
|
if gold_preproc:
|
||||||
|
@ -179,9 +182,10 @@ def write_parses(Language, dev_loc, model_dir, out_loc):
|
||||||
@plac.annotations(
|
@plac.annotations(
|
||||||
train_loc=("Location of training file or directory"),
|
train_loc=("Location of training file or directory"),
|
||||||
dev_loc=("Location of development file or directory"),
|
dev_loc=("Location of development file or directory"),
|
||||||
|
model_dir=("Location of output model directory",),
|
||||||
|
eval_only=("Skip training, and only evaluate", "flag", "e", bool),
|
||||||
corruption_level=("Amount of noise to add to training data", "option", "c", float),
|
corruption_level=("Amount of noise to add to training data", "option", "c", float),
|
||||||
gold_preproc=("Use gold-standard sentence boundaries in training?", "flag", "g", bool),
|
gold_preproc=("Use gold-standard sentence boundaries in training?", "flag", "g", bool),
|
||||||
model_dir=("Location of output model directory",),
|
|
||||||
out_loc=("Out location", "option", "o", str),
|
out_loc=("Out location", "option", "o", str),
|
||||||
n_sents=("Number of training sentences", "option", "n", int),
|
n_sents=("Number of training sentences", "option", "n", int),
|
||||||
n_iter=("Number of training iterations", "option", "i", int),
|
n_iter=("Number of training iterations", "option", "i", int),
|
||||||
|
@ -190,17 +194,20 @@ def write_parses(Language, dev_loc, model_dir, out_loc):
|
||||||
debug=("Debug mode", "flag", "d", bool)
|
debug=("Debug mode", "flag", "d", bool)
|
||||||
)
|
)
|
||||||
def main(train_loc, dev_loc, model_dir, n_sents=0, n_iter=15, out_loc="", verbose=False,
|
def main(train_loc, dev_loc, model_dir, n_sents=0, n_iter=15, out_loc="", verbose=False,
|
||||||
debug=False, corruption_level=0.0, gold_preproc=False, beam_width=1):
|
debug=False, corruption_level=0.0, gold_preproc=False, beam_width=1,
|
||||||
gold_train = list(read_json_file(train_loc))
|
eval_only=False):
|
||||||
train(English, gold_train, model_dir,
|
if not eval_only:
|
||||||
feat_set='basic' if not debug else 'debug',
|
gold_train = list(read_json_file(train_loc))
|
||||||
gold_preproc=gold_preproc, n_sents=n_sents,
|
train(English, gold_train, model_dir,
|
||||||
corruption_level=corruption_level, n_iter=n_iter,
|
feat_set='basic' if not debug else 'debug',
|
||||||
beam_width=beam_width)
|
gold_preproc=gold_preproc, n_sents=n_sents,
|
||||||
|
corruption_level=corruption_level, n_iter=n_iter,
|
||||||
|
beam_width=beam_width)
|
||||||
if out_loc:
|
if out_loc:
|
||||||
write_parses(English, dev_loc, model_dir, out_loc)
|
write_parses(English, dev_loc, model_dir, out_loc)
|
||||||
scorer = evaluate(English, list(read_json_file(dev_loc)),
|
scorer = evaluate(English, list(read_json_file(dev_loc)),
|
||||||
model_dir, gold_preproc=gold_preproc, verbose=verbose)
|
model_dir, gold_preproc=gold_preproc, verbose=verbose,
|
||||||
|
beam_width=beam_width)
|
||||||
print 'TOK', 100-scorer.token_acc
|
print 'TOK', 100-scorer.token_acc
|
||||||
print 'POS', scorer.tags_acc
|
print 'POS', scorer.tags_acc
|
||||||
print 'UAS', scorer.uas
|
print 'UAS', scorer.uas
|
||||||
|
|
Loading…
Reference in New Issue