2018-12-10 08:46:53 +00:00
|
|
|
import os
|
2019-12-16 12:12:19 +00:00
|
|
|
import tqdm
|
2017-05-17 10:04:50 +00:00
|
|
|
from pathlib import Path
|
2020-02-18 14:38:18 +00:00
|
|
|
from thinc.api import use_ops
|
2017-05-23 08:06:53 +00:00
|
|
|
from timeit import default_timer as timer
|
2018-06-25 11:40:17 +00:00
|
|
|
import shutil
|
💫 Replace ujson, msgpack and dill/pickle/cloudpickle with srsly (#3003)
Remove hacks and wrappers, keep code in sync across our libraries and move spaCy a few steps closer to only depending on packages with binary wheels 🎉
See here: https://github.com/explosion/srsly
Serialization is hard, especially across Python versions and multiple platforms. After dealing with many subtle bugs over the years (encodings, locales, large files) our libraries like spaCy and Prodigy have steadily grown a number of utility functions to wrap the multiple serialization formats we need to support (especially json, msgpack and pickle). These wrapping functions ended up duplicated across our codebases, so we wanted to put them in one place.
At the same time, we noticed that having a lot of small dependencies was making maintainence harder, and making installation slower. To solve this, we've made srsly standalone, by including the component packages directly within it. This way we can provide all the serialization utilities we need in a single binary wheel.
srsly currently includes forks of the following packages:
ujson
msgpack
msgpack-numpy
cloudpickle
* WIP: replace json/ujson with srsly
* Replace ujson in examples
Use regular json instead of srsly to make code easier to read and follow
* Update requirements
* Fix imports
* Fix typos
* Replace msgpack with srsly
* Fix warning
2018-12-03 00:28:22 +00:00
|
|
|
import srsly
|
Generalize handling of tokenizer special cases (#4259)
* Generalize handling of tokenizer special cases
Handle tokenizer special cases more generally by using the Matcher
internally to match special cases after the affix/token_match
tokenization is complete.
Instead of only matching special cases while processing balanced or
nearly balanced prefixes and suffixes, this recognizes special cases in
a wider range of contexts:
* Allows arbitrary numbers of prefixes/affixes around special cases
* Allows special cases separated by infixes
Existing tests/settings that couldn't be preserved as before:
* The emoticon '")' is no longer a supported special case
* The emoticon ':)' in "example:)" is a false positive again
When merged with #4258 (or the relevant cache bugfix), the affix and
token_match properties should be modified to flush and reload all
special cases to use the updated internal tokenization with the Matcher.
* Remove accidentally added test case
* Really remove accidentally added test
* Reload special cases when necessary
Reload special cases when affixes or token_match are modified. Skip
reloading during initialization.
* Update error code number
* Fix offset and whitespace in Matcher special cases
* Fix offset bugs when merging and splitting tokens
* Set final whitespace on final token in inserted special case
* Improve cache flushing in tokenizer
* Separate cache and specials memory (temporarily)
* Flush cache when adding special cases
* Repeated `self._cache = PreshMap()` and `self._specials = PreshMap()`
are necessary due to this bug:
https://github.com/explosion/preshed/issues/21
* Remove reinitialized PreshMaps on cache flush
* Update UD bin scripts
* Update imports for `bin/`
* Add all currently supported languages
* Update subtok merger for new Matcher validation
* Modify blinded check to look at tokens instead of lemmas (for corpora
with tokens but not lemmas like Telugu)
* Use special Matcher only for cases with affixes
* Reinsert specials cache checks during normal tokenization for special
cases as much as possible
* Additionally include specials cache checks while splitting on infixes
* Since the special Matcher needs consistent affix-only tokenization
for the special cases themselves, introduce the argument
`with_special_cases` in order to do tokenization with or without
specials cache checks
* After normal tokenization, postprocess with special cases Matcher for
special cases containing affixes
* Replace PhraseMatcher with Aho-Corasick
Replace PhraseMatcher with the Aho-Corasick algorithm over numpy arrays
of the hash values for the relevant attribute. The implementation is
based on FlashText.
The speed should be similar to the previous PhraseMatcher. It is now
possible to easily remove match IDs and matches don't go missing with
large keyword lists / vocabularies.
Fixes #4308.
* Restore support for pickling
* Fix internal keyword add/remove for numpy arrays
* Add test for #4248, clean up test
* Improve efficiency of special cases handling
* Use PhraseMatcher instead of Matcher
* Improve efficiency of merging/splitting special cases in document
* Process merge/splits in one pass without repeated token shifting
* Merge in place if no splits
* Update error message number
* Remove UD script modifications
Only used for timing/testing, should be a separate PR
* Remove final traces of UD script modifications
* Update UD bin scripts
* Update imports for `bin/`
* Add all currently supported languages
* Update subtok merger for new Matcher validation
* Modify blinded check to look at tokens instead of lemmas (for corpora
with tokens but not lemmas like Telugu)
* Add missing loop for match ID set in search loop
* Remove cruft in matching loop for partial matches
There was a bit of unnecessary code left over from FlashText in the
matching loop to handle partial token matches, which we don't have with
PhraseMatcher.
* Replace dict trie with MapStruct trie
* Fix how match ID hash is stored/added
* Update fix for match ID vocab
* Switch from map_get_unless_missing to map_get
* Switch from numpy array to Token.get_struct_attr
Access token attributes directly in Doc instead of making a copy of the
relevant values in a numpy array.
Add unsatisfactory warning for hash collision with reserved terminal
hash key. (Ideally it would change the reserved terminal hash and redo
the whole trie, but for now, I'm hoping there won't be collisions.)
* Restructure imports to export find_matches
* Implement full remove()
Remove unnecessary trie paths and free unused maps.
Parallel to Matcher, raise KeyError when attempting to remove a match ID
that has not been added.
* Switch to PhraseMatcher.find_matches
* Switch to local cdef functions for span filtering
* Switch special case reload threshold to variable
Refer to variable instead of hard-coded threshold
* Move more of special case retokenize to cdef nogil
Move as much of the special case retokenization to nogil as possible.
* Rewrap sort as stdsort for OS X
* Rewrap stdsort with specific types
* Switch to qsort
* Fix merge
* Improve cmp functions
* Fix realloc
* Fix realloc again
* Initialize span struct while retokenizing
* Temporarily skip retokenizing
* Revert "Move more of special case retokenize to cdef nogil"
This reverts commit 0b7e52c797cd8ff1548f214bd4186ebb3a7ce8b1.
* Revert "Switch to qsort"
This reverts commit a98d71a942fc9bca531cf5eb05cf89fa88153b60.
* Fix specials check while caching
* Modify URL test with emoticons
The multiple suffix tests result in the emoticon `:>`, which is now
retokenized into one token as a special case after the suffixes are
split off.
* Refactor _apply_special_cases()
* Use cdef ints for span info used in multiple spots
* Modify _filter_special_spans() to prefer earlier
Parallel to #4414, modify _filter_special_spans() so that the earlier
span is preferred for overlapping spans of the same length.
* Replace MatchStruct with Entity
Replace MatchStruct with Entity since the existing Entity struct is
nearly identical.
* Replace Entity with more general SpanC
* Replace MatchStruct with SpanC
* Add error in debug-data if no dev docs are available (see #4575)
* Update azure-pipelines.yml
* Revert "Update azure-pipelines.yml"
This reverts commit ed1060cf59e5895b5fe92ad5b894fd1078ec4c49.
* Use latest wasabi
* Reorganise install_requires
* add dframcy to universe.json (#4580)
* Update universe.json [ci skip]
* Fix multiprocessing for as_tuples=True (#4582)
* Fix conllu script (#4579)
* force extensions to avoid clash between example scripts
* fix arg order and default file encoding
* add example config for conllu script
* newline
* move extension definitions to main function
* few more encodings fixes
* Add load_from_docbin example [ci skip]
TODO: upload the file somewhere
* Update README.md
* Add warnings about 3.8 (resolves #4593) [ci skip]
* Fixed typo: Added space between "recognize" and "various" (#4600)
* Fix DocBin.merge() example (#4599)
* Replace function registries with catalogue (#4584)
* Replace functions registries with catalogue
* Update __init__.py
* Fix test
* Revert unrelated flag [ci skip]
* Bugfix/dep matcher issue 4590 (#4601)
* add contributor agreement for prilopes
* add test for issue #4590
* fix on_match params for DependencyMacther (#4590)
* Minor updates to language example sentences (#4608)
* Add punctuation to Spanish example sentences
* Combine multilanguage examples for lang xx
* Add punctuation to nb examples
* Always realloc to a larger size
Avoid potential (unlikely) edge case and cymem error seen in #4604.
* Add error in debug-data if no dev docs are available (see #4575)
* Update debug-data for GoldCorpus / Example
* Ignore None label in misaligned NER data
2019-11-13 20:24:35 +00:00
|
|
|
from wasabi import msg
|
2018-12-10 08:46:53 +00:00
|
|
|
import contextlib
|
|
|
|
import random
|
2017-03-23 10:08:41 +00:00
|
|
|
|
2020-02-28 10:57:41 +00:00
|
|
|
from ..util import create_default_optimizer
|
2020-02-11 22:45:41 +00:00
|
|
|
from ..util import use_gpu as set_gpu
|
2018-02-17 17:11:11 +00:00
|
|
|
from ..attrs import PROB, IS_OOV, CLUSTER, LANG
|
2018-03-27 17:23:02 +00:00
|
|
|
from ..gold import GoldCorpus
|
2017-03-23 10:08:41 +00:00
|
|
|
from .. import util
|
2017-09-25 17:00:47 +00:00
|
|
|
from .. import about
|
2018-11-30 19:16:14 +00:00
|
|
|
|
|
|
|
|
2020-01-01 12:15:46 +00:00
|
|
|
def train(
|
2019-09-15 20:31:31 +00:00
|
|
|
# fmt: off
|
2020-01-01 12:15:46 +00:00
|
|
|
lang: ("Model language", "positional", None, str),
|
|
|
|
output_path: ("Output directory to store model in", "positional", None, Path),
|
|
|
|
train_path: ("Location of JSON-formatted training data", "positional", None, Path),
|
|
|
|
dev_path: ("Location of JSON-formatted development data", "positional", None, Path),
|
|
|
|
raw_text: ("Path to jsonl file with unlabelled text documents.", "option", "rt", Path) = None,
|
|
|
|
base_model: ("Name of model to update (optional)", "option", "b", str) = None,
|
|
|
|
pipeline: ("Comma-separated names of pipeline components", "option", "p", str) = "tagger,parser,ner",
|
|
|
|
vectors: ("Model to load vectors from", "option", "v", str) = None,
|
2020-02-18 13:47:23 +00:00
|
|
|
replace_components: ("Replace components from base model", "flag", "R", bool) = False,
|
2020-01-01 12:15:46 +00:00
|
|
|
n_iter: ("Number of iterations", "option", "n", int) = 30,
|
|
|
|
n_early_stopping: ("Maximum number of training epochs without dev accuracy improvement", "option", "ne", int) = None,
|
|
|
|
n_examples: ("Number of examples", "option", "ns", int) = 0,
|
|
|
|
use_gpu: ("Use GPU", "option", "g", int) = -1,
|
|
|
|
version: ("Model version", "option", "V", str) = "0.0.0",
|
|
|
|
meta_path: ("Optional path to meta.json to use as base.", "option", "m", Path) = None,
|
|
|
|
init_tok2vec: ("Path to pretrained weights for the token-to-vector parts of the models. See 'spacy pretrain'. Experimental.", "option", "t2v", Path) = None,
|
|
|
|
parser_multitasks: ("Side objectives for parser CNN, e.g. 'dep' or 'dep,tag'", "option", "pt", str) = "",
|
|
|
|
entity_multitasks: ("Side objectives for NER CNN, e.g. 'dep' or 'dep,tag'", "option", "et", str) = "",
|
|
|
|
noise_level: ("Amount of corruption for data augmentation", "option", "nl", float) = 0.0,
|
|
|
|
orth_variant_level: ("Amount of orthography variation for data augmentation", "option", "ovl", float) = 0.0,
|
|
|
|
eval_beam_widths: ("Beam widths to evaluate, e.g. 4,8", "option", "bw", str) = "",
|
|
|
|
gold_preproc: ("Use gold preprocessing", "flag", "G", bool) = False,
|
|
|
|
learn_tokens: ("Make parser learn gold-standard tokenization", "flag", "T", bool) = False,
|
|
|
|
textcat_multilabel: ("Textcat classes aren't mutually exclusive (multilabel)", "flag", "TML", bool) = False,
|
|
|
|
textcat_arch: ("Textcat model architecture", "option", "ta", str) = "bow",
|
|
|
|
textcat_positive_label: ("Textcat positive label for binary classes with two labels", "option", "tpl", str) = None,
|
|
|
|
tag_map_path: ("Location of JSON-formatted tag map", "option", "tm", Path) = None,
|
|
|
|
verbose: ("Display more information for debug", "flag", "VV", bool) = False,
|
|
|
|
debug: ("Run data diagnostics before training", "flag", "D", bool) = False,
|
2019-09-15 20:31:31 +00:00
|
|
|
# fmt: on
|
2018-11-30 19:16:14 +00:00
|
|
|
):
|
2017-05-27 18:01:46 +00:00
|
|
|
"""
|
2018-11-30 19:16:14 +00:00
|
|
|
Train or update a spaCy model. Requires data to be formatted in spaCy's
|
|
|
|
JSON format. To convert data from other formats, use the `spacy convert`
|
|
|
|
command.
|
2017-05-27 18:01:46 +00:00
|
|
|
"""
|
2018-02-13 11:42:23 +00:00
|
|
|
util.fix_random_seed()
|
2018-11-30 19:16:14 +00:00
|
|
|
util.set_env_log(verbose)
|
|
|
|
|
|
|
|
# Make sure all files and paths exists if they are needed
|
|
|
|
train_path = util.ensure_path(train_path)
|
|
|
|
dev_path = util.ensure_path(dev_path)
|
2017-09-25 17:00:47 +00:00
|
|
|
meta_path = util.ensure_path(meta_path)
|
2019-04-15 10:04:36 +00:00
|
|
|
output_path = util.ensure_path(output_path)
|
💫 Better support for semi-supervised learning (#3035)
The new spacy pretrain command implemented BERT/ULMFit/etc-like transfer learning, using our Language Modelling with Approximate Outputs version of BERT's cloze task. Pretraining is convenient, but in some ways it's a bit of a strange solution. All we're doing is initialising the weights. At the same time, we're putting a lot of work into our optimisation so that it's less sensitive to initial conditions, and more likely to find good optima. I discuss this a bit in the pseudo-rehearsal blog post: https://explosion.ai/blog/pseudo-rehearsal-catastrophic-forgetting
Support semi-supervised learning in spacy train
One obvious way to improve these pretraining methods is to do multi-task learning, instead of just transfer learning. This has been shown to work very well: https://arxiv.org/pdf/1809.08370.pdf . This patch makes it easy to do this sort of thing.
Add a new argument to spacy train, --raw-text. This takes a jsonl file with unlabelled data that can be used in arbitrary ways to do semi-supervised learning.
Add a new method to the Language class and to pipeline components, .rehearse(). This is like .update(), but doesn't expect GoldParse objects. It takes a batch of Doc objects, and performs an update on some semi-supervised objective.
Move the BERT-LMAO objective out from spacy/cli/pretrain.py into spacy/_ml.py, so we can create a new pipeline component, ClozeMultitask. This can be specified as a parser or NER multitask in the spacy train command. Example usage:
python -m spacy train en ./tmp ~/data/en-core-web/train/nw.json ~/data/en-core-web/dev/nw.json --pipeline parser --raw-textt ~/data/unlabelled/reddit-100k.jsonl --vectors en_vectors_web_lg --parser-multitasks cloze
Implement rehearsal methods for pipeline components
The new --raw-text argument and nlp.rehearse() method also gives us a good place to implement the the idea in the pseudo-rehearsal blog post in the parser. This works as follows:
Add a new nlp.resume_training() method. This allocates copies of pre-trained models in the pipeline, setting things up for the rehearsal updates. It also returns an optimizer object. This also greatly reduces confusion around the nlp.begin_training() method, which randomises the weights, making it not suitable for adding new labels or otherwise fine-tuning a pre-trained model.
Implement rehearsal updates on the Parser class, making it available for the dependency parser and NER. During rehearsal, the initial model is used to supervise the model being trained. The current model is asked to match the predictions of the initial model on some data. This minimises catastrophic forgetting, by keeping the model's predictions close to the original. See the blog post for details.
Implement rehearsal updates for tagger
Implement rehearsal updates for text categoriz
2018-12-10 15:25:33 +00:00
|
|
|
if raw_text is not None:
|
|
|
|
raw_text = list(srsly.read_jsonl(raw_text))
|
2018-11-30 19:16:14 +00:00
|
|
|
if not train_path or not train_path.exists():
|
2018-12-08 10:49:43 +00:00
|
|
|
msg.fail("Training data not found", train_path, exits=1)
|
2018-11-30 19:16:14 +00:00
|
|
|
if not dev_path or not dev_path.exists():
|
2018-12-08 10:49:43 +00:00
|
|
|
msg.fail("Development data not found", dev_path, exits=1)
|
2017-09-25 17:00:47 +00:00
|
|
|
if meta_path is not None and not meta_path.exists():
|
2018-12-08 10:49:43 +00:00
|
|
|
msg.fail("Can't find model meta.json", meta_path, exits=1)
|
💫 Replace ujson, msgpack and dill/pickle/cloudpickle with srsly (#3003)
Remove hacks and wrappers, keep code in sync across our libraries and move spaCy a few steps closer to only depending on packages with binary wheels 🎉
See here: https://github.com/explosion/srsly
Serialization is hard, especially across Python versions and multiple platforms. After dealing with many subtle bugs over the years (encodings, locales, large files) our libraries like spaCy and Prodigy have steadily grown a number of utility functions to wrap the multiple serialization formats we need to support (especially json, msgpack and pickle). These wrapping functions ended up duplicated across our codebases, so we wanted to put them in one place.
At the same time, we noticed that having a lot of small dependencies was making maintainence harder, and making installation slower. To solve this, we've made srsly standalone, by including the component packages directly within it. This way we can provide all the serialization utilities we need in a single binary wheel.
srsly currently includes forks of the following packages:
ujson
msgpack
msgpack-numpy
cloudpickle
* WIP: replace json/ujson with srsly
* Replace ujson in examples
Use regular json instead of srsly to make code easier to read and follow
* Update requirements
* Fix imports
* Fix typos
* Replace msgpack with srsly
* Fix warning
2018-12-03 00:28:22 +00:00
|
|
|
meta = srsly.read_json(meta_path) if meta_path else {}
|
2018-11-30 19:16:14 +00:00
|
|
|
if output_path.exists() and [p for p in output_path.iterdir() if p.is_dir()]:
|
2018-12-08 10:49:43 +00:00
|
|
|
msg.warn(
|
|
|
|
"Output directory is not empty",
|
|
|
|
"This can lead to unintended side effects when saving the model. "
|
|
|
|
"Please use an empty directory or a different path instead. If "
|
|
|
|
"the specified output path doesn't exist, the directory will be "
|
|
|
|
"created for you.",
|
|
|
|
)
|
2018-03-27 17:23:02 +00:00
|
|
|
if not output_path.exists():
|
|
|
|
output_path.mkdir()
|
2020-02-18 14:38:18 +00:00
|
|
|
msg.good(f"Created output directory: {output_path}")
|
2017-03-23 10:08:41 +00:00
|
|
|
|
2019-12-13 09:46:18 +00:00
|
|
|
tag_map = {}
|
|
|
|
if tag_map_path is not None:
|
|
|
|
tag_map = srsly.read_json(tag_map_path)
|
2018-12-07 19:54:35 +00:00
|
|
|
# Take dropout and batch size as generators of values -- dropout
|
|
|
|
# starts high and decays sharply, to force the optimizer to explore.
|
|
|
|
# Batch size starts at 1 and grows, so that we make updates quickly
|
|
|
|
# at the beginning of training.
|
|
|
|
dropout_rates = util.decaying(
|
2018-12-08 18:59:11 +00:00
|
|
|
util.env_opt("dropout_from", 0.2),
|
|
|
|
util.env_opt("dropout_to", 0.2),
|
2018-12-07 19:54:35 +00:00
|
|
|
util.env_opt("dropout_decay", 0.0),
|
|
|
|
)
|
|
|
|
batch_sizes = util.compounding(
|
|
|
|
util.env_opt("batch_from", 100.0),
|
2018-12-17 23:15:39 +00:00
|
|
|
util.env_opt("batch_to", 1000.0),
|
2018-12-07 19:54:35 +00:00
|
|
|
util.env_opt("batch_compound", 1.001),
|
|
|
|
)
|
|
|
|
|
2019-03-16 14:02:39 +00:00
|
|
|
if not eval_beam_widths:
|
|
|
|
eval_beam_widths = [1]
|
|
|
|
else:
|
|
|
|
eval_beam_widths = [int(bw) for bw in eval_beam_widths.split(",")]
|
|
|
|
if 1 not in eval_beam_widths:
|
|
|
|
eval_beam_widths.append(1)
|
|
|
|
eval_beam_widths.sort()
|
2019-03-16 14:55:31 +00:00
|
|
|
has_beam_widths = eval_beam_widths != [1]
|
2019-03-16 14:02:39 +00:00
|
|
|
|
2020-02-27 17:42:27 +00:00
|
|
|
default_dir = Path(__file__).parent.parent / "ml" / "models" / "defaults"
|
|
|
|
|
2018-11-30 19:16:14 +00:00
|
|
|
# Set up the base model and pipeline. If a base model is specified, load
|
|
|
|
# the model and make sure the pipeline matches the pipeline setting. If
|
|
|
|
# training starts from a blank model, intitalize the language class.
|
|
|
|
pipeline = [p.strip() for p in pipeline.split(",")]
|
2019-12-22 00:53:56 +00:00
|
|
|
msg.text(f"Training pipeline: {pipeline}")
|
2020-01-16 00:58:51 +00:00
|
|
|
disabled_pipes = None
|
|
|
|
pipes_added = False
|
2020-02-11 22:45:41 +00:00
|
|
|
if use_gpu >= 0:
|
|
|
|
activated_gpu = None
|
|
|
|
try:
|
|
|
|
activated_gpu = set_gpu(use_gpu)
|
|
|
|
except Exception as e:
|
2020-02-18 14:38:18 +00:00
|
|
|
msg.warn(f"Exception: {e}")
|
2020-02-11 22:45:41 +00:00
|
|
|
if activated_gpu is not None:
|
2020-02-18 14:38:18 +00:00
|
|
|
msg.text(f"Using GPU: {use_gpu}")
|
2020-02-11 22:45:41 +00:00
|
|
|
else:
|
2020-02-18 14:38:18 +00:00
|
|
|
msg.warn(f"Unable to activate GPU: {use_gpu}")
|
2020-02-11 22:45:41 +00:00
|
|
|
msg.text("Using CPU only")
|
|
|
|
use_gpu = -1
|
2018-11-30 19:16:14 +00:00
|
|
|
if base_model:
|
2019-12-22 00:53:56 +00:00
|
|
|
msg.text(f"Starting with base model '{base_model}'")
|
2018-11-30 19:16:14 +00:00
|
|
|
nlp = util.load_model(base_model)
|
|
|
|
if nlp.lang != lang:
|
2018-12-08 10:49:43 +00:00
|
|
|
msg.fail(
|
2019-12-22 00:53:56 +00:00
|
|
|
f"Model language ('{nlp.lang}') doesn't match language "
|
|
|
|
f"specified as `lang` argument ('{lang}') ",
|
2018-12-08 10:49:43 +00:00
|
|
|
exits=1,
|
|
|
|
)
|
2020-02-27 17:42:27 +00:00
|
|
|
if vectors:
|
|
|
|
msg.text(f"Loading vectors from model '{vectors}'")
|
2020-03-25 09:20:11 +00:00
|
|
|
_load_vectors(nlp, vectors)
|
2020-02-27 17:42:27 +00:00
|
|
|
|
2020-05-18 20:27:10 +00:00
|
|
|
nlp.select_pipes(disable=[p for p in nlp.pipe_names if p not in pipeline])
|
2018-11-30 19:16:14 +00:00
|
|
|
for pipe in pipeline:
|
2020-02-27 17:42:27 +00:00
|
|
|
# first, create the model.
|
|
|
|
# Bit of a hack after the refactor to get the vectors into a default config
|
|
|
|
# use train-from-config instead :-)
|
|
|
|
if pipe == "parser":
|
|
|
|
config_loc = default_dir / "parser_defaults.cfg"
|
|
|
|
elif pipe == "tagger":
|
|
|
|
config_loc = default_dir / "tagger_defaults.cfg"
|
|
|
|
elif pipe == "ner":
|
|
|
|
config_loc = default_dir / "ner_defaults.cfg"
|
|
|
|
elif pipe == "textcat":
|
|
|
|
config_loc = default_dir / "textcat_defaults.cfg"
|
2020-03-06 13:45:02 +00:00
|
|
|
elif pipe == "senter":
|
|
|
|
config_loc = default_dir / "senter_defaults.cfg"
|
2020-02-27 17:42:27 +00:00
|
|
|
else:
|
|
|
|
raise ValueError(f"Component {pipe} currently not supported.")
|
|
|
|
pipe_cfg = util.load_config(config_loc, create_objects=False)
|
|
|
|
if vectors:
|
2020-02-28 10:57:41 +00:00
|
|
|
pretrained_config = {
|
|
|
|
"@architectures": "spacy.VocabVectors.v1",
|
|
|
|
"name": vectors,
|
|
|
|
}
|
2020-02-27 17:42:27 +00:00
|
|
|
pipe_cfg["model"]["tok2vec"]["pretrained_vectors"] = pretrained_config
|
|
|
|
|
2020-01-16 00:58:51 +00:00
|
|
|
if pipe == "parser":
|
2020-02-27 17:42:27 +00:00
|
|
|
pipe_cfg["learn_tokens"] = learn_tokens
|
2020-01-16 00:58:51 +00:00
|
|
|
elif pipe == "textcat":
|
2020-02-27 17:42:27 +00:00
|
|
|
pipe_cfg["exclusive_classes"] = not textcat_multilabel
|
|
|
|
pipe_cfg["architecture"] = textcat_arch
|
|
|
|
pipe_cfg["positive_label"] = textcat_positive_label
|
|
|
|
|
2018-11-30 19:16:14 +00:00
|
|
|
if pipe not in nlp.pipe_names:
|
2020-02-18 14:38:18 +00:00
|
|
|
msg.text(f"Adding component to base model '{pipe}'")
|
2019-08-23 15:54:00 +00:00
|
|
|
nlp.add_pipe(nlp.create_pipe(pipe, config=pipe_cfg))
|
2020-01-16 00:58:51 +00:00
|
|
|
pipes_added = True
|
|
|
|
elif replace_components:
|
2020-02-18 14:38:18 +00:00
|
|
|
msg.text(f"Replacing component from base model '{pipe}'")
|
2020-01-16 00:58:51 +00:00
|
|
|
nlp.replace_pipe(pipe, nlp.create_pipe(pipe, config=pipe_cfg))
|
|
|
|
pipes_added = True
|
2019-09-15 20:31:31 +00:00
|
|
|
else:
|
|
|
|
if pipe == "textcat":
|
|
|
|
textcat_cfg = nlp.get_pipe("textcat").cfg
|
|
|
|
base_cfg = {
|
|
|
|
"exclusive_classes": textcat_cfg["exclusive_classes"],
|
|
|
|
"architecture": textcat_cfg["architecture"],
|
2019-09-18 18:27:03 +00:00
|
|
|
"positive_label": textcat_cfg["positive_label"],
|
2019-09-15 20:31:31 +00:00
|
|
|
}
|
|
|
|
if base_cfg != pipe_cfg:
|
2019-09-18 18:27:03 +00:00
|
|
|
msg.fail(
|
2019-12-22 00:53:56 +00:00
|
|
|
f"The base textcat model configuration does"
|
|
|
|
f"not match the provided training options. "
|
|
|
|
f"Existing cfg: {base_cfg}, provided cfg: {pipe_cfg}",
|
2019-09-18 18:27:03 +00:00
|
|
|
exits=1,
|
2019-09-15 20:31:31 +00:00
|
|
|
)
|
2020-02-18 14:38:18 +00:00
|
|
|
msg.text(f"Extending component from base model '{pipe}'")
|
2020-05-18 20:27:10 +00:00
|
|
|
disabled_pipes = nlp.select_pipes(
|
|
|
|
disable=[p for p in nlp.pipe_names if p not in pipeline]
|
2020-02-18 13:47:23 +00:00
|
|
|
)
|
2018-11-30 19:16:14 +00:00
|
|
|
else:
|
2019-12-22 00:53:56 +00:00
|
|
|
msg.text(f"Starting with blank model '{lang}'")
|
2018-11-30 19:16:14 +00:00
|
|
|
lang_cls = util.get_lang_class(lang)
|
|
|
|
nlp = lang_cls()
|
2020-02-28 10:57:41 +00:00
|
|
|
|
2020-02-27 17:42:27 +00:00
|
|
|
if vectors:
|
|
|
|
msg.text(f"Loading vectors from model '{vectors}'")
|
2020-03-25 09:20:11 +00:00
|
|
|
_load_vectors(nlp, vectors)
|
2020-02-27 17:42:27 +00:00
|
|
|
|
2018-11-30 19:16:14 +00:00
|
|
|
for pipe in pipeline:
|
2020-02-27 17:42:27 +00:00
|
|
|
# first, create the model.
|
|
|
|
# Bit of a hack after the refactor to get the vectors into a default config
|
|
|
|
# use train-from-config instead :-)
|
2019-08-23 15:54:00 +00:00
|
|
|
if pipe == "parser":
|
2020-02-27 17:42:27 +00:00
|
|
|
config_loc = default_dir / "parser_defaults.cfg"
|
|
|
|
elif pipe == "tagger":
|
|
|
|
config_loc = default_dir / "tagger_defaults.cfg"
|
2020-04-02 12:46:32 +00:00
|
|
|
elif pipe == "morphologizer":
|
|
|
|
config_loc = default_dir / "morphologizer_defaults.cfg"
|
2020-02-27 17:42:27 +00:00
|
|
|
elif pipe == "ner":
|
|
|
|
config_loc = default_dir / "ner_defaults.cfg"
|
2019-09-15 20:31:31 +00:00
|
|
|
elif pipe == "textcat":
|
2020-02-27 17:42:27 +00:00
|
|
|
config_loc = default_dir / "textcat_defaults.cfg"
|
2020-03-06 13:45:02 +00:00
|
|
|
elif pipe == "senter":
|
|
|
|
config_loc = default_dir / "senter_defaults.cfg"
|
2019-08-23 15:54:00 +00:00
|
|
|
else:
|
2020-02-27 17:42:27 +00:00
|
|
|
raise ValueError(f"Component {pipe} currently not supported.")
|
|
|
|
pipe_cfg = util.load_config(config_loc, create_objects=False)
|
|
|
|
if vectors:
|
2020-02-28 10:57:41 +00:00
|
|
|
pretrained_config = {
|
|
|
|
"@architectures": "spacy.VocabVectors.v1",
|
|
|
|
"name": vectors,
|
|
|
|
}
|
2020-02-27 17:42:27 +00:00
|
|
|
pipe_cfg["model"]["tok2vec"]["pretrained_vectors"] = pretrained_config
|
|
|
|
|
|
|
|
if pipe == "parser":
|
|
|
|
pipe_cfg["learn_tokens"] = learn_tokens
|
|
|
|
elif pipe == "textcat":
|
|
|
|
pipe_cfg["exclusive_classes"] = not textcat_multilabel
|
|
|
|
pipe_cfg["architecture"] = textcat_arch
|
|
|
|
pipe_cfg["positive_label"] = textcat_positive_label
|
|
|
|
|
|
|
|
pipe = nlp.create_pipe(pipe, config=pipe_cfg)
|
|
|
|
nlp.add_pipe(pipe)
|
2017-03-23 10:08:41 +00:00
|
|
|
|
2019-12-13 09:46:18 +00:00
|
|
|
# Update tag map with provided mapping
|
|
|
|
nlp.vocab.morphology.tag_map.update(tag_map)
|
|
|
|
|
2018-11-30 19:16:14 +00:00
|
|
|
# Multitask objectives
|
|
|
|
multitask_options = [("parser", parser_multitasks), ("ner", entity_multitasks)]
|
|
|
|
for pipe_name, multitasks in multitask_options:
|
|
|
|
if multitasks:
|
|
|
|
if pipe_name not in pipeline:
|
2018-12-08 10:49:43 +00:00
|
|
|
msg.fail(
|
2019-12-22 00:53:56 +00:00
|
|
|
f"Can't use multitask objective without '{pipe_name}' in "
|
|
|
|
f"the pipeline"
|
2018-12-08 10:49:43 +00:00
|
|
|
)
|
2018-11-30 19:16:14 +00:00
|
|
|
pipe = nlp.get_pipe(pipe_name)
|
|
|
|
for objective in multitasks.split(","):
|
|
|
|
pipe.add_multitask_objective(objective)
|
|
|
|
|
|
|
|
# Prepare training corpus
|
2019-12-22 00:53:56 +00:00
|
|
|
msg.text(f"Counting training words (limit={n_examples})")
|
2018-11-30 19:16:14 +00:00
|
|
|
corpus = GoldCorpus(train_path, dev_path, limit=n_examples)
|
|
|
|
n_train_words = corpus.count_train()
|
|
|
|
|
2020-01-16 00:58:51 +00:00
|
|
|
if base_model and not pipes_added:
|
2018-11-30 19:16:14 +00:00
|
|
|
# Start with an existing model, use default optimizer
|
2020-01-29 16:06:46 +00:00
|
|
|
optimizer = create_default_optimizer()
|
2018-11-30 19:16:14 +00:00
|
|
|
else:
|
|
|
|
# Start with a blank model, call begin_training
|
2020-02-16 16:16:41 +00:00
|
|
|
cfg = {"device": use_gpu}
|
2020-02-26 07:44:22 +00:00
|
|
|
optimizer = nlp.begin_training(lambda: corpus.train_examples, **cfg)
|
2017-09-02 17:46:01 +00:00
|
|
|
nlp._optimizer = None
|
2017-05-25 21:16:30 +00:00
|
|
|
|
2020-02-27 17:42:27 +00:00
|
|
|
# Load in pretrained weights (TODO: this may be broken in the config rewrite)
|
2018-11-30 19:16:14 +00:00
|
|
|
if init_tok2vec is not None:
|
|
|
|
components = _load_pretrained_tok2vec(nlp, init_tok2vec)
|
2019-12-22 00:53:56 +00:00
|
|
|
msg.text(f"Loaded pretrained tok2vec for: {components}")
|
2018-11-30 19:16:14 +00:00
|
|
|
|
2019-09-15 20:31:31 +00:00
|
|
|
# Verify textcat config
|
|
|
|
if "textcat" in pipeline:
|
2020-01-16 00:58:51 +00:00
|
|
|
textcat_labels = nlp.get_pipe("textcat").cfg.get("labels", [])
|
2019-09-15 20:31:31 +00:00
|
|
|
if textcat_positive_label and textcat_positive_label not in textcat_labels:
|
|
|
|
msg.fail(
|
2019-12-22 00:53:56 +00:00
|
|
|
f"The textcat_positive_label (tpl) '{textcat_positive_label}' "
|
|
|
|
f"does not match any label in the training data.",
|
2019-09-15 20:31:31 +00:00
|
|
|
exits=1,
|
|
|
|
)
|
|
|
|
if textcat_positive_label and len(textcat_labels) != 2:
|
|
|
|
msg.fail(
|
2019-12-22 00:53:56 +00:00
|
|
|
"A textcat_positive_label (tpl) '{textcat_positive_label}' was "
|
|
|
|
"provided for training data that does not appear to be a "
|
|
|
|
"binary classification problem with two labels.",
|
2019-09-15 20:31:31 +00:00
|
|
|
exits=1,
|
|
|
|
)
|
2019-11-11 16:35:27 +00:00
|
|
|
train_data = corpus.train_data(
|
2019-10-28 11:43:55 +00:00
|
|
|
nlp,
|
|
|
|
noise_level=noise_level,
|
|
|
|
gold_preproc=gold_preproc,
|
|
|
|
max_length=0,
|
|
|
|
ignore_misaligned=True,
|
2019-09-15 20:31:31 +00:00
|
|
|
)
|
|
|
|
train_labels = set()
|
|
|
|
if textcat_multilabel:
|
|
|
|
multilabel_found = False
|
2019-11-11 16:35:27 +00:00
|
|
|
for ex in train_data:
|
|
|
|
train_labels.update(ex.gold.cats.keys())
|
|
|
|
if list(ex.gold.cats.values()).count(1.0) != 1:
|
2019-09-15 20:31:31 +00:00
|
|
|
multilabel_found = True
|
|
|
|
if not multilabel_found and not base_model:
|
|
|
|
msg.warn(
|
|
|
|
"The textcat training instances look like they have "
|
|
|
|
"mutually-exclusive classes. Remove the flag "
|
|
|
|
"'--textcat-multilabel' to train a classifier with "
|
|
|
|
"mutually-exclusive classes."
|
|
|
|
)
|
|
|
|
if not textcat_multilabel:
|
2019-11-11 16:35:27 +00:00
|
|
|
for ex in train_data:
|
|
|
|
train_labels.update(ex.gold.cats.keys())
|
|
|
|
if list(ex.gold.cats.values()).count(1.0) != 1 and not base_model:
|
2019-09-15 20:31:31 +00:00
|
|
|
msg.warn(
|
|
|
|
"Some textcat training instances do not have exactly "
|
|
|
|
"one positive label. Modifying training options to "
|
|
|
|
"include the flag '--textcat-multilabel' for classes "
|
|
|
|
"that are not mutually exclusive."
|
|
|
|
)
|
|
|
|
nlp.get_pipe("textcat").cfg["exclusive_classes"] = False
|
|
|
|
textcat_multilabel = True
|
|
|
|
break
|
|
|
|
if base_model and set(textcat_labels) != train_labels:
|
|
|
|
msg.fail(
|
2019-12-22 00:53:56 +00:00
|
|
|
f"Cannot extend textcat model using data with different "
|
|
|
|
f"labels. Base model labels: {textcat_labels}, training data "
|
|
|
|
f"labels: {list(train_labels)}",
|
2019-09-18 18:27:03 +00:00
|
|
|
exits=1,
|
2019-09-15 20:31:31 +00:00
|
|
|
)
|
|
|
|
if textcat_multilabel:
|
|
|
|
msg.text(
|
2019-12-22 00:53:56 +00:00
|
|
|
f"Textcat evaluation score: ROC AUC score macro-averaged across "
|
|
|
|
f"the labels '{', '.join(textcat_labels)}'"
|
2019-09-15 20:31:31 +00:00
|
|
|
)
|
|
|
|
elif textcat_positive_label and len(textcat_labels) == 2:
|
|
|
|
msg.text(
|
2019-12-22 00:53:56 +00:00
|
|
|
f"Textcat evaluation score: F1-score for the "
|
|
|
|
f"label '{textcat_positive_label}'"
|
2019-09-15 20:31:31 +00:00
|
|
|
)
|
|
|
|
elif len(textcat_labels) > 1:
|
|
|
|
if len(textcat_labels) == 2:
|
|
|
|
msg.warn(
|
|
|
|
"If the textcat component is a binary classifier with "
|
|
|
|
"exclusive classes, provide '--textcat_positive_label' for "
|
|
|
|
"an evaluation on the positive class."
|
|
|
|
)
|
|
|
|
msg.text(
|
2019-12-22 00:53:56 +00:00
|
|
|
f"Textcat evaluation score: F1-score macro-averaged across "
|
|
|
|
f"the labels '{', '.join(textcat_labels)}'"
|
2019-09-15 20:31:31 +00:00
|
|
|
)
|
|
|
|
else:
|
|
|
|
msg.fail(
|
|
|
|
"Unsupported textcat configuration. Use `spacy debug-data` "
|
|
|
|
"for more information."
|
|
|
|
)
|
|
|
|
|
2018-12-08 10:49:43 +00:00
|
|
|
# fmt: off
|
2019-09-15 20:31:31 +00:00
|
|
|
row_head, output_stats = _configure_training_output(pipeline, use_gpu, has_beam_widths)
|
|
|
|
row_widths = [len(w) for w in row_head]
|
2019-03-16 14:55:31 +00:00
|
|
|
row_settings = {"widths": row_widths, "aligns": tuple(["r" for i in row_head]), "spacing": 2}
|
2018-12-08 10:49:43 +00:00
|
|
|
# fmt: on
|
|
|
|
print("")
|
|
|
|
msg.row(row_head, **row_settings)
|
|
|
|
msg.row(["-" * width for width in row_settings["widths"]], **row_settings)
|
2017-05-26 10:52:09 +00:00
|
|
|
try:
|
2019-04-15 10:04:36 +00:00
|
|
|
iter_since_best = 0
|
2019-04-22 12:31:11 +00:00
|
|
|
best_score = 0.0
|
2017-05-26 10:52:09 +00:00
|
|
|
for i in range(n_iter):
|
2019-11-23 13:32:15 +00:00
|
|
|
train_data = corpus.train_dataset(
|
2019-09-11 12:00:36 +00:00
|
|
|
nlp,
|
|
|
|
noise_level=noise_level,
|
|
|
|
orth_variant_level=orth_variant_level,
|
|
|
|
gold_preproc=gold_preproc,
|
|
|
|
max_length=0,
|
2019-10-28 11:43:55 +00:00
|
|
|
ignore_misaligned=True,
|
2018-11-30 19:16:14 +00:00
|
|
|
)
|
💫 Better support for semi-supervised learning (#3035)
The new spacy pretrain command implemented BERT/ULMFit/etc-like transfer learning, using our Language Modelling with Approximate Outputs version of BERT's cloze task. Pretraining is convenient, but in some ways it's a bit of a strange solution. All we're doing is initialising the weights. At the same time, we're putting a lot of work into our optimisation so that it's less sensitive to initial conditions, and more likely to find good optima. I discuss this a bit in the pseudo-rehearsal blog post: https://explosion.ai/blog/pseudo-rehearsal-catastrophic-forgetting
Support semi-supervised learning in spacy train
One obvious way to improve these pretraining methods is to do multi-task learning, instead of just transfer learning. This has been shown to work very well: https://arxiv.org/pdf/1809.08370.pdf . This patch makes it easy to do this sort of thing.
Add a new argument to spacy train, --raw-text. This takes a jsonl file with unlabelled data that can be used in arbitrary ways to do semi-supervised learning.
Add a new method to the Language class and to pipeline components, .rehearse(). This is like .update(), but doesn't expect GoldParse objects. It takes a batch of Doc objects, and performs an update on some semi-supervised objective.
Move the BERT-LMAO objective out from spacy/cli/pretrain.py into spacy/_ml.py, so we can create a new pipeline component, ClozeMultitask. This can be specified as a parser or NER multitask in the spacy train command. Example usage:
python -m spacy train en ./tmp ~/data/en-core-web/train/nw.json ~/data/en-core-web/dev/nw.json --pipeline parser --raw-textt ~/data/unlabelled/reddit-100k.jsonl --vectors en_vectors_web_lg --parser-multitasks cloze
Implement rehearsal methods for pipeline components
The new --raw-text argument and nlp.rehearse() method also gives us a good place to implement the the idea in the pseudo-rehearsal blog post in the parser. This works as follows:
Add a new nlp.resume_training() method. This allocates copies of pre-trained models in the pipeline, setting things up for the rehearsal updates. It also returns an optimizer object. This also greatly reduces confusion around the nlp.begin_training() method, which randomises the weights, making it not suitable for adding new labels or otherwise fine-tuning a pre-trained model.
Implement rehearsal updates on the Parser class, making it available for the dependency parser and NER. During rehearsal, the initial model is used to supervise the model being trained. The current model is asked to match the predictions of the initial model on some data. This minimises catastrophic forgetting, by keeping the model's predictions close to the original. See the blog post for details.
Implement rehearsal updates for tagger
Implement rehearsal updates for text categoriz
2018-12-10 15:25:33 +00:00
|
|
|
if raw_text:
|
|
|
|
random.shuffle(raw_text)
|
|
|
|
raw_batches = util.minibatch(
|
|
|
|
(nlp.make_doc(rt["text"]) for rt in raw_text), size=8
|
|
|
|
)
|
2018-03-27 17:23:02 +00:00
|
|
|
words_seen = 0
|
2018-12-20 12:57:00 +00:00
|
|
|
with tqdm.tqdm(total=n_train_words, leave=False) as pbar:
|
2017-05-26 10:52:09 +00:00
|
|
|
losses = {}
|
2019-11-11 16:35:27 +00:00
|
|
|
for batch in util.minibatch_by_words(train_data, size=batch_sizes):
|
2017-11-03 00:54:54 +00:00
|
|
|
if not batch:
|
|
|
|
continue
|
2020-02-16 16:16:41 +00:00
|
|
|
try:
|
|
|
|
nlp.update(
|
2020-02-26 07:44:22 +00:00
|
|
|
batch,
|
2020-02-16 16:16:41 +00:00
|
|
|
sgd=optimizer,
|
|
|
|
drop=next(dropout_rates),
|
|
|
|
losses=losses,
|
|
|
|
)
|
|
|
|
except ValueError as e:
|
2020-03-25 11:28:12 +00:00
|
|
|
err = "Error during training"
|
2020-02-16 16:16:41 +00:00
|
|
|
if init_tok2vec:
|
2020-03-25 11:28:12 +00:00
|
|
|
err += " Did you provide the same parameters during 'train' as during 'pretrain'?"
|
2020-03-26 12:38:14 +00:00
|
|
|
msg.fail(err, f"Original error message: {e}", exits=1)
|
💫 Better support for semi-supervised learning (#3035)
The new spacy pretrain command implemented BERT/ULMFit/etc-like transfer learning, using our Language Modelling with Approximate Outputs version of BERT's cloze task. Pretraining is convenient, but in some ways it's a bit of a strange solution. All we're doing is initialising the weights. At the same time, we're putting a lot of work into our optimisation so that it's less sensitive to initial conditions, and more likely to find good optima. I discuss this a bit in the pseudo-rehearsal blog post: https://explosion.ai/blog/pseudo-rehearsal-catastrophic-forgetting
Support semi-supervised learning in spacy train
One obvious way to improve these pretraining methods is to do multi-task learning, instead of just transfer learning. This has been shown to work very well: https://arxiv.org/pdf/1809.08370.pdf . This patch makes it easy to do this sort of thing.
Add a new argument to spacy train, --raw-text. This takes a jsonl file with unlabelled data that can be used in arbitrary ways to do semi-supervised learning.
Add a new method to the Language class and to pipeline components, .rehearse(). This is like .update(), but doesn't expect GoldParse objects. It takes a batch of Doc objects, and performs an update on some semi-supervised objective.
Move the BERT-LMAO objective out from spacy/cli/pretrain.py into spacy/_ml.py, so we can create a new pipeline component, ClozeMultitask. This can be specified as a parser or NER multitask in the spacy train command. Example usage:
python -m spacy train en ./tmp ~/data/en-core-web/train/nw.json ~/data/en-core-web/dev/nw.json --pipeline parser --raw-textt ~/data/unlabelled/reddit-100k.jsonl --vectors en_vectors_web_lg --parser-multitasks cloze
Implement rehearsal methods for pipeline components
The new --raw-text argument and nlp.rehearse() method also gives us a good place to implement the the idea in the pseudo-rehearsal blog post in the parser. This works as follows:
Add a new nlp.resume_training() method. This allocates copies of pre-trained models in the pipeline, setting things up for the rehearsal updates. It also returns an optimizer object. This also greatly reduces confusion around the nlp.begin_training() method, which randomises the weights, making it not suitable for adding new labels or otherwise fine-tuning a pre-trained model.
Implement rehearsal updates on the Parser class, making it available for the dependency parser and NER. During rehearsal, the initial model is used to supervise the model being trained. The current model is asked to match the predictions of the initial model on some data. This minimises catastrophic forgetting, by keeping the model's predictions close to the original. See the blog post for details.
Implement rehearsal updates for tagger
Implement rehearsal updates for text categoriz
2018-12-10 15:25:33 +00:00
|
|
|
if raw_text:
|
|
|
|
# If raw text is available, perform 'rehearsal' updates,
|
|
|
|
# which use unlabelled data to reduce overfitting.
|
|
|
|
raw_batch = list(next(raw_batches))
|
|
|
|
nlp.rehearse(raw_batch, sgd=optimizer, losses=losses)
|
2019-11-11 16:35:27 +00:00
|
|
|
docs = [ex.doc for ex in batch]
|
💫 Better support for semi-supervised learning (#3035)
The new spacy pretrain command implemented BERT/ULMFit/etc-like transfer learning, using our Language Modelling with Approximate Outputs version of BERT's cloze task. Pretraining is convenient, but in some ways it's a bit of a strange solution. All we're doing is initialising the weights. At the same time, we're putting a lot of work into our optimisation so that it's less sensitive to initial conditions, and more likely to find good optima. I discuss this a bit in the pseudo-rehearsal blog post: https://explosion.ai/blog/pseudo-rehearsal-catastrophic-forgetting
Support semi-supervised learning in spacy train
One obvious way to improve these pretraining methods is to do multi-task learning, instead of just transfer learning. This has been shown to work very well: https://arxiv.org/pdf/1809.08370.pdf . This patch makes it easy to do this sort of thing.
Add a new argument to spacy train, --raw-text. This takes a jsonl file with unlabelled data that can be used in arbitrary ways to do semi-supervised learning.
Add a new method to the Language class and to pipeline components, .rehearse(). This is like .update(), but doesn't expect GoldParse objects. It takes a batch of Doc objects, and performs an update on some semi-supervised objective.
Move the BERT-LMAO objective out from spacy/cli/pretrain.py into spacy/_ml.py, so we can create a new pipeline component, ClozeMultitask. This can be specified as a parser or NER multitask in the spacy train command. Example usage:
python -m spacy train en ./tmp ~/data/en-core-web/train/nw.json ~/data/en-core-web/dev/nw.json --pipeline parser --raw-textt ~/data/unlabelled/reddit-100k.jsonl --vectors en_vectors_web_lg --parser-multitasks cloze
Implement rehearsal methods for pipeline components
The new --raw-text argument and nlp.rehearse() method also gives us a good place to implement the the idea in the pseudo-rehearsal blog post in the parser. This works as follows:
Add a new nlp.resume_training() method. This allocates copies of pre-trained models in the pipeline, setting things up for the rehearsal updates. It also returns an optimizer object. This also greatly reduces confusion around the nlp.begin_training() method, which randomises the weights, making it not suitable for adding new labels or otherwise fine-tuning a pre-trained model.
Implement rehearsal updates on the Parser class, making it available for the dependency parser and NER. During rehearsal, the initial model is used to supervise the model being trained. The current model is asked to match the predictions of the initial model on some data. This minimises catastrophic forgetting, by keeping the model's predictions close to the original. See the blog post for details.
Implement rehearsal updates for tagger
Implement rehearsal updates for text categoriz
2018-12-10 15:25:33 +00:00
|
|
|
if not int(os.environ.get("LOG_FRIENDLY", 0)):
|
2018-12-10 08:46:53 +00:00
|
|
|
pbar.update(sum(len(doc) for doc in docs))
|
2018-03-27 17:23:02 +00:00
|
|
|
words_seen += sum(len(doc) for doc in docs)
|
2017-05-26 10:52:09 +00:00
|
|
|
with nlp.use_params(optimizer.averages):
|
2017-06-03 18:28:20 +00:00
|
|
|
util.set_env_log(False)
|
2019-12-25 16:59:52 +00:00
|
|
|
epoch_model_path = output_path / f"model{i}"
|
2017-06-03 18:28:20 +00:00
|
|
|
nlp.to_disk(epoch_model_path)
|
2017-10-10 17:51:20 +00:00
|
|
|
nlp_loaded = util.load_model_from_path(epoch_model_path)
|
2019-03-16 14:02:39 +00:00
|
|
|
for beam_width in eval_beam_widths:
|
2019-03-15 14:18:28 +00:00
|
|
|
for name, component in nlp_loaded.pipeline:
|
|
|
|
if hasattr(component, "cfg"):
|
|
|
|
component.cfg["beam_width"] = beam_width
|
2019-11-11 16:35:27 +00:00
|
|
|
dev_dataset = list(
|
|
|
|
corpus.dev_dataset(
|
2019-10-28 11:43:55 +00:00
|
|
|
nlp_loaded,
|
|
|
|
gold_preproc=gold_preproc,
|
|
|
|
ignore_misaligned=True,
|
|
|
|
)
|
2019-03-16 14:55:13 +00:00
|
|
|
)
|
2019-11-11 16:35:27 +00:00
|
|
|
nwords = sum(len(ex.doc) for ex in dev_dataset)
|
2019-03-15 14:18:28 +00:00
|
|
|
start_time = timer()
|
2019-11-11 16:35:27 +00:00
|
|
|
scorer = nlp_loaded.evaluate(dev_dataset, verbose=verbose)
|
2019-03-15 14:18:28 +00:00
|
|
|
end_time = timer()
|
|
|
|
if use_gpu < 0:
|
|
|
|
gpu_wps = None
|
2018-11-30 19:16:14 +00:00
|
|
|
cpu_wps = nwords / (end_time - start_time)
|
2019-03-15 14:18:28 +00:00
|
|
|
else:
|
|
|
|
gpu_wps = nwords / (end_time - start_time)
|
2020-01-29 16:06:46 +00:00
|
|
|
with use_ops("numpy"):
|
2019-03-15 14:18:28 +00:00
|
|
|
nlp_loaded = util.load_model_from_path(epoch_model_path)
|
2019-04-15 10:04:36 +00:00
|
|
|
for name, component in nlp_loaded.pipeline:
|
|
|
|
if hasattr(component, "cfg"):
|
|
|
|
component.cfg["beam_width"] = beam_width
|
2019-11-11 16:35:27 +00:00
|
|
|
dev_dataset = list(
|
|
|
|
corpus.dev_dataset(
|
2019-10-28 11:43:55 +00:00
|
|
|
nlp_loaded,
|
|
|
|
gold_preproc=gold_preproc,
|
|
|
|
ignore_misaligned=True,
|
|
|
|
)
|
2019-03-15 14:18:28 +00:00
|
|
|
)
|
|
|
|
start_time = timer()
|
2019-11-11 16:35:27 +00:00
|
|
|
scorer = nlp_loaded.evaluate(dev_dataset, verbose=verbose)
|
2019-03-15 14:18:28 +00:00
|
|
|
end_time = timer()
|
|
|
|
cpu_wps = nwords / (end_time - start_time)
|
2019-12-25 16:59:52 +00:00
|
|
|
acc_loc = output_path / f"model{i}" / "accuracy.json"
|
2019-03-15 14:18:28 +00:00
|
|
|
srsly.write_json(acc_loc, scorer.scores)
|
|
|
|
|
|
|
|
# Update model meta.json
|
|
|
|
meta["lang"] = nlp.lang
|
|
|
|
meta["pipeline"] = nlp.pipe_names
|
|
|
|
if beam_width == 1:
|
2019-03-16 14:55:13 +00:00
|
|
|
meta["speed"] = {
|
|
|
|
"nwords": nwords,
|
|
|
|
"cpu": cpu_wps,
|
|
|
|
"gpu": gpu_wps,
|
|
|
|
}
|
2020-01-16 00:58:51 +00:00
|
|
|
meta.setdefault("accuracy", {})
|
|
|
|
for component in nlp.pipe_names:
|
|
|
|
for metric in _get_metrics(component):
|
|
|
|
meta["accuracy"][metric] = scorer.scores[metric]
|
2019-03-15 14:18:28 +00:00
|
|
|
else:
|
|
|
|
meta.setdefault("beam_accuracy", {})
|
|
|
|
meta.setdefault("beam_speed", {})
|
2020-01-16 00:58:51 +00:00
|
|
|
for component in nlp.pipe_names:
|
|
|
|
for metric in _get_metrics(component):
|
|
|
|
meta["beam_accuracy"][metric] = scorer.scores[metric]
|
2019-03-16 14:55:13 +00:00
|
|
|
meta["beam_speed"][beam_width] = {
|
|
|
|
"nwords": nwords,
|
|
|
|
"cpu": cpu_wps,
|
|
|
|
"gpu": gpu_wps,
|
|
|
|
}
|
2019-03-15 14:18:28 +00:00
|
|
|
meta["vectors"] = {
|
|
|
|
"width": nlp.vocab.vectors_length,
|
|
|
|
"vectors": len(nlp.vocab.vectors),
|
|
|
|
"keys": nlp.vocab.vectors.n_keys,
|
2019-03-16 14:55:13 +00:00
|
|
|
"name": nlp.vocab.vectors.name,
|
|
|
|
}
|
2019-12-25 16:59:52 +00:00
|
|
|
meta.setdefault("name", f"model{i}")
|
2019-03-15 14:18:28 +00:00
|
|
|
meta.setdefault("version", version)
|
2019-09-18 22:56:07 +00:00
|
|
|
meta["labels"] = nlp.meta["labels"]
|
2019-12-25 16:59:52 +00:00
|
|
|
meta_loc = output_path / f"model{i}" / "meta.json"
|
2019-03-15 14:18:28 +00:00
|
|
|
srsly.write_json(meta_loc, meta)
|
|
|
|
util.set_env_log(verbose)
|
|
|
|
|
|
|
|
progress = _get_progress(
|
2019-03-16 15:02:47 +00:00
|
|
|
i,
|
|
|
|
losses,
|
|
|
|
scorer.scores,
|
2019-09-15 20:31:31 +00:00
|
|
|
output_stats,
|
2019-03-16 15:02:47 +00:00
|
|
|
beam_width=beam_width if has_beam_widths else None,
|
|
|
|
cpu_wps=cpu_wps,
|
|
|
|
gpu_wps=gpu_wps,
|
2019-03-15 14:18:28 +00:00
|
|
|
)
|
2019-09-15 20:31:31 +00:00
|
|
|
if i == 0 and "textcat" in pipeline:
|
|
|
|
textcats_per_cat = scorer.scores.get("textcats_per_cat", {})
|
|
|
|
for cat, cat_score in textcats_per_cat.items():
|
|
|
|
if cat_score.get("roc_auc_score", 0) < 0:
|
|
|
|
msg.warn(
|
2019-12-22 00:53:56 +00:00
|
|
|
f"Textcat ROC AUC score is undefined due to "
|
|
|
|
f"only one value in label '{cat}'."
|
2019-09-15 20:31:31 +00:00
|
|
|
)
|
2019-03-15 14:18:28 +00:00
|
|
|
msg.row(progress, **row_settings)
|
2019-04-22 12:31:11 +00:00
|
|
|
# Early stopping
|
2019-04-22 12:31:25 +00:00
|
|
|
if n_early_stopping is not None:
|
2019-04-15 10:04:36 +00:00
|
|
|
current_score = _score_for_model(meta)
|
|
|
|
if current_score < best_score:
|
|
|
|
iter_since_best += 1
|
|
|
|
else:
|
|
|
|
iter_since_best = 0
|
|
|
|
best_score = current_score
|
2019-04-22 12:31:25 +00:00
|
|
|
if iter_since_best >= n_early_stopping:
|
2019-04-22 12:31:11 +00:00
|
|
|
msg.text(
|
2019-12-22 00:53:56 +00:00
|
|
|
f"Early stopping, best iteration is: {i - iter_since_best}"
|
2019-04-22 12:31:11 +00:00
|
|
|
)
|
|
|
|
msg.text(
|
2019-12-22 00:53:56 +00:00
|
|
|
f"Best score = {best_score}; Final iteration score = {current_score}"
|
2019-04-22 12:31:11 +00:00
|
|
|
)
|
2019-04-15 10:04:36 +00:00
|
|
|
break
|
2020-02-15 14:50:17 +00:00
|
|
|
except Exception as e:
|
2020-02-18 14:38:18 +00:00
|
|
|
msg.warn(f"Aborting and saving final best model. Encountered exception: {e}")
|
2017-05-26 10:52:09 +00:00
|
|
|
finally:
|
2020-01-16 00:58:51 +00:00
|
|
|
best_pipes = nlp.pipe_names
|
|
|
|
if disabled_pipes:
|
|
|
|
disabled_pipes.restore()
|
2018-12-08 10:49:43 +00:00
|
|
|
with nlp.use_params(optimizer.averages):
|
|
|
|
final_model_path = output_path / "model-final"
|
|
|
|
nlp.to_disk(final_model_path)
|
2020-03-03 20:43:25 +00:00
|
|
|
meta_loc = output_path / "model-final" / "meta.json"
|
|
|
|
final_meta = srsly.read_json(meta_loc)
|
|
|
|
final_meta.setdefault("accuracy", {})
|
|
|
|
final_meta["accuracy"].update(meta.get("accuracy", {}))
|
|
|
|
final_meta.setdefault("speed", {})
|
|
|
|
final_meta["speed"].setdefault("cpu", None)
|
|
|
|
final_meta["speed"].setdefault("gpu", None)
|
2020-03-26 09:46:50 +00:00
|
|
|
meta.setdefault("speed", {})
|
|
|
|
meta["speed"].setdefault("cpu", None)
|
|
|
|
meta["speed"].setdefault("gpu", None)
|
2020-03-03 20:43:25 +00:00
|
|
|
# combine cpu and gpu speeds with the base model speeds
|
|
|
|
if final_meta["speed"]["cpu"] and meta["speed"]["cpu"]:
|
2020-03-25 11:28:12 +00:00
|
|
|
speed = _get_total_speed(
|
|
|
|
[final_meta["speed"]["cpu"], meta["speed"]["cpu"]]
|
|
|
|
)
|
2020-03-03 20:43:25 +00:00
|
|
|
final_meta["speed"]["cpu"] = speed
|
|
|
|
if final_meta["speed"]["gpu"] and meta["speed"]["gpu"]:
|
2020-03-25 11:28:12 +00:00
|
|
|
speed = _get_total_speed(
|
|
|
|
[final_meta["speed"]["gpu"], meta["speed"]["gpu"]]
|
|
|
|
)
|
2020-03-03 20:43:25 +00:00
|
|
|
final_meta["speed"]["gpu"] = speed
|
|
|
|
# if there were no speeds to update, overwrite with meta
|
2020-03-25 11:28:12 +00:00
|
|
|
if (
|
|
|
|
final_meta["speed"]["cpu"] is None
|
|
|
|
and final_meta["speed"]["gpu"] is None
|
|
|
|
):
|
2020-03-03 20:43:25 +00:00
|
|
|
final_meta["speed"].update(meta["speed"])
|
|
|
|
# note: beam speeds are not combined with the base model
|
|
|
|
if has_beam_widths:
|
|
|
|
final_meta.setdefault("beam_accuracy", {})
|
|
|
|
final_meta["beam_accuracy"].update(meta.get("beam_accuracy", {}))
|
|
|
|
final_meta.setdefault("beam_speed", {})
|
|
|
|
final_meta["beam_speed"].update(meta.get("beam_speed", {}))
|
|
|
|
srsly.write_json(meta_loc, final_meta)
|
2018-12-08 10:49:43 +00:00
|
|
|
msg.good("Saved model to output directory", final_model_path)
|
|
|
|
with msg.loading("Creating best model..."):
|
2020-01-16 00:58:51 +00:00
|
|
|
best_model_path = _collate_best_model(final_meta, output_path, best_pipes)
|
2018-12-08 10:49:43 +00:00
|
|
|
msg.good("Created best model", best_model_path)
|
2018-11-30 19:16:14 +00:00
|
|
|
|
2019-04-22 12:31:11 +00:00
|
|
|
|
2019-04-15 10:04:36 +00:00
|
|
|
def _score_for_model(meta):
|
|
|
|
""" Returns mean score between tasks in pipeline that can be used for early stopping. """
|
|
|
|
mean_acc = list()
|
2019-04-22 12:31:11 +00:00
|
|
|
pipes = meta["pipeline"]
|
|
|
|
acc = meta["accuracy"]
|
|
|
|
if "tagger" in pipes:
|
|
|
|
mean_acc.append(acc["tags_acc"])
|
2020-04-02 12:46:32 +00:00
|
|
|
if "morphologizer" in pipes:
|
|
|
|
mean_acc.append((acc["morphs_acc"] + acc["pos_acc"]) / 2)
|
2019-04-22 12:31:11 +00:00
|
|
|
if "parser" in pipes:
|
|
|
|
mean_acc.append((acc["uas"] + acc["las"]) / 2)
|
|
|
|
if "ner" in pipes:
|
|
|
|
mean_acc.append((acc["ents_p"] + acc["ents_r"] + acc["ents_f"]) / 3)
|
2019-09-15 20:31:31 +00:00
|
|
|
if "textcat" in pipes:
|
|
|
|
mean_acc.append(acc["textcat_score"])
|
2020-03-06 13:45:02 +00:00
|
|
|
if "senter" in pipes:
|
2020-01-08 15:52:14 +00:00
|
|
|
mean_acc.append((acc["sent_p"] + acc["sent_r"] + acc["sent_f"]) / 3)
|
2019-04-15 10:04:36 +00:00
|
|
|
return sum(mean_acc) / len(mean_acc)
|
2018-11-30 19:16:14 +00:00
|
|
|
|
2019-04-22 12:31:11 +00:00
|
|
|
|
2018-12-10 08:46:53 +00:00
|
|
|
@contextlib.contextmanager
|
|
|
|
def _create_progress_bar(total):
|
💫 Better support for semi-supervised learning (#3035)
The new spacy pretrain command implemented BERT/ULMFit/etc-like transfer learning, using our Language Modelling with Approximate Outputs version of BERT's cloze task. Pretraining is convenient, but in some ways it's a bit of a strange solution. All we're doing is initialising the weights. At the same time, we're putting a lot of work into our optimisation so that it's less sensitive to initial conditions, and more likely to find good optima. I discuss this a bit in the pseudo-rehearsal blog post: https://explosion.ai/blog/pseudo-rehearsal-catastrophic-forgetting
Support semi-supervised learning in spacy train
One obvious way to improve these pretraining methods is to do multi-task learning, instead of just transfer learning. This has been shown to work very well: https://arxiv.org/pdf/1809.08370.pdf . This patch makes it easy to do this sort of thing.
Add a new argument to spacy train, --raw-text. This takes a jsonl file with unlabelled data that can be used in arbitrary ways to do semi-supervised learning.
Add a new method to the Language class and to pipeline components, .rehearse(). This is like .update(), but doesn't expect GoldParse objects. It takes a batch of Doc objects, and performs an update on some semi-supervised objective.
Move the BERT-LMAO objective out from spacy/cli/pretrain.py into spacy/_ml.py, so we can create a new pipeline component, ClozeMultitask. This can be specified as a parser or NER multitask in the spacy train command. Example usage:
python -m spacy train en ./tmp ~/data/en-core-web/train/nw.json ~/data/en-core-web/dev/nw.json --pipeline parser --raw-textt ~/data/unlabelled/reddit-100k.jsonl --vectors en_vectors_web_lg --parser-multitasks cloze
Implement rehearsal methods for pipeline components
The new --raw-text argument and nlp.rehearse() method also gives us a good place to implement the the idea in the pseudo-rehearsal blog post in the parser. This works as follows:
Add a new nlp.resume_training() method. This allocates copies of pre-trained models in the pipeline, setting things up for the rehearsal updates. It also returns an optimizer object. This also greatly reduces confusion around the nlp.begin_training() method, which randomises the weights, making it not suitable for adding new labels or otherwise fine-tuning a pre-trained model.
Implement rehearsal updates on the Parser class, making it available for the dependency parser and NER. During rehearsal, the initial model is used to supervise the model being trained. The current model is asked to match the predictions of the initial model on some data. This minimises catastrophic forgetting, by keeping the model's predictions close to the original. See the blog post for details.
Implement rehearsal updates for tagger
Implement rehearsal updates for text categoriz
2018-12-10 15:25:33 +00:00
|
|
|
if int(os.environ.get("LOG_FRIENDLY", 0)):
|
2018-12-10 08:46:53 +00:00
|
|
|
yield
|
|
|
|
else:
|
|
|
|
pbar = tqdm.tqdm(total=total, leave=False)
|
|
|
|
yield pbar
|
|
|
|
|
|
|
|
|
2018-11-30 19:16:14 +00:00
|
|
|
def _load_vectors(nlp, vectors):
|
2020-02-27 17:42:27 +00:00
|
|
|
loaded_model = util.load_model(vectors, vocab=nlp.vocab)
|
2018-11-30 19:16:14 +00:00
|
|
|
for lex in nlp.vocab:
|
|
|
|
values = {}
|
|
|
|
for attr, func in nlp.vocab.lex_attr_getters.items():
|
|
|
|
# These attrs are expected to be set by data. Others should
|
|
|
|
# be set by calling the language functions.
|
|
|
|
if attr not in (CLUSTER, PROB, IS_OOV, LANG):
|
|
|
|
values[lex.vocab.strings[attr]] = func(lex.orth_)
|
|
|
|
lex.set_attrs(**values)
|
|
|
|
lex.is_oov = False
|
2020-02-27 17:42:27 +00:00
|
|
|
return loaded_model
|
2018-06-25 14:36:42 +00:00
|
|
|
|
2018-06-24 21:39:52 +00:00
|
|
|
|
2018-11-15 21:17:16 +00:00
|
|
|
def _load_pretrained_tok2vec(nlp, loc):
|
2019-10-02 08:37:39 +00:00
|
|
|
"""Load pretrained weights for the 'token-to-vector' part of the component
|
2018-11-15 21:17:16 +00:00
|
|
|
models, which is typically a CNN. See 'spacy pretrain'. Experimental.
|
|
|
|
"""
|
2018-11-30 19:16:14 +00:00
|
|
|
with loc.open("rb") as file_:
|
2018-11-15 21:17:16 +00:00
|
|
|
weights_data = file_.read()
|
|
|
|
loaded = []
|
|
|
|
for name, component in nlp.pipeline:
|
2020-02-27 17:42:27 +00:00
|
|
|
if hasattr(component, "model") and component.model.has_ref("tok2vec"):
|
|
|
|
component.get_ref("tok2vec").from_bytes(weights_data)
|
2018-11-15 21:17:16 +00:00
|
|
|
loaded.append(name)
|
|
|
|
return loaded
|
|
|
|
|
|
|
|
|
2018-06-24 21:39:52 +00:00
|
|
|
def _collate_best_model(meta, output_path, components):
|
|
|
|
bests = {}
|
2020-01-16 00:58:51 +00:00
|
|
|
meta.setdefault("accuracy", {})
|
2018-06-24 21:39:52 +00:00
|
|
|
for component in components:
|
|
|
|
bests[component] = _find_best(output_path, component)
|
2018-11-30 19:16:14 +00:00
|
|
|
best_dest = output_path / "model-best"
|
2019-12-22 00:53:56 +00:00
|
|
|
shutil.copytree(str(output_path / "model-final"), str(best_dest))
|
2018-06-24 21:39:52 +00:00
|
|
|
for component, best_component_src in bests.items():
|
2019-12-22 00:53:56 +00:00
|
|
|
shutil.rmtree(str(best_dest / component))
|
2020-02-18 13:47:23 +00:00
|
|
|
shutil.copytree(str(best_component_src / component), str(best_dest / component))
|
💫 Replace ujson, msgpack and dill/pickle/cloudpickle with srsly (#3003)
Remove hacks and wrappers, keep code in sync across our libraries and move spaCy a few steps closer to only depending on packages with binary wheels 🎉
See here: https://github.com/explosion/srsly
Serialization is hard, especially across Python versions and multiple platforms. After dealing with many subtle bugs over the years (encodings, locales, large files) our libraries like spaCy and Prodigy have steadily grown a number of utility functions to wrap the multiple serialization formats we need to support (especially json, msgpack and pickle). These wrapping functions ended up duplicated across our codebases, so we wanted to put them in one place.
At the same time, we noticed that having a lot of small dependencies was making maintainence harder, and making installation slower. To solve this, we've made srsly standalone, by including the component packages directly within it. This way we can provide all the serialization utilities we need in a single binary wheel.
srsly currently includes forks of the following packages:
ujson
msgpack
msgpack-numpy
cloudpickle
* WIP: replace json/ujson with srsly
* Replace ujson in examples
Use regular json instead of srsly to make code easier to read and follow
* Update requirements
* Fix imports
* Fix typos
* Replace msgpack with srsly
* Fix warning
2018-12-03 00:28:22 +00:00
|
|
|
accs = srsly.read_json(best_component_src / "accuracy.json")
|
2018-06-24 21:39:52 +00:00
|
|
|
for metric in _get_metrics(component):
|
2018-11-30 19:16:14 +00:00
|
|
|
meta["accuracy"][metric] = accs[metric]
|
💫 Replace ujson, msgpack and dill/pickle/cloudpickle with srsly (#3003)
Remove hacks and wrappers, keep code in sync across our libraries and move spaCy a few steps closer to only depending on packages with binary wheels 🎉
See here: https://github.com/explosion/srsly
Serialization is hard, especially across Python versions and multiple platforms. After dealing with many subtle bugs over the years (encodings, locales, large files) our libraries like spaCy and Prodigy have steadily grown a number of utility functions to wrap the multiple serialization formats we need to support (especially json, msgpack and pickle). These wrapping functions ended up duplicated across our codebases, so we wanted to put them in one place.
At the same time, we noticed that having a lot of small dependencies was making maintainence harder, and making installation slower. To solve this, we've made srsly standalone, by including the component packages directly within it. This way we can provide all the serialization utilities we need in a single binary wheel.
srsly currently includes forks of the following packages:
ujson
msgpack
msgpack-numpy
cloudpickle
* WIP: replace json/ujson with srsly
* Replace ujson in examples
Use regular json instead of srsly to make code easier to read and follow
* Update requirements
* Fix imports
* Fix typos
* Replace msgpack with srsly
* Fix warning
2018-12-03 00:28:22 +00:00
|
|
|
srsly.write_json(best_dest / "meta.json", meta)
|
2018-12-08 10:49:43 +00:00
|
|
|
return best_dest
|
2018-06-24 21:39:52 +00:00
|
|
|
|
|
|
|
|
|
|
|
def _find_best(experiment_dir, component):
|
|
|
|
accuracies = []
|
|
|
|
for epoch_model in experiment_dir.iterdir():
|
|
|
|
if epoch_model.is_dir() and epoch_model.parts[-1] != "model-final":
|
💫 Replace ujson, msgpack and dill/pickle/cloudpickle with srsly (#3003)
Remove hacks and wrappers, keep code in sync across our libraries and move spaCy a few steps closer to only depending on packages with binary wheels 🎉
See here: https://github.com/explosion/srsly
Serialization is hard, especially across Python versions and multiple platforms. After dealing with many subtle bugs over the years (encodings, locales, large files) our libraries like spaCy and Prodigy have steadily grown a number of utility functions to wrap the multiple serialization formats we need to support (especially json, msgpack and pickle). These wrapping functions ended up duplicated across our codebases, so we wanted to put them in one place.
At the same time, we noticed that having a lot of small dependencies was making maintainence harder, and making installation slower. To solve this, we've made srsly standalone, by including the component packages directly within it. This way we can provide all the serialization utilities we need in a single binary wheel.
srsly currently includes forks of the following packages:
ujson
msgpack
msgpack-numpy
cloudpickle
* WIP: replace json/ujson with srsly
* Replace ujson in examples
Use regular json instead of srsly to make code easier to read and follow
* Update requirements
* Fix imports
* Fix typos
* Replace msgpack with srsly
* Fix warning
2018-12-03 00:28:22 +00:00
|
|
|
accs = srsly.read_json(epoch_model / "accuracy.json")
|
2018-06-24 21:39:52 +00:00
|
|
|
scores = [accs.get(metric, 0.0) for metric in _get_metrics(component)]
|
2020-03-26 09:46:50 +00:00
|
|
|
# remove per_type dicts from score list for max() comparison
|
|
|
|
scores = [score for score in scores if isinstance(score, float)]
|
2018-06-24 21:39:52 +00:00
|
|
|
accuracies.append((scores, epoch_model))
|
|
|
|
if accuracies:
|
|
|
|
return max(accuracies)[1]
|
|
|
|
else:
|
|
|
|
return None
|
|
|
|
|
2018-11-30 19:16:14 +00:00
|
|
|
|
2018-06-24 21:39:52 +00:00
|
|
|
def _get_metrics(component):
|
|
|
|
if component == "parser":
|
2020-04-02 12:46:32 +00:00
|
|
|
return ("las", "uas", "las_per_type", "sent_f", "token_acc")
|
2018-06-24 21:39:52 +00:00
|
|
|
elif component == "tagger":
|
2020-03-03 20:43:25 +00:00
|
|
|
return ("tags_acc", "token_acc")
|
2020-04-02 12:46:32 +00:00
|
|
|
elif component == "morphologizer":
|
|
|
|
return ("morphs_acc", "pos_acc", "token_acc")
|
2018-06-24 21:39:52 +00:00
|
|
|
elif component == "ner":
|
2020-03-03 20:43:25 +00:00
|
|
|
return ("ents_f", "ents_p", "ents_r", "ents_per_type", "token_acc")
|
2020-03-06 13:45:02 +00:00
|
|
|
elif component == "senter":
|
2020-04-02 12:46:32 +00:00
|
|
|
return ("sent_f", "sent_p", "sent_r", "token_acc")
|
2020-01-16 00:58:51 +00:00
|
|
|
elif component == "textcat":
|
2020-03-03 20:43:25 +00:00
|
|
|
return ("textcat_score", "token_acc")
|
2018-06-24 21:39:52 +00:00
|
|
|
return ("token_acc",)
|
2017-05-19 23:15:50 +00:00
|
|
|
|
|
|
|
|
2019-09-15 20:31:31 +00:00
|
|
|
def _configure_training_output(pipeline, use_gpu, has_beam_widths):
|
|
|
|
row_head = ["Itn"]
|
|
|
|
output_stats = []
|
|
|
|
for pipe in pipeline:
|
|
|
|
if pipe == "tagger":
|
|
|
|
row_head.extend(["Tag Loss ", " Tag % "])
|
|
|
|
output_stats.extend(["tag_loss", "tags_acc"])
|
2020-04-02 12:46:32 +00:00
|
|
|
elif pipe == "morphologizer" or pipe == "morphologizertagger":
|
|
|
|
row_head.extend(["Morph Loss ", " Morph % ", " POS % "])
|
|
|
|
output_stats.extend(["morph_loss", "morphs_acc", "pos_acc"])
|
2019-09-15 20:31:31 +00:00
|
|
|
elif pipe == "parser":
|
2020-02-18 13:47:23 +00:00
|
|
|
row_head.extend(
|
|
|
|
["Dep Loss ", " UAS ", " LAS ", "Sent P", "Sent R", "Sent F"]
|
|
|
|
)
|
|
|
|
output_stats.extend(
|
|
|
|
["dep_loss", "uas", "las", "sent_p", "sent_r", "sent_f"]
|
|
|
|
)
|
2019-09-15 20:31:31 +00:00
|
|
|
elif pipe == "ner":
|
|
|
|
row_head.extend(["NER Loss ", "NER P ", "NER R ", "NER F "])
|
|
|
|
output_stats.extend(["ner_loss", "ents_p", "ents_r", "ents_f"])
|
|
|
|
elif pipe == "textcat":
|
|
|
|
row_head.extend(["Textcat Loss", "Textcat"])
|
|
|
|
output_stats.extend(["textcat_loss", "textcat_score"])
|
2020-03-06 13:45:02 +00:00
|
|
|
elif pipe == "senter":
|
|
|
|
row_head.extend(["Senter Loss", "Sent P", "Sent R", "Sent F"])
|
|
|
|
output_stats.extend(["senter_loss", "sent_p", "sent_r", "sent_f"])
|
2019-09-15 20:31:31 +00:00
|
|
|
row_head.extend(["Token %", "CPU WPS"])
|
|
|
|
output_stats.extend(["token_acc", "cpu_wps"])
|
|
|
|
|
|
|
|
if use_gpu >= 0:
|
|
|
|
row_head.extend(["GPU WPS"])
|
|
|
|
output_stats.extend(["gpu_wps"])
|
|
|
|
|
|
|
|
if has_beam_widths:
|
|
|
|
row_head.insert(1, "Beam W.")
|
2019-11-28 10:10:07 +00:00
|
|
|
# remove duplicates
|
2019-12-22 00:53:56 +00:00
|
|
|
row_head_dict = {k: 1 for k in row_head}
|
|
|
|
output_stats_dict = {k: 1 for k in output_stats}
|
2019-11-28 10:10:07 +00:00
|
|
|
return row_head_dict.keys(), output_stats_dict.keys()
|
2019-09-15 20:31:31 +00:00
|
|
|
|
|
|
|
|
|
|
|
def _get_progress(
|
|
|
|
itn, losses, dev_scores, output_stats, beam_width=None, cpu_wps=0.0, gpu_wps=0.0
|
|
|
|
):
|
2017-05-16 14:17:30 +00:00
|
|
|
scores = {}
|
2019-09-15 20:31:31 +00:00
|
|
|
for stat in output_stats:
|
|
|
|
scores[stat] = 0.0
|
2018-11-30 19:16:14 +00:00
|
|
|
scores["dep_loss"] = losses.get("parser", 0.0)
|
|
|
|
scores["ner_loss"] = losses.get("ner", 0.0)
|
|
|
|
scores["tag_loss"] = losses.get("tagger", 0.0)
|
2020-04-02 12:46:32 +00:00
|
|
|
scores["morph_loss"] = losses.get("morphologizer", 0.0)
|
2019-09-15 20:31:31 +00:00
|
|
|
scores["textcat_loss"] = losses.get("textcat", 0.0)
|
2020-03-06 13:45:02 +00:00
|
|
|
scores["senter_loss"] = losses.get("senter", 0.0)
|
2018-11-30 19:16:14 +00:00
|
|
|
scores["cpu_wps"] = cpu_wps
|
|
|
|
scores["gpu_wps"] = gpu_wps or 0.0
|
2019-09-15 20:31:31 +00:00
|
|
|
scores.update(dev_scores)
|
|
|
|
formatted_scores = []
|
|
|
|
for stat in output_stats:
|
|
|
|
format_spec = "{:.3f}"
|
|
|
|
if stat.endswith("_wps"):
|
|
|
|
format_spec = "{:.0f}"
|
|
|
|
formatted_scores.append(format_spec.format(scores[stat]))
|
|
|
|
result = [itn + 1]
|
|
|
|
result.extend(formatted_scores)
|
2019-03-16 15:02:47 +00:00
|
|
|
if beam_width is not None:
|
|
|
|
result.insert(1, beam_width)
|
|
|
|
return result
|
2020-03-03 20:43:25 +00:00
|
|
|
|
|
|
|
|
|
|
|
def _get_total_speed(speeds):
|
|
|
|
seconds_per_word = 0.0
|
|
|
|
for words_per_second in speeds:
|
|
|
|
if words_per_second is None:
|
|
|
|
return None
|
|
|
|
seconds_per_word += 1.0 / words_per_second
|
|
|
|
return 1.0 / seconds_per_word
|