diff --git a/extra/experiments/onto-joint/defaults.cfg b/extra/experiments/onto-joint/defaults.cfg
deleted file mode 100644
index 7954b57b5..000000000
--- a/extra/experiments/onto-joint/defaults.cfg
+++ /dev/null
@@ -1,133 +0,0 @@
-[paths]
-train = ""
-dev = ""
-raw = null
-init_tok2vec = null
-
-[system]
-seed = 0
-use_pytorch_for_gpu_memory = false
-
-[training]
-seed = ${system:seed}
-dropout = 0.1
-init_tok2vec = ${paths:init_tok2vec}
-vectors = null
-accumulate_gradient = 1
-max_steps = 0
-max_epochs = 0
-patience = 10000
-eval_frequency = 200
-score_weights = {"dep_las": 0.4, "ents_f": 0.4, "tag_acc": 0.2}
-frozen_components = []
-
-[training.train_corpus]
-@readers = "spacy.Corpus.v1"
-path = ${paths:train}
-gold_preproc = true
-max_length = 0
-limit = 0
-
-[training.dev_corpus]
-@readers = "spacy.Corpus.v1"
-path = ${paths:dev}
-gold_preproc = ${training.read_train:gold_preproc}
-max_length = 0
-limit = 0
-
-[training.batcher]
-@batchers = "spacy.batch_by_words.v1"
-discard_oversize = false
-tolerance = 0.2
-
-[training.batcher.size]
-@schedules = "compounding.v1"
-start = 100
-stop = 1000
-compound = 1.001
-
-[training.optimizer]
-@optimizers = "Adam.v1"
-beta1 = 0.9
-beta2 = 0.999
-L2_is_weight_decay = true
-L2 = 0.01
-grad_clip = 1.0
-use_averages = false
-eps = 1e-8
-learn_rate = 0.001
-
-[nlp]
-lang = "en"
-load_vocab_data = false
-pipeline = ["tok2vec", "ner", "tagger", "parser"]
-
-[nlp.tokenizer]
-@tokenizers = "spacy.Tokenizer.v1"
-
-[nlp.lemmatizer]
-@lemmatizers = "spacy.Lemmatizer.v1"
-
-[components]
-
-[components.tok2vec]
-factory = "tok2vec"
-
-[components.ner]
-factory = "ner"
-learn_tokens = false
-min_action_freq = 1
-
-[components.tagger]
-factory = "tagger"
-
-[components.parser]
-factory = "parser"
-learn_tokens = false
-min_action_freq = 30
-
-[components.tagger.model]
-@architectures = "spacy.Tagger.v1"
-
-[components.tagger.model.tok2vec]
-@architectures = "spacy.Tok2VecListener.v1"
-width = ${components.tok2vec.model.encode:width}
-
-[components.parser.model]
-@architectures = "spacy.TransitionBasedParser.v1"
-nr_feature_tokens = 8
-hidden_width = 128
-maxout_pieces = 2
-use_upper = true
-
-[components.parser.model.tok2vec]
-@architectures = "spacy.Tok2VecListener.v1"
-width = ${components.tok2vec.model.encode:width}
-
-[components.ner.model]
-@architectures = "spacy.TransitionBasedParser.v1"
-nr_feature_tokens = 3
-hidden_width = 128
-maxout_pieces = 2
-use_upper = true
-
-[components.ner.model.tok2vec]
-@architectures = "spacy.Tok2VecListener.v1"
-width = ${components.tok2vec.model.encode:width}
-
-[components.tok2vec.model]
-@architectures = "spacy.Tok2Vec.v1"
-
-[components.tok2vec.model.embed]
-@architectures = "spacy.MultiHashEmbed.v1"
-width = ${components.tok2vec.model.encode:width}
-rows = 2000
-also_embed_subwords = true
-also_use_static_vectors = false
-
-[components.tok2vec.model.encode]
-@architectures = "spacy.MaxoutWindowEncoder.v1"
-width = 96
-depth = 4
-window_size = 1
-maxout_pieces = 3
diff --git a/extra/experiments/onto-joint/pretrain.cfg b/extra/experiments/onto-joint/pretrain.cfg
deleted file mode 100644
index 211339603..000000000
--- a/extra/experiments/onto-joint/pretrain.cfg
+++ /dev/null
@@ -1,152 +0,0 @@
-# Training hyper-parameters and additional features.
-[training]
-# Whether to train on sequences with 'gold standard' sentence boundaries
-# and tokens. If you set this to true, take care to ensure your run-time
-# data is passed in sentence-by-sentence via some prior preprocessing.
-gold_preproc = false
-# Limitations on training document length or number of examples.
-max_length = 0
-limit = 0
-# Data augmentation
-orth_variant_level = 0.0
-dropout = 0.1
-# Controls early-stopping. 0 or -1 mean unlimited.
-patience = 1600
-max_epochs = 0
-max_steps = 20000
-eval_frequency = 400
-# Other settings
-seed = 0
-accumulate_gradient = 1
-use_pytorch_for_gpu_memory = false
-# Control how scores are printed and checkpoints are evaluated.
-scores = ["speed", "tags_acc", "uas", "las", "ents_f"]
-score_weights = {"las": 0.4, "ents_f": 0.4, "tags_acc": 0.2}
-# These settings are invalid for the transformer models.
-init_tok2vec = null
-discard_oversize = false
-omit_extra_lookups = false
-batch_by = "words"
-use_gpu = -1
-raw_text = null
-tag_map = null
-
-[training.batch_size]
-@schedules = "compounding.v1"
-start = 1000
-stop = 1000
-compound = 1.001
-
-[training.optimizer]
-@optimizers = "Adam.v1"
-beta1 = 0.9
-beta2 = 0.999
-L2_is_weight_decay = true
-L2 = 0.01
-grad_clip = 1.0
-use_averages = true
-eps = 1e-8
-learn_rate = 0.001
-
-[pretraining]
-max_epochs = 1000
-min_length = 5
-max_length = 500
-dropout = 0.2
-n_save_every = null
-batch_size = 3000
-seed = ${training:seed}
-use_pytorch_for_gpu_memory = ${training:use_pytorch_for_gpu_memory}
-tok2vec_model = "nlp.pipeline.tok2vec.model"
-
-[pretraining.objective]
-type = "characters"
-n_characters = 4
-
-[pretraining.optimizer]
-@optimizers = "Adam.v1"
-beta1 = 0.9
-beta2 = 0.999
-L2_is_weight_decay = true
-L2 = 0.01
-grad_clip = 1.0
-use_averages = true
-eps = 1e-8
-learn_rate = 0.001
-
-[nlp]
-lang = "en"
-vectors = null
-base_model = null
-
-[nlp.pipeline]
-
-[nlp.pipeline.tok2vec]
-factory = "tok2vec"
-
-[nlp.pipeline.senter]
-factory = "senter"
-
-[nlp.pipeline.ner]
-factory = "ner"
-learn_tokens = false
-min_action_freq = 1
-beam_width = 1
-beam_update_prob = 1.0
-
-[nlp.pipeline.tagger]
-factory = "tagger"
-
-[nlp.pipeline.parser]
-factory = "parser"
-learn_tokens = false
-min_action_freq = 1
-beam_width = 1
-beam_update_prob = 1.0
-
-[nlp.pipeline.senter.model]
-@architectures = "spacy.Tagger.v1"
-
-[nlp.pipeline.senter.model.tok2vec]
-@architectures = "spacy.Tok2VecTensors.v1"
-width = ${nlp.pipeline.tok2vec.model:width}
-
-[nlp.pipeline.tagger.model]
-@architectures = "spacy.Tagger.v1"
-
-[nlp.pipeline.tagger.model.tok2vec]
-@architectures = "spacy.Tok2VecTensors.v1"
-width = ${nlp.pipeline.tok2vec.model:width}
-
-[nlp.pipeline.parser.model]
-@architectures = "spacy.TransitionBasedParser.v1"
-nr_feature_tokens = 8
-hidden_width = 128
-maxout_pieces = 3
-use_upper = false
-
-[nlp.pipeline.parser.model.tok2vec]
-@architectures = "spacy.Tok2VecTensors.v1"
-width = ${nlp.pipeline.tok2vec.model:width}
-
-[nlp.pipeline.ner.model]
-@architectures = "spacy.TransitionBasedParser.v1"
-nr_feature_tokens = 3
-hidden_width = 128
-maxout_pieces = 3
-use_upper = false
-
-[nlp.pipeline.ner.model.tok2vec]
-@architectures = "spacy.Tok2VecTensors.v1"
-width = ${nlp.pipeline.tok2vec.model:width}
-
-[nlp.pipeline.tok2vec.model]
-@architectures = "spacy.HashEmbedCNN.v1"
-pretrained_vectors = ${nlp:vectors}
-width = 256
-depth = 6
-window_size = 1
-embed_size = 10000
-maxout_pieces = 3
-subword_features = true
-dropout = null
diff --git a/extra/experiments/onto-ner.cfg b/extra/experiments/onto-ner.cfg
deleted file mode 100644
index eab68a27f..000000000
--- a/extra/experiments/onto-ner.cfg
+++ /dev/null
@@ -1,73 +0,0 @@
-# Training hyper-parameters and additional features.
-[training]
-# Whether to train on sequences with 'gold standard' sentence boundaries
-# and tokens. If you set this to true, take care to ensure your run-time
-# data is passed in sentence-by-sentence via some prior preprocessing.
-gold_preproc = false
-# Limitations on training document length or number of examples.
-max_length = 3000
-limit = 0
-# Data augmentation
-orth_variant_level = 0.0
-dropout = 0.1
-# Controls early-stopping. 0 or -1 mean unlimited.
-patience = 100000
-max_epochs = 0
-max_steps = 0
-eval_frequency = 1000
-# Other settings
-seed = 0
-accumulate_gradient = 1
-use_pytorch_for_gpu_memory = false
-# Control how scores are printed and checkpoints are evaluated.
-scores = ["speed", "ents_p", "ents_r", "ents_f"]
-score_weights = {"ents_f": 1.0}
-# These settings are invalid for the transformer models.
-init_tok2vec = null
-discard_oversize = false
-omit_extra_lookups = false
-batch_by = "words"
-
-[training.batch_size]
-@schedules = "compounding.v1"
-start = 100
-stop = 1000
-compound = 1.001
-
-[training.optimizer]
-@optimizers = "Adam.v1"
-beta1 = 0.9
-beta2 = 0.999
-L2_is_weight_decay = true
-L2 = 0.01
-grad_clip = 1.0
-use_averages = true
-eps = 1e-8
-learn_rate = 0.001
-
-[nlp]
-lang = "en"
-vectors = null
-
-[nlp.pipeline.ner]
-factory = "ner"
-learn_tokens = false
-min_action_freq = 1
-
-[nlp.pipeline.ner.model]
-@architectures = "spacy.TransitionBasedParser.v1"
-nr_feature_tokens = 3
-hidden_width = 64
-maxout_pieces = 2
-use_upper = true
-
-[nlp.pipeline.ner.model.tok2vec]
-@architectures = "spacy.HashEmbedCNN.v1"
-pretrained_vectors = ${nlp:vectors}
-width = 96
-depth = 4
-window_size = 1
-embed_size = 2000
-maxout_pieces = 3
-subword_features = true
-dropout = ${training:dropout}
diff --git a/extra/experiments/ptb-joint-pos-dep/bilstm_tok2vec.cfg b/extra/experiments/ptb-joint-pos-dep/bilstm_tok2vec.cfg
deleted file mode 100644
index f1b702a4e..000000000
--- a/extra/experiments/ptb-joint-pos-dep/bilstm_tok2vec.cfg
+++ /dev/null
@@ -1,73 +0,0 @@
-[training]
-patience = 10000
-eval_frequency = 200
-dropout = 0.2
-init_tok2vec = null
-vectors = null
-max_epochs = 100
-orth_variant_level = 0.0
-gold_preproc = true
-max_length = 0
-use_gpu = 0
-scores = ["tags_acc", "uas", "las"]
-score_weights = {"las": 0.8, "tags_acc": 0.2}
-limit = 0
-seed = 0
-accumulate_gradient = 2
-discard_oversize = false
-
-[training.batch_size]
-@schedules = "compounding.v1"
-start = 100
-stop = 1000
-compound = 1.001
-
-[training.optimizer]
-@optimizers = "Adam.v1"
-learn_rate = 0.001
-beta1 = 0.9
-beta2 = 0.999
-
-[nlp]
-lang = "en"
-vectors = ${training:vectors}
-
-[nlp.pipeline.tok2vec]
-factory = "tok2vec"
-
-[nlp.pipeline.tagger]
-factory = "tagger"
-
-[nlp.pipeline.parser]
-factory = "parser"
-learn_tokens = false
-min_action_freq = 1
-beam_width = 1
-beam_update_prob = 1.0
-
-[nlp.pipeline.tagger.model]
-@architectures = "spacy.Tagger.v1"
-
-[nlp.pipeline.tagger.model.tok2vec]
-@architectures = "spacy.Tok2VecTensors.v1"
-width = ${nlp.pipeline.tok2vec.model:width}
-
-[nlp.pipeline.parser.model]
-@architectures = "spacy.TransitionBasedParser.v1"
-nr_feature_tokens = 8
-hidden_width = 64
-maxout_pieces = 3
-
-[nlp.pipeline.parser.model.tok2vec]
-@architectures = "spacy.Tok2VecTensors.v1"
-width = ${nlp.pipeline.tok2vec.model:width}
-
-[nlp.pipeline.tok2vec.model]
-@architectures = "spacy.HashEmbedBiLSTM.v1"
-pretrained_vectors = ${nlp:vectors}
-width = 96
-depth = 4
-embed_size = 2000
-subword_features = true
-maxout_pieces = 3
-dropout = null
diff --git a/extra/experiments/ptb-joint-pos-dep/defaults.cfg b/extra/experiments/ptb-joint-pos-dep/defaults.cfg
deleted file mode 100644
index 8f9c5666e..000000000
--- a/extra/experiments/ptb-joint-pos-dep/defaults.cfg
+++ /dev/null
@@ -1,110 +0,0 @@
-[paths]
-train = ""
-dev = ""
-raw = null
-init_tok2vec = null
-
-[system]
-seed = 0
-use_pytorch_for_gpu_memory = false
-
-[training]
-seed = ${system:seed}
-dropout = 0.2
-init_tok2vec = ${paths:init_tok2vec}
-vectors = null
-accumulate_gradient = 1
-max_steps = 0
-max_epochs = 0
-patience = 10000
-eval_frequency = 200
-score_weights = {"dep_las": 0.8, "tag_acc": 0.2}
-
-[training.read_train]
-@readers = "spacy.Corpus.v1"
-path = ${paths:train}
-gold_preproc = true
-max_length = 0
-limit = 0
-
-[training.read_dev]
-@readers = "spacy.Corpus.v1"
-path = ${paths:dev}
-gold_preproc = ${training.read_train:gold_preproc}
-max_length = 0
-limit = 0
-
-[training.batcher]
-@batchers = "spacy.batch_by_words.v1"
-discard_oversize = false
-tolerance = 0.2
-
-[training.batcher.size]
-@schedules = "compounding.v1"
-start = 100
-stop = 1000
-compound = 1.001
-
-[training.optimizer]
-@optimizers = "Adam.v1"
-learn_rate = 0.001
-beta1 = 0.9
-beta2 = 0.999
-
-[nlp]
-lang = "en"
-pipeline = ["tok2vec", "tagger", "parser"]
-load_vocab_data = false
-
-[nlp.tokenizer]
-@tokenizers = "spacy.Tokenizer.v1"
-
-[nlp.lemmatizer]
-@lemmatizers = "spacy.Lemmatizer.v1"
-
-[components]
-
-[components.tok2vec]
-factory = "tok2vec"
-
-[components.tagger]
-factory = "tagger"
-
-[components.parser]
-factory = "parser"
-learn_tokens = false
-min_action_freq = 1
-
-[components.tagger.model]
-@architectures = "spacy.Tagger.v1"
-
-[components.tagger.model.tok2vec]
-@architectures = "spacy.Tok2VecListener.v1"
-width = ${components.tok2vec.model.encode:width}
-
-[components.parser.model]
-@architectures = "spacy.TransitionBasedParser.v1"
-nr_feature_tokens = 8
-hidden_width = 64
-maxout_pieces = 3
-
-[components.parser.model.tok2vec]
-@architectures = "spacy.Tok2VecListener.v1"
-width = ${components.tok2vec.model.encode:width}
-
-[components.tok2vec.model]
-@architectures = "spacy.Tok2Vec.v1"
-
-[components.tok2vec.model.embed]
-@architectures = "spacy.MultiHashEmbed.v1"
-width = ${components.tok2vec.model.encode:width}
-rows = 2000
-also_embed_subwords = true
-also_use_static_vectors = false
-
-[components.tok2vec.model.encode]
-@architectures = "spacy.MaxoutWindowEncoder.v1"
-width = 96
-depth = 4
-window_size = 1
-maxout_pieces = 3
diff --git a/extra/experiments/tok2vec-ner/charembed_tok2vec.cfg b/extra/experiments/tok2vec-ner/charembed_tok2vec.cfg
deleted file mode 100644
index eca6a22fa..000000000
--- a/extra/experiments/tok2vec-ner/charembed_tok2vec.cfg
+++ /dev/null
@@ -1,69 +0,0 @@
-[training]
-use_gpu = -1
-limit = 0
-dropout = 0.2
-patience = 10000
-eval_frequency = 200
-scores = ["ents_f"]
-score_weights = {"ents_f": 1}
-orth_variant_level = 0.0
-gold_preproc = true
-max_length = 0
-batch_size = 25
-seed = 0
-accumulate_gradient = 2
-discard_oversize = false
-
-[training.optimizer]
-@optimizers = "Adam.v1"
-learn_rate = 0.001
-beta1 = 0.9
-beta2 = 0.999
-
-[nlp]
-lang = "en"
-vectors = null
-
-[nlp.pipeline.tok2vec]
-factory = "tok2vec"
-
-[nlp.pipeline.tok2vec.model]
-@architectures = "spacy.Tok2Vec.v1"
-
-[nlp.pipeline.tok2vec.model.extract]
-@architectures = "spacy.CharacterEmbed.v1"
-width = 96
-nM = 64
-nC = 8
-rows = 2000
-columns = ["ID", "NORM", "PREFIX", "SUFFIX", "SHAPE", "ORTH"]
-dropout = null
-
-[nlp.pipeline.tok2vec.model.extract.features]
-@architectures = "spacy.Doc2Feats.v1"
-columns = ${nlp.pipeline.tok2vec.model.extract:columns}
-
-[nlp.pipeline.tok2vec.model.embed]
-@architectures = "spacy.LayerNormalizedMaxout.v1"
-width = ${nlp.pipeline.tok2vec.model.extract:width}
-maxout_pieces = 4
-
-[nlp.pipeline.tok2vec.model.encode]
-@architectures = "spacy.MaxoutWindowEncoder.v1"
-width = ${nlp.pipeline.tok2vec.model.extract:width}
-window_size = 1
-maxout_pieces = 2
-depth = 2
-
-[nlp.pipeline.ner]
-factory = "ner"
-
-[nlp.pipeline.ner.model]
-@architectures = "spacy.TransitionBasedParser.v1"
-nr_feature_tokens = 6
-hidden_width = 64
-maxout_pieces = 2
-
-[nlp.pipeline.ner.model.tok2vec]
-@architectures = "spacy.Tok2VecTensors.v1"
-width = ${nlp.pipeline.tok2vec.model.extract:width}
diff --git a/extra/experiments/tok2vec-ner/multihashembed_tok2vec.cfg b/extra/experiments/tok2vec-ner/multihashembed_tok2vec.cfg
deleted file mode 100644
index e2ab148c6..000000000
--- a/extra/experiments/tok2vec-ner/multihashembed_tok2vec.cfg
+++ /dev/null
@@ -1,51 +0,0 @@
-[training]
-use_gpu = -1
-limit = 0
-dropout = 0.2
-patience = 10000
-eval_frequency = 200
-scores = ["ents_p", "ents_r", "ents_f"]
-score_weights = {"ents_f": 1}
-orth_variant_level = 0.0
-gold_preproc = true
-max_length = 0
-seed = 0
-accumulate_gradient = 2
-discard_oversize = false
-
-[training.batch_size]
-@schedules = "compounding.v1"
-start = 3000
-stop = 3000
-compound = 1.001
-
-
-[training.optimizer]
-@optimizers = "Adam.v1"
-learn_rate = 0.001
-beta1 = 0.9
-beta2 = 0.999
-
-[nlp]
-lang = "en"
-vectors = null
-
-[nlp.pipeline.ner]
-factory = "ner"
-
-[nlp.pipeline.ner.model]
-@architectures = "spacy.TransitionBasedParser.v1"
-nr_feature_tokens = 6
-hidden_width = 64
-maxout_pieces = 2
-
-[nlp.pipeline.ner.model.tok2vec]
-@architectures = "spacy.HashEmbedCNN.v1"
-width = 128
-depth = 4
-embed_size = 7000
-maxout_pieces = 3
-window_size = 1
-subword_features = true
-pretrained_vectors = null
-dropout = null