2020-04-10 18:34:23 +00:00
|
|
|
import pytest
|
|
|
|
import torch
|
2020-04-22 00:33:10 +00:00
|
|
|
|
2020-04-10 18:34:23 +00:00
|
|
|
from pytorch_lightning import Trainer
|
|
|
|
from pytorch_lightning.utilities.exceptions import MisconfigurationException
|
2020-05-04 20:51:39 +00:00
|
|
|
from tests.base import EvalModelTemplate
|
2020-04-10 18:34:23 +00:00
|
|
|
|
|
|
|
|
|
|
|
def test_error_on_more_than_1_optimizer(tmpdir):
|
2020-05-02 12:38:22 +00:00
|
|
|
""" Check that error is thrown when more than 1 optimizer is passed """
|
2020-04-10 18:34:23 +00:00
|
|
|
|
2020-05-10 17:15:28 +00:00
|
|
|
model = EvalModelTemplate()
|
2020-05-04 20:51:39 +00:00
|
|
|
model.configure_optimizers = model.configure_optimizers__multiple_schedulers
|
2020-04-10 18:34:23 +00:00
|
|
|
|
|
|
|
# logger file to get meta
|
|
|
|
trainer = Trainer(
|
2020-06-12 18:37:52 +00:00
|
|
|
default_root_dir=tmpdir,
|
2020-04-10 18:34:23 +00:00
|
|
|
max_epochs=1
|
|
|
|
)
|
|
|
|
|
|
|
|
with pytest.raises(MisconfigurationException):
|
|
|
|
trainer.lr_find(model)
|
|
|
|
|
|
|
|
|
|
|
|
def test_model_reset_correctly(tmpdir):
|
2020-05-02 12:38:22 +00:00
|
|
|
""" Check that model weights are correctly reset after lr_find() """
|
2020-04-10 18:34:23 +00:00
|
|
|
|
2020-05-10 17:15:28 +00:00
|
|
|
model = EvalModelTemplate()
|
2020-04-10 18:34:23 +00:00
|
|
|
|
|
|
|
# logger file to get meta
|
|
|
|
trainer = Trainer(
|
2020-06-12 18:37:52 +00:00
|
|
|
default_root_dir=tmpdir,
|
2020-04-10 18:34:23 +00:00
|
|
|
max_epochs=1
|
|
|
|
)
|
|
|
|
|
|
|
|
before_state_dict = model.state_dict()
|
|
|
|
|
|
|
|
_ = trainer.lr_find(model, num_training=5)
|
|
|
|
|
|
|
|
after_state_dict = model.state_dict()
|
|
|
|
|
|
|
|
for key in before_state_dict.keys():
|
|
|
|
assert torch.all(torch.eq(before_state_dict[key], after_state_dict[key])), \
|
|
|
|
'Model was not reset correctly after learning rate finder'
|
|
|
|
|
|
|
|
|
|
|
|
def test_trainer_reset_correctly(tmpdir):
|
2020-05-02 12:38:22 +00:00
|
|
|
""" Check that all trainer parameters are reset correctly after lr_find() """
|
2020-04-10 18:34:23 +00:00
|
|
|
|
2020-05-10 17:15:28 +00:00
|
|
|
model = EvalModelTemplate()
|
2020-04-10 18:34:23 +00:00
|
|
|
|
|
|
|
# logger file to get meta
|
|
|
|
trainer = Trainer(
|
2020-06-12 18:37:52 +00:00
|
|
|
default_root_dir=tmpdir,
|
2020-04-10 18:34:23 +00:00
|
|
|
max_epochs=1
|
|
|
|
)
|
|
|
|
|
|
|
|
changed_attributes = ['callbacks', 'logger', 'max_steps', 'auto_lr_find',
|
2020-05-25 11:49:23 +00:00
|
|
|
'early_stop_callback', 'accumulate_grad_batches',
|
|
|
|
'enable_early_stop', 'checkpoint_callback']
|
2020-04-10 18:34:23 +00:00
|
|
|
attributes_before = {}
|
|
|
|
for ca in changed_attributes:
|
|
|
|
attributes_before[ca] = getattr(trainer, ca)
|
|
|
|
|
|
|
|
_ = trainer.lr_find(model, num_training=5)
|
|
|
|
|
|
|
|
attributes_after = {}
|
|
|
|
for ca in changed_attributes:
|
|
|
|
attributes_after[ca] = getattr(trainer, ca)
|
|
|
|
|
|
|
|
for key in changed_attributes:
|
|
|
|
assert attributes_before[key] == attributes_after[key], \
|
|
|
|
f'Attribute {key} was not reset correctly after learning rate finder'
|
|
|
|
|
|
|
|
|
|
|
|
def test_trainer_arg_bool(tmpdir):
|
2020-05-13 18:40:44 +00:00
|
|
|
""" Test that setting trainer arg to bool works """
|
2020-05-10 17:15:28 +00:00
|
|
|
hparams = EvalModelTemplate.get_default_hparams()
|
2020-05-24 22:59:08 +00:00
|
|
|
model = EvalModelTemplate(**hparams)
|
|
|
|
before_lr = hparams.get('learning_rate')
|
2020-05-04 20:51:39 +00:00
|
|
|
|
2020-04-10 18:34:23 +00:00
|
|
|
# logger file to get meta
|
|
|
|
trainer = Trainer(
|
2020-06-12 18:37:52 +00:00
|
|
|
default_root_dir=tmpdir,
|
2020-06-01 15:00:32 +00:00
|
|
|
max_epochs=2,
|
2020-04-10 18:34:23 +00:00
|
|
|
auto_lr_find=True
|
|
|
|
)
|
|
|
|
|
|
|
|
trainer.fit(model)
|
2020-05-24 22:59:08 +00:00
|
|
|
after_lr = model.learning_rate
|
2020-04-10 18:34:23 +00:00
|
|
|
assert before_lr != after_lr, \
|
|
|
|
'Learning rate was not altered after running learning rate finder'
|
|
|
|
|
|
|
|
|
|
|
|
def test_trainer_arg_str(tmpdir):
|
2020-05-13 18:40:44 +00:00
|
|
|
""" Test that setting trainer arg to string works """
|
2020-05-24 22:59:08 +00:00
|
|
|
model = EvalModelTemplate()
|
|
|
|
model.my_fancy_lr = 1.0 # update with non-standard field
|
2020-05-04 20:51:39 +00:00
|
|
|
|
2020-05-24 22:59:08 +00:00
|
|
|
before_lr = model.my_fancy_lr
|
2020-04-10 18:34:23 +00:00
|
|
|
# logger file to get meta
|
|
|
|
trainer = Trainer(
|
2020-06-12 18:37:52 +00:00
|
|
|
default_root_dir=tmpdir,
|
2020-06-01 15:00:32 +00:00
|
|
|
max_epochs=2,
|
2020-04-10 18:34:23 +00:00
|
|
|
auto_lr_find='my_fancy_lr'
|
|
|
|
)
|
|
|
|
|
|
|
|
trainer.fit(model)
|
2020-05-24 22:59:08 +00:00
|
|
|
after_lr = model.my_fancy_lr
|
2020-04-10 18:34:23 +00:00
|
|
|
assert before_lr != after_lr, \
|
|
|
|
'Learning rate was not altered after running learning rate finder'
|
|
|
|
|
|
|
|
|
|
|
|
def test_call_to_trainer_method(tmpdir):
|
2020-05-13 18:40:44 +00:00
|
|
|
""" Test that directly calling the trainer method works """
|
2020-04-10 18:34:23 +00:00
|
|
|
|
2020-05-10 17:15:28 +00:00
|
|
|
hparams = EvalModelTemplate.get_default_hparams()
|
2020-05-24 22:59:08 +00:00
|
|
|
model = EvalModelTemplate(**hparams)
|
2020-05-04 20:51:39 +00:00
|
|
|
|
2020-05-24 22:59:08 +00:00
|
|
|
before_lr = hparams.get('learning_rate')
|
2020-04-10 18:34:23 +00:00
|
|
|
# logger file to get meta
|
|
|
|
trainer = Trainer(
|
2020-06-12 18:37:52 +00:00
|
|
|
default_root_dir=tmpdir,
|
2020-06-01 15:00:32 +00:00
|
|
|
max_epochs=2,
|
2020-04-10 18:34:23 +00:00
|
|
|
)
|
|
|
|
|
|
|
|
lrfinder = trainer.lr_find(model, mode='linear')
|
|
|
|
after_lr = lrfinder.suggestion()
|
2020-05-24 22:59:08 +00:00
|
|
|
model.learning_rate = after_lr
|
2020-04-10 18:34:23 +00:00
|
|
|
trainer.fit(model)
|
|
|
|
|
|
|
|
assert before_lr != after_lr, \
|
|
|
|
'Learning rate was not altered after running learning rate finder'
|
2020-05-13 18:40:44 +00:00
|
|
|
|
|
|
|
|
2020-06-03 12:28:19 +00:00
|
|
|
@pytest.mark.skip('TODO: speed up this test')
|
2020-05-13 18:40:44 +00:00
|
|
|
def test_accumulation_and_early_stopping(tmpdir):
|
|
|
|
""" Test that early stopping of learning rate finder works, and that
|
|
|
|
accumulation also works for this feature """
|
|
|
|
|
|
|
|
hparams = EvalModelTemplate.get_default_hparams()
|
2020-05-24 22:59:08 +00:00
|
|
|
model = EvalModelTemplate(**hparams)
|
2020-05-13 18:40:44 +00:00
|
|
|
|
2020-05-24 22:59:08 +00:00
|
|
|
before_lr = hparams.get('learning_rate')
|
2020-05-13 18:40:44 +00:00
|
|
|
# logger file to get meta
|
|
|
|
trainer = Trainer(
|
2020-06-12 18:37:52 +00:00
|
|
|
default_root_dir=tmpdir,
|
2020-06-01 15:00:32 +00:00
|
|
|
accumulate_grad_batches=2,
|
2020-05-13 18:40:44 +00:00
|
|
|
)
|
|
|
|
|
|
|
|
lrfinder = trainer.lr_find(model, early_stop_threshold=None)
|
|
|
|
after_lr = lrfinder.suggestion()
|
|
|
|
|
|
|
|
assert before_lr != after_lr, \
|
|
|
|
'Learning rate was not altered after running learning rate finder'
|
|
|
|
assert len(lrfinder.results['lr']) == 100, \
|
|
|
|
'Early stopping for learning rate finder did not work'
|
|
|
|
assert lrfinder._total_batch_idx == 100 * 2, \
|
|
|
|
'Accumulation parameter did not work'
|
|
|
|
|
|
|
|
|
|
|
|
def test_suggestion_parameters_work(tmpdir):
|
|
|
|
""" Test that default skipping does not alter results in basic case """
|
|
|
|
|
|
|
|
hparams = EvalModelTemplate.get_default_hparams()
|
2020-05-24 22:59:08 +00:00
|
|
|
model = EvalModelTemplate(**hparams)
|
2020-05-13 18:40:44 +00:00
|
|
|
|
|
|
|
# logger file to get meta
|
|
|
|
trainer = Trainer(
|
2020-06-12 18:37:52 +00:00
|
|
|
default_root_dir=tmpdir,
|
2020-06-01 15:00:32 +00:00
|
|
|
max_epochs=3,
|
2020-05-13 18:40:44 +00:00
|
|
|
)
|
|
|
|
|
|
|
|
lrfinder = trainer.lr_find(model)
|
|
|
|
lr1 = lrfinder.suggestion(skip_begin=10) # default
|
|
|
|
lr2 = lrfinder.suggestion(skip_begin=80) # way too high, should have an impact
|
|
|
|
|
|
|
|
assert lr1 != lr2, \
|
|
|
|
'Skipping parameter did not influence learning rate'
|
2020-05-19 06:39:19 +00:00
|
|
|
|
|
|
|
|
|
|
|
def test_suggestion_with_non_finite_values(tmpdir):
|
|
|
|
""" Test that non-finite values does not alter results """
|
|
|
|
|
|
|
|
hparams = EvalModelTemplate.get_default_hparams()
|
|
|
|
model = EvalModelTemplate(hparams)
|
|
|
|
|
|
|
|
# logger file to get meta
|
|
|
|
trainer = Trainer(
|
2020-06-12 18:37:52 +00:00
|
|
|
default_root_dir=tmpdir,
|
2020-06-01 15:00:32 +00:00
|
|
|
max_epochs=3
|
2020-05-19 06:39:19 +00:00
|
|
|
)
|
|
|
|
|
|
|
|
lrfinder = trainer.lr_find(model)
|
|
|
|
before_lr = lrfinder.suggestion()
|
|
|
|
lrfinder.results['loss'][-1] = float('nan')
|
|
|
|
after_lr = lrfinder.suggestion()
|
|
|
|
|
|
|
|
assert before_lr == after_lr, \
|
|
|
|
'Learning rate was altered because of non-finite loss values'
|