refactor imports
This commit is contained in:
Jiri BOROVEC
parent
e3297412fa
commit
b86aee9262
|
|
@ -154,7 +154,7 @@ class LightningTestModelBase(LightningModule):
|
|||
transform = transforms.Compose([transforms.ToTensor(),
|
||||
transforms.Normalize((0.5,), (1.0,))])
|
||||
dataset = TestingMNIST(root=self.hparams.data_root, train=train,
|
||||
transform=transform, download=True, num_samples=800)
|
||||
transform=transform, download=True, num_samples=2000)
|
||||
|
||||
# when using multi-node we need to add the datasampler
|
||||
train_sampler = None
|
||||
|
|
|
|||
|
|
@ -7,27 +7,31 @@ import torch
|
|||
from pytorch_lightning import Trainer
|
||||
from pytorch_lightning.callbacks import ModelCheckpoint
|
||||
from pytorch_lightning.testing import LightningTestModel
|
||||
from . import testing_utils
|
||||
from .utils import (
|
||||
reset_seed, can_run_gpu_test, get_hparams, set_random_master_port, init_save_dir,
|
||||
get_test_tube_logger, init_checkpoint_callback, load_model, run_prediction, clear_save_dir,
|
||||
assert_ok_test_acc
|
||||
)
|
||||
|
||||
|
||||
def test_running_test_pretrained_model_ddp():
|
||||
"""Verify test() on pretrained model"""
|
||||
if not testing_utils.can_run_gpu_test():
|
||||
if not can_run_gpu_test():
|
||||
return
|
||||
|
||||
testing_utils.reset_seed()
|
||||
testing_utils.set_random_master_port()
|
||||
reset_seed()
|
||||
set_random_master_port()
|
||||
|
||||
hparams = testing_utils.get_hparams()
|
||||
hparams = get_hparams()
|
||||
model = LightningTestModel(hparams)
|
||||
|
||||
save_dir = testing_utils.init_save_dir()
|
||||
save_dir = init_save_dir()
|
||||
|
||||
# exp file to get meta
|
||||
logger = testing_utils.get_test_tube_logger(False)
|
||||
logger = get_test_tube_logger(False)
|
||||
|
||||
# exp file to get weights
|
||||
checkpoint = testing_utils.init_checkpoint_callback(logger)
|
||||
checkpoint = init_checkpoint_callback(logger)
|
||||
|
||||
trainer_options = dict(
|
||||
show_progress_bar=False,
|
||||
|
|
@ -49,7 +53,7 @@ def test_running_test_pretrained_model_ddp():
|
|||
|
||||
# correct result and ok accuracy
|
||||
assert result == 1, 'training failed to complete'
|
||||
pretrained_model = testing_utils.load_model(logger.experiment,
|
||||
pretrained_model = load_model(logger.experiment,
|
||||
trainer.checkpoint_callback.filepath,
|
||||
module_class=LightningTestModel)
|
||||
|
||||
|
|
@ -58,25 +62,25 @@ def test_running_test_pretrained_model_ddp():
|
|||
new_trainer.test(pretrained_model)
|
||||
|
||||
for dataloader in model.test_dataloader():
|
||||
testing_utils.run_prediction(dataloader, pretrained_model)
|
||||
run_prediction(dataloader, pretrained_model)
|
||||
|
||||
testing_utils.clear_save_dir()
|
||||
clear_save_dir()
|
||||
|
||||
|
||||
def test_running_test_pretrained_model():
|
||||
testing_utils.reset_seed()
|
||||
reset_seed()
|
||||
|
||||
"""Verify test() on pretrained model"""
|
||||
hparams = testing_utils.get_hparams()
|
||||
hparams = get_hparams()
|
||||
model = LightningTestModel(hparams)
|
||||
|
||||
save_dir = testing_utils.init_save_dir()
|
||||
save_dir = init_save_dir()
|
||||
|
||||
# logger file to get meta
|
||||
logger = testing_utils.get_test_tube_logger(False)
|
||||
logger = get_test_tube_logger(False)
|
||||
|
||||
# logger file to get weights
|
||||
checkpoint = testing_utils.init_checkpoint_callback(logger)
|
||||
checkpoint = init_checkpoint_callback(logger)
|
||||
|
||||
trainer_options = dict(
|
||||
show_progress_bar=False,
|
||||
|
|
@ -93,7 +97,7 @@ def test_running_test_pretrained_model():
|
|||
|
||||
# correct result and ok accuracy
|
||||
assert result == 1, 'training failed to complete'
|
||||
pretrained_model = testing_utils.load_model(
|
||||
pretrained_model = load_model(
|
||||
logger.experiment, trainer.checkpoint_callback.filepath, module_class=LightningTestModel
|
||||
)
|
||||
|
||||
|
|
@ -101,18 +105,18 @@ def test_running_test_pretrained_model():
|
|||
new_trainer.test(pretrained_model)
|
||||
|
||||
# test we have good test accuracy
|
||||
testing_utils.assert_ok_test_acc(new_trainer)
|
||||
testing_utils.clear_save_dir()
|
||||
assert_ok_test_acc(new_trainer)
|
||||
clear_save_dir()
|
||||
|
||||
|
||||
def test_load_model_from_checkpoint():
|
||||
testing_utils.reset_seed()
|
||||
reset_seed()
|
||||
|
||||
"""Verify test() on pretrained model"""
|
||||
hparams = testing_utils.get_hparams()
|
||||
hparams = get_hparams()
|
||||
model = LightningTestModel(hparams)
|
||||
|
||||
save_dir = testing_utils.init_save_dir()
|
||||
save_dir = init_save_dir()
|
||||
|
||||
trainer_options = dict(
|
||||
show_progress_bar=False,
|
||||
|
|
@ -142,27 +146,27 @@ def test_load_model_from_checkpoint():
|
|||
new_trainer.test(pretrained_model)
|
||||
|
||||
# test we have good test accuracy
|
||||
testing_utils.assert_ok_test_acc(new_trainer)
|
||||
testing_utils.clear_save_dir()
|
||||
assert_ok_test_acc(new_trainer)
|
||||
clear_save_dir()
|
||||
|
||||
|
||||
def test_running_test_pretrained_model_dp():
|
||||
testing_utils.reset_seed()
|
||||
reset_seed()
|
||||
|
||||
"""Verify test() on pretrained model"""
|
||||
if not testing_utils.can_run_gpu_test():
|
||||
if not can_run_gpu_test():
|
||||
return
|
||||
|
||||
hparams = testing_utils.get_hparams()
|
||||
hparams = get_hparams()
|
||||
model = LightningTestModel(hparams)
|
||||
|
||||
save_dir = testing_utils.init_save_dir()
|
||||
save_dir = init_save_dir()
|
||||
|
||||
# logger file to get meta
|
||||
logger = testing_utils.get_test_tube_logger(False)
|
||||
logger = get_test_tube_logger(False)
|
||||
|
||||
# logger file to get weights
|
||||
checkpoint = testing_utils.init_checkpoint_callback(logger)
|
||||
checkpoint = init_checkpoint_callback(logger)
|
||||
|
||||
trainer_options = dict(
|
||||
show_progress_bar=True,
|
||||
|
|
@ -181,7 +185,7 @@ def test_running_test_pretrained_model_dp():
|
|||
|
||||
# correct result and ok accuracy
|
||||
assert result == 1, 'training failed to complete'
|
||||
pretrained_model = testing_utils.load_model(logger.experiment,
|
||||
pretrained_model = load_model(logger.experiment,
|
||||
trainer.checkpoint_callback.filepath,
|
||||
module_class=LightningTestModel)
|
||||
|
||||
|
|
@ -189,8 +193,8 @@ def test_running_test_pretrained_model_dp():
|
|||
new_trainer.test(pretrained_model)
|
||||
|
||||
# test we have good test accuracy
|
||||
testing_utils.assert_ok_test_acc(new_trainer)
|
||||
testing_utils.clear_save_dir()
|
||||
assert_ok_test_acc(new_trainer)
|
||||
clear_save_dir()
|
||||
|
||||
|
||||
def test_dp_resume():
|
||||
|
|
@ -198,12 +202,12 @@ def test_dp_resume():
|
|||
Make sure DP continues training correctly
|
||||
:return:
|
||||
"""
|
||||
if not testing_utils.can_run_gpu_test():
|
||||
if not can_run_gpu_test():
|
||||
return
|
||||
|
||||
testing_utils.reset_seed()
|
||||
reset_seed()
|
||||
|
||||
hparams = testing_utils.get_hparams()
|
||||
hparams = get_hparams()
|
||||
model = LightningTestModel(hparams)
|
||||
|
||||
trainer_options = dict(
|
||||
|
|
@ -213,14 +217,14 @@ def test_dp_resume():
|
|||
distributed_backend='dp',
|
||||
)
|
||||
|
||||
save_dir = testing_utils.init_save_dir()
|
||||
save_dir = init_save_dir()
|
||||
|
||||
# get logger
|
||||
logger = testing_utils.get_test_tube_logger(debug=False)
|
||||
logger = get_test_tube_logger(debug=False)
|
||||
|
||||
# exp file to get weights
|
||||
# logger file to get weights
|
||||
checkpoint = testing_utils.init_checkpoint_callback(logger)
|
||||
checkpoint = init_checkpoint_callback(logger)
|
||||
|
||||
# add these to the trainer options
|
||||
trainer_options['logger'] = logger
|
||||
|
|
@ -244,7 +248,7 @@ def test_dp_resume():
|
|||
trainer.hpc_save(save_dir, logger)
|
||||
|
||||
# init new trainer
|
||||
new_logger = testing_utils.get_test_tube_logger(version=logger.version)
|
||||
new_logger = get_test_tube_logger(version=logger.version)
|
||||
trainer_options['logger'] = new_logger
|
||||
trainer_options['checkpoint_callback'] = ModelCheckpoint(save_dir)
|
||||
trainer_options['train_percent_check'] = 0.2
|
||||
|
|
@ -262,7 +266,7 @@ def test_dp_resume():
|
|||
dp_model.eval()
|
||||
|
||||
dataloader = trainer.get_train_dataloader()
|
||||
testing_utils.run_prediction(dataloader, dp_model, dp=True)
|
||||
run_prediction(dataloader, dp_model, dp=True)
|
||||
|
||||
# new model
|
||||
model = LightningTestModel(hparams)
|
||||
|
|
@ -275,7 +279,7 @@ def test_dp_resume():
|
|||
model.freeze()
|
||||
model.unfreeze()
|
||||
|
||||
testing_utils.clear_save_dir()
|
||||
clear_save_dir()
|
||||
|
||||
|
||||
def test_cpu_restore_training():
|
||||
|
|
@ -283,16 +287,16 @@ def test_cpu_restore_training():
|
|||
Verify continue training session on CPU
|
||||
:return:
|
||||
"""
|
||||
testing_utils.reset_seed()
|
||||
reset_seed()
|
||||
|
||||
hparams = testing_utils.get_hparams()
|
||||
hparams = get_hparams()
|
||||
model = LightningTestModel(hparams)
|
||||
|
||||
save_dir = testing_utils.init_save_dir()
|
||||
save_dir = init_save_dir()
|
||||
|
||||
# logger file to get meta
|
||||
test_logger_version = 10
|
||||
logger = testing_utils.get_test_tube_logger(False, version=test_logger_version)
|
||||
logger = get_test_tube_logger(False, version=test_logger_version)
|
||||
|
||||
trainer_options = dict(
|
||||
max_nb_epochs=2,
|
||||
|
|
@ -314,7 +318,7 @@ def test_cpu_restore_training():
|
|||
# wipe-out trainer and model
|
||||
# retrain with not much data... this simulates picking training back up after slurm
|
||||
# we want to see if the weights come back correctly
|
||||
new_logger = testing_utils.get_test_tube_logger(False, version=test_logger_version)
|
||||
new_logger = get_test_tube_logger(False, version=test_logger_version)
|
||||
trainer_options = dict(
|
||||
max_nb_epochs=2,
|
||||
val_check_interval=0.50,
|
||||
|
|
@ -335,7 +339,7 @@ def test_cpu_restore_training():
|
|||
# haven't trained with the new loaded model
|
||||
trainer.model.eval()
|
||||
for dataloader in trainer.get_val_dataloaders():
|
||||
testing_utils.run_prediction(dataloader, trainer.model)
|
||||
run_prediction(dataloader, trainer.model)
|
||||
|
||||
model.on_sanity_check_start = assert_good_acc
|
||||
|
||||
|
|
@ -343,7 +347,7 @@ def test_cpu_restore_training():
|
|||
# and our hook to predict using current model before any more weight updates
|
||||
trainer.fit(model)
|
||||
|
||||
testing_utils.clear_save_dir()
|
||||
clear_save_dir()
|
||||
|
||||
|
||||
def test_model_saving_loading():
|
||||
|
|
@ -351,15 +355,15 @@ def test_model_saving_loading():
|
|||
Tests use case where trainer saves the model, and user loads it from tags independently
|
||||
:return:
|
||||
"""
|
||||
testing_utils.reset_seed()
|
||||
reset_seed()
|
||||
|
||||
hparams = testing_utils.get_hparams()
|
||||
hparams = get_hparams()
|
||||
model = LightningTestModel(hparams)
|
||||
|
||||
save_dir = testing_utils.init_save_dir()
|
||||
save_dir = init_save_dir()
|
||||
|
||||
# logger file to get meta
|
||||
logger = testing_utils.get_test_tube_logger(False)
|
||||
logger = get_test_tube_logger(False)
|
||||
|
||||
trainer_options = dict(
|
||||
max_nb_epochs=1,
|
||||
|
|
@ -402,7 +406,7 @@ def test_model_saving_loading():
|
|||
new_pred = model_2(x)
|
||||
assert torch.all(torch.eq(pred_before_saving, new_pred)).item() == 1
|
||||
|
||||
testing_utils.clear_save_dir()
|
||||
clear_save_dir()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
|
|
|||
|
|
@ -12,7 +12,11 @@ from pytorch_lightning.testing import (
|
|||
LightningTestModelBase,
|
||||
LightningTestMixin,
|
||||
)
|
||||
from . import testing_utils
|
||||
from .utils import (
|
||||
reset_seed, can_run_gpu_test, get_hparams, run_model_test, init_save_dir,
|
||||
get_test_tube_logger, init_checkpoint_callback, clear_save_dir,
|
||||
get_model, run_model_test_no_loggers, assert_ok_test_acc
|
||||
)
|
||||
|
||||
|
||||
def test_early_stopping_cpu_model():
|
||||
|
|
@ -20,7 +24,7 @@ def test_early_stopping_cpu_model():
|
|||
Test each of the trainer options
|
||||
:return:
|
||||
"""
|
||||
testing_utils.reset_seed()
|
||||
reset_seed()
|
||||
|
||||
stopping = EarlyStopping(monitor='val_loss', min_delta=1e-2)
|
||||
trainer_options = dict(
|
||||
|
|
@ -30,13 +34,13 @@ def test_early_stopping_cpu_model():
|
|||
track_grad_norm=2,
|
||||
print_nan_grads=True,
|
||||
show_progress_bar=True,
|
||||
logger=testing_utils.get_test_tube_logger(),
|
||||
logger=get_test_tube_logger(),
|
||||
train_percent_check=0.1,
|
||||
val_percent_check=0.1
|
||||
)
|
||||
|
||||
model, hparams = testing_utils.get_model()
|
||||
testing_utils.run_model_test(trainer_options, model, hparams, on_gpu=False)
|
||||
model, hparams = get_model()
|
||||
run_model_test(trainer_options, model, hparams, on_gpu=False)
|
||||
|
||||
# test freeze on cpu
|
||||
model.freeze()
|
||||
|
|
@ -48,7 +52,7 @@ def test_lbfgs_cpu_model():
|
|||
Test each of the trainer options
|
||||
:return:
|
||||
"""
|
||||
testing_utils.reset_seed()
|
||||
reset_seed()
|
||||
|
||||
trainer_options = dict(
|
||||
max_nb_epochs=1,
|
||||
|
|
@ -59,11 +63,10 @@ def test_lbfgs_cpu_model():
|
|||
val_percent_check=0.2
|
||||
)
|
||||
|
||||
model, hparams = testing_utils.get_model(use_test_model=True, lbfgs=True)
|
||||
testing_utils.run_model_test_no_loggers(trainer_options,
|
||||
model, hparams, on_gpu=False, min_acc=0.30)
|
||||
model, hparams = get_model(use_test_model=True, lbfgs=True)
|
||||
run_model_test_no_loggers(trainer_options, model, hparams, on_gpu=False, min_acc=0.30)
|
||||
|
||||
testing_utils.clear_save_dir()
|
||||
clear_save_dir()
|
||||
|
||||
|
||||
def test_default_logger_callbacks_cpu_model():
|
||||
|
|
@ -71,7 +74,7 @@ def test_default_logger_callbacks_cpu_model():
|
|||
Test each of the trainer options
|
||||
:return:
|
||||
"""
|
||||
testing_utils.reset_seed()
|
||||
reset_seed()
|
||||
|
||||
trainer_options = dict(
|
||||
max_nb_epochs=1,
|
||||
|
|
@ -83,30 +86,30 @@ def test_default_logger_callbacks_cpu_model():
|
|||
val_percent_check=0.01
|
||||
)
|
||||
|
||||
model, hparams = testing_utils.get_model()
|
||||
testing_utils.run_model_test_no_loggers(trainer_options, model, hparams, on_gpu=False)
|
||||
model, hparams = get_model()
|
||||
run_model_test_no_loggers(trainer_options, model, hparams, on_gpu=False)
|
||||
|
||||
# test freeze on cpu
|
||||
model.freeze()
|
||||
model.unfreeze()
|
||||
|
||||
testing_utils.clear_save_dir()
|
||||
clear_save_dir()
|
||||
|
||||
|
||||
def test_running_test_after_fitting():
|
||||
"""Verify test() on fitted model"""
|
||||
testing_utils.reset_seed()
|
||||
reset_seed()
|
||||
|
||||
hparams = testing_utils.get_hparams()
|
||||
hparams = get_hparams()
|
||||
model = LightningTestModel(hparams)
|
||||
|
||||
save_dir = testing_utils.init_save_dir()
|
||||
save_dir = init_save_dir()
|
||||
|
||||
# logger file to get meta
|
||||
logger = testing_utils.get_test_tube_logger(False)
|
||||
logger = get_test_tube_logger(False)
|
||||
|
||||
# logger file to get weights
|
||||
checkpoint = testing_utils.init_checkpoint_callback(logger)
|
||||
checkpoint = init_checkpoint_callback(logger)
|
||||
|
||||
trainer_options = dict(
|
||||
show_progress_bar=False,
|
||||
|
|
@ -127,29 +130,29 @@ def test_running_test_after_fitting():
|
|||
trainer.test()
|
||||
|
||||
# test we have good test accuracy
|
||||
testing_utils.assert_ok_test_acc(trainer)
|
||||
assert_ok_test_acc(trainer)
|
||||
|
||||
testing_utils.clear_save_dir()
|
||||
clear_save_dir()
|
||||
|
||||
|
||||
def test_running_test_without_val():
|
||||
testing_utils.reset_seed()
|
||||
reset_seed()
|
||||
|
||||
"""Verify test() works on a model with no val_loader"""
|
||||
|
||||
class CurrentTestModel(LightningTestMixin, LightningTestModelBase):
|
||||
pass
|
||||
|
||||
hparams = testing_utils.get_hparams()
|
||||
hparams = get_hparams()
|
||||
model = CurrentTestModel(hparams)
|
||||
|
||||
save_dir = testing_utils.init_save_dir()
|
||||
save_dir = init_save_dir()
|
||||
|
||||
# logger file to get meta
|
||||
logger = testing_utils.get_test_tube_logger(False)
|
||||
logger = get_test_tube_logger(False)
|
||||
|
||||
# logger file to get weights
|
||||
checkpoint = testing_utils.init_checkpoint_callback(logger)
|
||||
checkpoint = init_checkpoint_callback(logger)
|
||||
|
||||
trainer_options = dict(
|
||||
show_progress_bar=False,
|
||||
|
|
@ -170,15 +173,15 @@ def test_running_test_without_val():
|
|||
trainer.test()
|
||||
|
||||
# test we have good test accuracy
|
||||
testing_utils.assert_ok_test_acc(trainer)
|
||||
assert_ok_test_acc(trainer)
|
||||
|
||||
testing_utils.clear_save_dir()
|
||||
clear_save_dir()
|
||||
|
||||
|
||||
def test_single_gpu_batch_parse():
|
||||
testing_utils.reset_seed()
|
||||
reset_seed()
|
||||
|
||||
if not testing_utils.can_run_gpu_test():
|
||||
if not can_run_gpu_test():
|
||||
return
|
||||
|
||||
trainer = Trainer()
|
||||
|
|
@ -224,12 +227,12 @@ def test_simple_cpu():
|
|||
Verify continue training session on CPU
|
||||
:return:
|
||||
"""
|
||||
testing_utils.reset_seed()
|
||||
reset_seed()
|
||||
|
||||
hparams = testing_utils.get_hparams()
|
||||
hparams = get_hparams()
|
||||
model = LightningTestModel(hparams)
|
||||
|
||||
save_dir = testing_utils.init_save_dir()
|
||||
save_dir = init_save_dir()
|
||||
|
||||
# logger file to get meta
|
||||
trainer_options = dict(
|
||||
|
|
@ -245,7 +248,7 @@ def test_simple_cpu():
|
|||
# traning complete
|
||||
assert result == 1, 'amp + ddp model failed to complete'
|
||||
|
||||
testing_utils.clear_save_dir()
|
||||
clear_save_dir()
|
||||
|
||||
|
||||
def test_cpu_model():
|
||||
|
|
@ -253,19 +256,19 @@ def test_cpu_model():
|
|||
Make sure model trains on CPU
|
||||
:return:
|
||||
"""
|
||||
testing_utils.reset_seed()
|
||||
reset_seed()
|
||||
|
||||
trainer_options = dict(
|
||||
show_progress_bar=False,
|
||||
logger=testing_utils.get_test_tube_logger(),
|
||||
logger=get_test_tube_logger(),
|
||||
max_nb_epochs=1,
|
||||
train_percent_check=0.4,
|
||||
val_percent_check=0.4
|
||||
)
|
||||
|
||||
model, hparams = testing_utils.get_model()
|
||||
model, hparams = get_model()
|
||||
|
||||
testing_utils.run_model_test(trainer_options, model, hparams, on_gpu=False)
|
||||
run_model_test(trainer_options, model, hparams, on_gpu=False)
|
||||
|
||||
|
||||
def test_all_features_cpu_model():
|
||||
|
|
@ -273,7 +276,7 @@ def test_all_features_cpu_model():
|
|||
Test each of the trainer options
|
||||
:return:
|
||||
"""
|
||||
testing_utils.reset_seed()
|
||||
reset_seed()
|
||||
|
||||
trainer_options = dict(
|
||||
gradient_clip_val=1.0,
|
||||
|
|
@ -281,15 +284,15 @@ def test_all_features_cpu_model():
|
|||
track_grad_norm=2,
|
||||
print_nan_grads=True,
|
||||
show_progress_bar=False,
|
||||
logger=testing_utils.get_test_tube_logger(),
|
||||
logger=get_test_tube_logger(),
|
||||
accumulate_grad_batches=2,
|
||||
max_nb_epochs=1,
|
||||
train_percent_check=0.4,
|
||||
val_percent_check=0.4
|
||||
)
|
||||
|
||||
model, hparams = testing_utils.get_model()
|
||||
testing_utils.run_model_test(trainer_options, model, hparams, on_gpu=False)
|
||||
model, hparams = get_model()
|
||||
run_model_test(trainer_options, model, hparams, on_gpu=False)
|
||||
|
||||
|
||||
def test_tbptt_cpu_model():
|
||||
|
|
@ -297,9 +300,9 @@ def test_tbptt_cpu_model():
|
|||
Test truncated back propagation through time works.
|
||||
:return:
|
||||
"""
|
||||
testing_utils.reset_seed()
|
||||
reset_seed()
|
||||
|
||||
save_dir = testing_utils.init_save_dir()
|
||||
save_dir = init_save_dir()
|
||||
|
||||
truncated_bptt_steps = 2
|
||||
sequence_size = 30
|
||||
|
|
@ -354,7 +357,7 @@ def test_tbptt_cpu_model():
|
|||
weights_summary=None,
|
||||
)
|
||||
|
||||
hparams = testing_utils.get_hparams()
|
||||
hparams = get_hparams()
|
||||
hparams.batch_size = batch_size
|
||||
hparams.in_features = truncated_bptt_steps
|
||||
hparams.hidden_dim = truncated_bptt_steps
|
||||
|
|
@ -368,7 +371,7 @@ def test_tbptt_cpu_model():
|
|||
|
||||
assert result == 1, 'training failed to complete'
|
||||
|
||||
testing_utils.clear_save_dir()
|
||||
clear_save_dir()
|
||||
|
||||
|
||||
def test_single_gpu_model():
|
||||
|
|
@ -376,13 +379,13 @@ def test_single_gpu_model():
|
|||
Make sure single GPU works (DP mode)
|
||||
:return:
|
||||
"""
|
||||
testing_utils.reset_seed()
|
||||
reset_seed()
|
||||
|
||||
if not torch.cuda.is_available():
|
||||
warnings.warn('test_single_gpu_model cannot run.'
|
||||
' Rerun on a GPU node to run this test')
|
||||
return
|
||||
model, hparams = testing_utils.get_model()
|
||||
model, hparams = get_model()
|
||||
|
||||
trainer_options = dict(
|
||||
show_progress_bar=False,
|
||||
|
|
@ -392,7 +395,7 @@ def test_single_gpu_model():
|
|||
gpus=1
|
||||
)
|
||||
|
||||
testing_utils.run_model_test(trainer_options, model, hparams)
|
||||
run_model_test(trainer_options, model, hparams)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
|
|
|||
|
|
@ -15,7 +15,10 @@ from pytorch_lightning.trainer.dp_mixin import (
|
|||
determine_root_gpu_device,
|
||||
)
|
||||
from pytorch_lightning.utilities.debugging import MisconfigurationException
|
||||
from . import testing_utils
|
||||
from .utils import (
|
||||
reset_seed, can_run_gpu_test, get_hparams, run_model_test, set_random_master_port,
|
||||
init_save_dir, get_test_tube_logger, clear_save_dir
|
||||
)
|
||||
|
||||
PRETEND_N_OF_GPUS = 16
|
||||
|
||||
|
|
@ -25,13 +28,13 @@ def test_multi_gpu_model_ddp2():
|
|||
Make sure DDP2 works
|
||||
:return:
|
||||
"""
|
||||
if not testing_utils.can_run_gpu_test():
|
||||
if not can_run_gpu_test():
|
||||
return
|
||||
|
||||
testing_utils.reset_seed()
|
||||
testing_utils.set_random_master_port()
|
||||
reset_seed()
|
||||
set_random_master_port()
|
||||
|
||||
model, hparams = testing_utils.get_model()
|
||||
model, hparams = get_model()
|
||||
trainer_options = dict(
|
||||
show_progress_bar=True,
|
||||
max_nb_epochs=1,
|
||||
|
|
@ -42,7 +45,7 @@ def test_multi_gpu_model_ddp2():
|
|||
distributed_backend='ddp2'
|
||||
)
|
||||
|
||||
testing_utils.run_model_test(trainer_options, model, hparams)
|
||||
run_model_test(trainer_options, model, hparams)
|
||||
|
||||
|
||||
def test_multi_gpu_model_ddp():
|
||||
|
|
@ -50,13 +53,13 @@ def test_multi_gpu_model_ddp():
|
|||
Make sure DDP works
|
||||
:return:
|
||||
"""
|
||||
if not testing_utils.can_run_gpu_test():
|
||||
if not can_run_gpu_test():
|
||||
return
|
||||
|
||||
testing_utils.reset_seed()
|
||||
testing_utils.set_random_master_port()
|
||||
reset_seed()
|
||||
set_random_master_port()
|
||||
|
||||
model, hparams = testing_utils.get_model()
|
||||
model, hparams = get_model()
|
||||
trainer_options = dict(
|
||||
show_progress_bar=False,
|
||||
max_nb_epochs=1,
|
||||
|
|
@ -66,14 +69,14 @@ def test_multi_gpu_model_ddp():
|
|||
distributed_backend='ddp'
|
||||
)
|
||||
|
||||
testing_utils.run_model_test(trainer_options, model, hparams)
|
||||
run_model_test(trainer_options, model, hparams)
|
||||
|
||||
|
||||
def test_optimizer_return_options():
|
||||
testing_utils.reset_seed()
|
||||
reset_seed()
|
||||
|
||||
trainer = Trainer()
|
||||
model, hparams = testing_utils.get_model()
|
||||
model, hparams = get_model()
|
||||
|
||||
# single optimizer
|
||||
opt_a = torch.optim.Adam(model.parameters(), lr=0.002)
|
||||
|
|
@ -105,15 +108,15 @@ def test_cpu_slurm_save_load():
|
|||
Verify model save/load/checkpoint on CPU
|
||||
:return:
|
||||
"""
|
||||
testing_utils.reset_seed()
|
||||
reset_seed()
|
||||
|
||||
hparams = testing_utils.get_hparams()
|
||||
hparams = get_hparams()
|
||||
model = LightningTestModel(hparams)
|
||||
|
||||
save_dir = testing_utils.init_save_dir()
|
||||
save_dir = init_save_dir()
|
||||
|
||||
# logger file to get meta
|
||||
logger = testing_utils.get_test_tube_logger(False)
|
||||
logger = get_test_tube_logger(False)
|
||||
|
||||
version = logger.version
|
||||
|
||||
|
|
@ -149,7 +152,7 @@ def test_cpu_slurm_save_load():
|
|||
assert os.path.exists(saved_filepath)
|
||||
|
||||
# new logger file to get meta
|
||||
logger = testing_utils.get_test_tube_logger(False, version=version)
|
||||
logger = get_test_tube_logger(False, version=version)
|
||||
|
||||
trainer_options = dict(
|
||||
max_nb_epochs=1,
|
||||
|
|
@ -174,7 +177,7 @@ def test_cpu_slurm_save_load():
|
|||
# and our hook to predict using current model before any more weight updates
|
||||
trainer.fit(model)
|
||||
|
||||
testing_utils.clear_save_dir()
|
||||
clear_save_dir()
|
||||
|
||||
|
||||
def test_multi_gpu_none_backend():
|
||||
|
|
@ -183,12 +186,12 @@ def test_multi_gpu_none_backend():
|
|||
distributed_backend = None
|
||||
:return:
|
||||
"""
|
||||
testing_utils.reset_seed()
|
||||
reset_seed()
|
||||
|
||||
if not testing_utils.can_run_gpu_test():
|
||||
if not can_run_gpu_test():
|
||||
return
|
||||
|
||||
model, hparams = testing_utils.get_model()
|
||||
model, hparams = get_model()
|
||||
trainer_options = dict(
|
||||
show_progress_bar=False,
|
||||
max_nb_epochs=1,
|
||||
|
|
@ -198,7 +201,7 @@ def test_multi_gpu_none_backend():
|
|||
)
|
||||
|
||||
with pytest.raises(MisconfigurationException):
|
||||
testing_utils.run_model_test(trainer_options, model, hparams)
|
||||
run_model_test(trainer_options, model, hparams)
|
||||
|
||||
|
||||
def test_multi_gpu_model_dp():
|
||||
|
|
@ -206,12 +209,12 @@ def test_multi_gpu_model_dp():
|
|||
Make sure DP works
|
||||
:return:
|
||||
"""
|
||||
testing_utils.reset_seed()
|
||||
reset_seed()
|
||||
|
||||
if not testing_utils.can_run_gpu_test():
|
||||
if not can_run_gpu_test():
|
||||
return
|
||||
|
||||
model, hparams = testing_utils.get_model()
|
||||
model, hparams = get_model()
|
||||
trainer_options = dict(
|
||||
show_progress_bar=False,
|
||||
distributed_backend='dp',
|
||||
|
|
@ -221,7 +224,7 @@ def test_multi_gpu_model_dp():
|
|||
gpus='-1'
|
||||
)
|
||||
|
||||
testing_utils.run_model_test(trainer_options, model, hparams)
|
||||
run_model_test(trainer_options, model, hparams)
|
||||
|
||||
# test memory helper functions
|
||||
memory.get_memory_profile('min_max')
|
||||
|
|
@ -232,16 +235,16 @@ def test_ddp_sampler_error():
|
|||
Make sure DDP + AMP work
|
||||
:return:
|
||||
"""
|
||||
if not testing_utils.can_run_gpu_test():
|
||||
if not can_run_gpu_test():
|
||||
return
|
||||
|
||||
testing_utils.reset_seed()
|
||||
testing_utils.set_random_master_port()
|
||||
reset_seed()
|
||||
set_random_master_port()
|
||||
|
||||
hparams = testing_utils.get_hparams()
|
||||
hparams = get_hparams()
|
||||
model = LightningTestModel(hparams, force_remove_distributed_sampler=True)
|
||||
|
||||
logger = testing_utils.get_test_tube_logger(True)
|
||||
logger = get_test_tube_logger(True)
|
||||
|
||||
trainer = Trainer(
|
||||
logger=logger,
|
||||
|
|
@ -255,7 +258,7 @@ def test_ddp_sampler_error():
|
|||
with pytest.warns(UserWarning):
|
||||
trainer.get_dataloaders(model)
|
||||
|
||||
testing_utils.clear_save_dir()
|
||||
clear_save_dir()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
|
|
|
|||
|
|
@ -16,7 +16,9 @@ from pytorch_lightning.testing import (
|
|||
)
|
||||
from pytorch_lightning.trainer import trainer_io
|
||||
from pytorch_lightning.trainer.logging_mixin import TrainerLoggingMixin
|
||||
from . import testing_utils
|
||||
from .utils import (
|
||||
reset_seed, get_hparams, init_save_dir, get_test_tube_logger, run_prediction, clear_save_dir
|
||||
)
|
||||
|
||||
|
||||
def test_no_val_module():
|
||||
|
|
@ -24,19 +26,19 @@ def test_no_val_module():
|
|||
Tests use case where trainer saves the model, and user loads it from tags independently
|
||||
:return:
|
||||
"""
|
||||
testing_utils.reset_seed()
|
||||
reset_seed()
|
||||
|
||||
hparams = testing_utils.get_hparams()
|
||||
hparams = get_hparams()
|
||||
|
||||
class CurrentTestModel(LightningTestModelBase):
|
||||
pass
|
||||
|
||||
model = CurrentTestModel(hparams)
|
||||
|
||||
save_dir = testing_utils.init_save_dir()
|
||||
save_dir = init_save_dir()
|
||||
|
||||
# logger file to get meta
|
||||
logger = testing_utils.get_test_tube_logger(False)
|
||||
logger = get_test_tube_logger(False)
|
||||
|
||||
trainer_options = dict(
|
||||
max_nb_epochs=1,
|
||||
|
|
@ -63,7 +65,7 @@ def test_no_val_module():
|
|||
model_2.eval()
|
||||
|
||||
# make prediction
|
||||
testing_utils.clear_save_dir()
|
||||
clear_save_dir()
|
||||
|
||||
|
||||
def test_no_val_end_module():
|
||||
|
|
@ -71,18 +73,18 @@ def test_no_val_end_module():
|
|||
Tests use case where trainer saves the model, and user loads it from tags independently
|
||||
:return:
|
||||
"""
|
||||
testing_utils.reset_seed()
|
||||
reset_seed()
|
||||
|
||||
class CurrentTestModel(LightningValidationStepMixin, LightningTestModelBase):
|
||||
pass
|
||||
|
||||
hparams = testing_utils.get_hparams()
|
||||
hparams = get_hparams()
|
||||
model = CurrentTestModel(hparams)
|
||||
|
||||
save_dir = testing_utils.init_save_dir()
|
||||
save_dir = init_save_dir()
|
||||
|
||||
# logger file to get meta
|
||||
logger = testing_utils.get_test_tube_logger(False)
|
||||
logger = get_test_tube_logger(False)
|
||||
|
||||
trainer_options = dict(
|
||||
max_nb_epochs=1,
|
||||
|
|
@ -109,11 +111,11 @@ def test_no_val_end_module():
|
|||
model_2.eval()
|
||||
|
||||
# make prediction
|
||||
testing_utils.clear_save_dir()
|
||||
clear_save_dir()
|
||||
|
||||
|
||||
def test_gradient_accumulation_scheduling():
|
||||
testing_utils.reset_seed()
|
||||
reset_seed()
|
||||
|
||||
"""
|
||||
Test grad accumulation by the freq of optimizer updates
|
||||
|
|
@ -170,7 +172,7 @@ def test_gradient_accumulation_scheduling():
|
|||
# clear gradients
|
||||
optimizer.zero_grad()
|
||||
|
||||
hparams = testing_utils.get_hparams()
|
||||
hparams = get_hparams()
|
||||
model = LightningTestModel(hparams)
|
||||
schedule = {1: 2, 3: 4}
|
||||
|
||||
|
|
@ -187,13 +189,13 @@ def test_gradient_accumulation_scheduling():
|
|||
|
||||
|
||||
def test_loading_meta_tags():
|
||||
testing_utils.reset_seed()
|
||||
reset_seed()
|
||||
|
||||
from argparse import Namespace
|
||||
hparams = testing_utils.get_hparams()
|
||||
hparams = get_hparams()
|
||||
|
||||
# save tags
|
||||
logger = testing_utils.get_test_tube_logger(False)
|
||||
logger = get_test_tube_logger(False)
|
||||
logger.log_hyperparams(Namespace(some_str='a_str', an_int=1, a_float=2.0))
|
||||
logger.log_hyperparams(hparams)
|
||||
logger.save()
|
||||
|
|
@ -206,12 +208,12 @@ def test_loading_meta_tags():
|
|||
|
||||
assert tags.batch_size == 32 and tags.hidden_dim == 1000
|
||||
|
||||
testing_utils.clear_save_dir()
|
||||
clear_save_dir()
|
||||
|
||||
|
||||
def test_dp_output_reduce():
|
||||
mixin = TrainerLoggingMixin()
|
||||
testing_utils.reset_seed()
|
||||
reset_seed()
|
||||
|
||||
# test identity when we have a single gpu
|
||||
out = torch.rand(3, 1)
|
||||
|
|
@ -240,11 +242,11 @@ def test_model_checkpoint_options():
|
|||
def mock_save_function(filepath):
|
||||
open(filepath, 'a').close()
|
||||
|
||||
hparams = testing_utils.get_hparams()
|
||||
hparams = get_hparams()
|
||||
model = LightningTestModel(hparams)
|
||||
|
||||
# simulated losses
|
||||
save_dir = testing_utils.init_save_dir()
|
||||
save_dir = init_save_dir()
|
||||
losses = [10, 9, 2.8, 5, 2.5]
|
||||
|
||||
# -----------------
|
||||
|
|
@ -262,7 +264,7 @@ def test_model_checkpoint_options():
|
|||
for i in range(0, len(losses)):
|
||||
assert f'_ckpt_epoch_{i}.ckpt' in file_lists
|
||||
|
||||
testing_utils.clear_save_dir()
|
||||
clear_save_dir()
|
||||
|
||||
# -----------------
|
||||
# CASE K=0 (none)
|
||||
|
|
@ -275,7 +277,7 @@ def test_model_checkpoint_options():
|
|||
|
||||
assert len(file_lists) == 0, "Should save 0 models when save_top_k=0"
|
||||
|
||||
testing_utils.clear_save_dir()
|
||||
clear_save_dir()
|
||||
|
||||
# -----------------
|
||||
# CASE K=1 (2.5, epoch 4)
|
||||
|
|
@ -289,7 +291,7 @@ def test_model_checkpoint_options():
|
|||
assert len(file_lists) == 1, "Should save 1 model when save_top_k=1"
|
||||
assert 'test_prefix_ckpt_epoch_4.ckpt' in file_lists
|
||||
|
||||
testing_utils.clear_save_dir()
|
||||
clear_save_dir()
|
||||
|
||||
# -----------------
|
||||
# CASE K=2 (2.5 epoch 4, 2.8 epoch 2)
|
||||
|
|
@ -308,7 +310,7 @@ def test_model_checkpoint_options():
|
|||
assert '_ckpt_epoch_2.ckpt' in file_lists
|
||||
assert 'other_file.ckpt' in file_lists
|
||||
|
||||
testing_utils.clear_save_dir()
|
||||
clear_save_dir()
|
||||
|
||||
# -----------------
|
||||
# CASE K=4 (save all 4 models)
|
||||
|
|
@ -323,7 +325,7 @@ def test_model_checkpoint_options():
|
|||
|
||||
assert len(file_lists) == 4, 'Should save all 4 models when save_top_k=4 within same epoch'
|
||||
|
||||
testing_utils.clear_save_dir()
|
||||
clear_save_dir()
|
||||
|
||||
# -----------------
|
||||
# CASE K=3 (save the 2nd, 3rd, 4th model)
|
||||
|
|
@ -341,13 +343,13 @@ def test_model_checkpoint_options():
|
|||
assert '_ckpt_epoch_0_v1.ckpt' in file_lists
|
||||
assert '_ckpt_epoch_0.ckpt' in file_lists
|
||||
|
||||
testing_utils.clear_save_dir()
|
||||
clear_save_dir()
|
||||
|
||||
|
||||
def test_model_freeze_unfreeze():
|
||||
testing_utils.reset_seed()
|
||||
reset_seed()
|
||||
|
||||
hparams = testing_utils.get_hparams()
|
||||
hparams = get_hparams()
|
||||
model = LightningTestModel(hparams)
|
||||
|
||||
model.freeze()
|
||||
|
|
@ -359,7 +361,7 @@ def test_multiple_val_dataloader():
|
|||
Verify multiple val_dataloader
|
||||
:return:
|
||||
"""
|
||||
testing_utils.reset_seed()
|
||||
reset_seed()
|
||||
|
||||
class CurrentTestModel(
|
||||
LightningValidationMultipleDataloadersMixin,
|
||||
|
|
@ -367,7 +369,7 @@ def test_multiple_val_dataloader():
|
|||
):
|
||||
pass
|
||||
|
||||
hparams = testing_utils.get_hparams()
|
||||
hparams = get_hparams()
|
||||
model = CurrentTestModel(hparams)
|
||||
|
||||
# logger file to get meta
|
||||
|
|
@ -390,7 +392,7 @@ def test_multiple_val_dataloader():
|
|||
|
||||
# make sure predictions are good for each val set
|
||||
for dataloader in trainer.get_val_dataloaders():
|
||||
testing_utils.run_prediction(dataloader, trainer.model)
|
||||
run_prediction(dataloader, trainer.model)
|
||||
|
||||
|
||||
def test_multiple_test_dataloader():
|
||||
|
|
@ -398,7 +400,7 @@ def test_multiple_test_dataloader():
|
|||
Verify multiple test_dataloader
|
||||
:return:
|
||||
"""
|
||||
testing_utils.reset_seed()
|
||||
reset_seed()
|
||||
|
||||
class CurrentTestModel(
|
||||
LightningTestMultipleDataloadersMixin,
|
||||
|
|
@ -406,7 +408,7 @@ def test_multiple_test_dataloader():
|
|||
):
|
||||
pass
|
||||
|
||||
hparams = testing_utils.get_hparams()
|
||||
hparams = get_hparams()
|
||||
model = CurrentTestModel(hparams)
|
||||
|
||||
# logger file to get meta
|
||||
|
|
@ -426,7 +428,7 @@ def test_multiple_test_dataloader():
|
|||
|
||||
# make sure predictions are good for each test set
|
||||
for dataloader in trainer.get_test_dataloaders():
|
||||
testing_utils.run_prediction(dataloader, trainer.model)
|
||||
run_prediction(dataloader, trainer.model)
|
||||
|
||||
# run the test method
|
||||
trainer.test()
|
||||
|
|
|
|||
|
|
@ -7,13 +7,9 @@ import torch
|
|||
from pytorch_lightning import Trainer
|
||||
from pytorch_lightning.testing import LightningTestModel
|
||||
from pytorch_lightning.logging import LightningLoggerBase, rank_zero_only
|
||||
from . import testing_utils
|
||||
|
||||
RANDOM_FILE_PATHS = list(np.random.randint(12000, 19000, 1000))
|
||||
ROOT_SEED = 1234
|
||||
torch.manual_seed(ROOT_SEED)
|
||||
np.random.seed(ROOT_SEED)
|
||||
RANDOM_SEEDS = list(np.random.randint(0, 10000, 1000))
|
||||
from .utils import (
|
||||
reset_seed, get_hparams, init_save_dir, get_test_tube_logger, clear_save_dir
|
||||
)
|
||||
|
||||
|
||||
def test_testtube_logger():
|
||||
|
|
@ -21,12 +17,12 @@ def test_testtube_logger():
|
|||
verify that basic functionality of test tube logger works
|
||||
"""
|
||||
reset_seed()
|
||||
hparams = testing_utils.get_hparams()
|
||||
hparams = get_hparams()
|
||||
model = LightningTestModel(hparams)
|
||||
|
||||
save_dir = testing_utils.init_save_dir()
|
||||
save_dir = init_save_dir()
|
||||
|
||||
logger = testing_utils.get_test_tube_logger(False)
|
||||
logger = get_test_tube_logger(False)
|
||||
|
||||
trainer_options = dict(
|
||||
max_nb_epochs=1,
|
||||
|
|
@ -39,7 +35,7 @@ def test_testtube_logger():
|
|||
|
||||
assert result == 1, "Training failed"
|
||||
|
||||
testing_utils.clear_save_dir()
|
||||
clear_save_dir()
|
||||
|
||||
|
||||
def test_testtube_pickle():
|
||||
|
|
@ -48,12 +44,12 @@ def test_testtube_pickle():
|
|||
"""
|
||||
reset_seed()
|
||||
|
||||
hparams = testing_utils.get_hparams()
|
||||
hparams = get_hparams()
|
||||
model = LightningTestModel(hparams)
|
||||
|
||||
save_dir = testing_utils.init_save_dir()
|
||||
save_dir = init_save_dir()
|
||||
|
||||
logger = testing_utils.get_test_tube_logger(False)
|
||||
logger = get_test_tube_logger(False)
|
||||
logger.log_hyperparams(hparams)
|
||||
logger.save()
|
||||
|
||||
|
|
@ -68,7 +64,7 @@ def test_testtube_pickle():
|
|||
trainer2 = pickle.loads(pkl_bytes)
|
||||
trainer2.logger.log_metrics({"acc": 1.0})
|
||||
|
||||
testing_utils.clear_save_dir()
|
||||
clear_save_dir()
|
||||
|
||||
|
||||
def test_mlflow_logger():
|
||||
|
|
@ -82,7 +78,7 @@ def test_mlflow_logger():
|
|||
except ModuleNotFoundError:
|
||||
return
|
||||
|
||||
hparams = testing_utils.get_hparams()
|
||||
hparams = get_hparams()
|
||||
model = LightningTestModel(hparams)
|
||||
|
||||
root_dir = os.path.dirname(os.path.realpath(__file__))
|
||||
|
|
@ -102,7 +98,7 @@ def test_mlflow_logger():
|
|||
print('result finished')
|
||||
assert result == 1, "Training failed"
|
||||
|
||||
testing_utils.clear_save_dir()
|
||||
clear_save_dir()
|
||||
|
||||
|
||||
def test_mlflow_pickle():
|
||||
|
|
@ -116,7 +112,7 @@ def test_mlflow_pickle():
|
|||
except ModuleNotFoundError:
|
||||
return
|
||||
|
||||
hparams = testing_utils.get_hparams()
|
||||
hparams = get_hparams()
|
||||
model = LightningTestModel(hparams)
|
||||
|
||||
root_dir = os.path.dirname(os.path.realpath(__file__))
|
||||
|
|
@ -134,7 +130,7 @@ def test_mlflow_pickle():
|
|||
trainer2 = pickle.loads(pkl_bytes)
|
||||
trainer2.logger.log_metrics({"acc": 1.0})
|
||||
|
||||
testing_utils.clear_save_dir()
|
||||
clear_save_dir()
|
||||
|
||||
|
||||
def test_comet_logger():
|
||||
|
|
@ -148,7 +144,7 @@ def test_comet_logger():
|
|||
except ModuleNotFoundError:
|
||||
return
|
||||
|
||||
hparams = testing_utils.get_hparams()
|
||||
hparams = get_hparams()
|
||||
model = LightningTestModel(hparams)
|
||||
|
||||
root_dir = os.path.dirname(os.path.realpath(__file__))
|
||||
|
|
@ -173,7 +169,7 @@ def test_comet_logger():
|
|||
print('result finished')
|
||||
assert result == 1, "Training failed"
|
||||
|
||||
testing_utils.clear_save_dir()
|
||||
clear_save_dir()
|
||||
|
||||
|
||||
def test_comet_pickle():
|
||||
|
|
@ -187,7 +183,7 @@ def test_comet_pickle():
|
|||
except ModuleNotFoundError:
|
||||
return
|
||||
|
||||
hparams = testing_utils.get_hparams()
|
||||
hparams = get_hparams()
|
||||
model = LightningTestModel(hparams)
|
||||
|
||||
root_dir = os.path.dirname(os.path.realpath(__file__))
|
||||
|
|
@ -210,7 +206,7 @@ def test_comet_pickle():
|
|||
trainer2 = pickle.loads(pkl_bytes)
|
||||
trainer2.logger.log_metrics({"acc": 1.0})
|
||||
|
||||
testing_utils.clear_save_dir()
|
||||
clear_save_dir()
|
||||
|
||||
|
||||
def test_custom_logger(tmpdir):
|
||||
|
|
@ -241,7 +237,7 @@ def test_custom_logger(tmpdir):
|
|||
def version(self):
|
||||
return "1"
|
||||
|
||||
hparams = testing_utils.get_hparams()
|
||||
hparams = get_hparams()
|
||||
model = LightningTestModel(hparams)
|
||||
|
||||
logger = CustomLogger()
|
||||
|
|
@ -259,9 +255,3 @@ def test_custom_logger(tmpdir):
|
|||
assert logger.hparams_logged == hparams
|
||||
assert logger.metrics_logged != {}
|
||||
assert logger.finalized_status == "success"
|
||||
|
||||
|
||||
def reset_seed():
|
||||
SEED = RANDOM_SEEDS.pop()
|
||||
torch.manual_seed(SEED)
|
||||
np.random.seed(SEED)
|
||||
|
|
|
|||
|
|
@ -9,7 +9,10 @@ from pytorch_lightning.testing import (
|
|||
LightningTestModel,
|
||||
)
|
||||
from pytorch_lightning.utilities.debugging import MisconfigurationException
|
||||
from . import testing_utils
|
||||
from .utils import (
|
||||
reset_seed, can_run_gpu_test, get_hparams, run_model_test, set_random_master_port, init_save_dir,
|
||||
get_test_tube_logger, init_checkpoint_callback, load_model, run_prediction, clear_save_dir, get_model
|
||||
)
|
||||
|
||||
|
||||
def test_amp_single_gpu():
|
||||
|
|
@ -17,12 +20,12 @@ def test_amp_single_gpu():
|
|||
Make sure DDP + AMP work
|
||||
:return:
|
||||
"""
|
||||
testing_utils.reset_seed()
|
||||
reset_seed()
|
||||
|
||||
if not testing_utils.can_run_gpu_test():
|
||||
if not can_run_gpu_test():
|
||||
return
|
||||
|
||||
hparams = testing_utils.get_hparams()
|
||||
hparams = get_hparams()
|
||||
model = LightningTestModel(hparams)
|
||||
|
||||
trainer_options = dict(
|
||||
|
|
@ -33,7 +36,7 @@ def test_amp_single_gpu():
|
|||
use_amp=True
|
||||
)
|
||||
|
||||
testing_utils.run_model_test(trainer_options, model, hparams)
|
||||
run_model_test(trainer_options, model, hparams)
|
||||
|
||||
|
||||
def test_no_amp_single_gpu():
|
||||
|
|
@ -41,12 +44,12 @@ def test_no_amp_single_gpu():
|
|||
Make sure DDP + AMP work
|
||||
:return:
|
||||
"""
|
||||
testing_utils.reset_seed()
|
||||
reset_seed()
|
||||
|
||||
if not testing_utils.can_run_gpu_test():
|
||||
if not can_run_gpu_test():
|
||||
return
|
||||
|
||||
hparams = testing_utils.get_hparams()
|
||||
hparams = get_hparams()
|
||||
model = LightningTestModel(hparams)
|
||||
|
||||
trainer_options = dict(
|
||||
|
|
@ -58,7 +61,7 @@ def test_no_amp_single_gpu():
|
|||
)
|
||||
|
||||
with pytest.raises((MisconfigurationException, ModuleNotFoundError)):
|
||||
testing_utils.run_model_test(trainer_options, model, hparams)
|
||||
run_model_test(trainer_options, model, hparams)
|
||||
|
||||
|
||||
def test_amp_gpu_ddp():
|
||||
|
|
@ -66,13 +69,13 @@ def test_amp_gpu_ddp():
|
|||
Make sure DDP + AMP work
|
||||
:return:
|
||||
"""
|
||||
if not testing_utils.can_run_gpu_test():
|
||||
if not can_run_gpu_test():
|
||||
return
|
||||
|
||||
testing_utils.reset_seed()
|
||||
testing_utils.set_random_master_port()
|
||||
reset_seed()
|
||||
set_random_master_port()
|
||||
|
||||
hparams = testing_utils.get_hparams()
|
||||
hparams = get_hparams()
|
||||
model = LightningTestModel(hparams)
|
||||
|
||||
trainer_options = dict(
|
||||
|
|
@ -83,7 +86,7 @@ def test_amp_gpu_ddp():
|
|||
use_amp=True
|
||||
)
|
||||
|
||||
testing_utils.run_model_test(trainer_options, model, hparams)
|
||||
run_model_test(trainer_options, model, hparams)
|
||||
|
||||
|
||||
def test_amp_gpu_ddp_slurm_managed():
|
||||
|
|
@ -91,16 +94,16 @@ def test_amp_gpu_ddp_slurm_managed():
|
|||
Make sure DDP + AMP work
|
||||
:return:
|
||||
"""
|
||||
if not testing_utils.can_run_gpu_test():
|
||||
if not can_run_gpu_test():
|
||||
return
|
||||
|
||||
testing_utils.reset_seed()
|
||||
reset_seed()
|
||||
|
||||
# simulate setting slurm flags
|
||||
testing_utils.set_random_master_port()
|
||||
set_random_master_port()
|
||||
os.environ['SLURM_LOCALID'] = str(0)
|
||||
|
||||
hparams = testing_utils.get_hparams()
|
||||
hparams = get_hparams()
|
||||
model = LightningTestModel(hparams)
|
||||
|
||||
trainer_options = dict(
|
||||
|
|
@ -111,13 +114,13 @@ def test_amp_gpu_ddp_slurm_managed():
|
|||
use_amp=True
|
||||
)
|
||||
|
||||
save_dir = testing_utils.init_save_dir()
|
||||
save_dir = init_save_dir()
|
||||
|
||||
# exp file to get meta
|
||||
logger = testing_utils.get_test_tube_logger(False)
|
||||
logger = get_test_tube_logger(False)
|
||||
|
||||
# exp file to get weights
|
||||
checkpoint = testing_utils.init_checkpoint_callback(logger)
|
||||
checkpoint = init_checkpoint_callback(logger)
|
||||
|
||||
# add these to the trainer options
|
||||
trainer_options['checkpoint_callback'] = checkpoint
|
||||
|
|
@ -138,12 +141,12 @@ def test_amp_gpu_ddp_slurm_managed():
|
|||
assert trainer.resolve_root_node_address('abc[23-24, 45-40, 40]') == 'abc23'
|
||||
|
||||
# test model loading with a map_location
|
||||
pretrained_model = testing_utils.load_model(logger.experiment,
|
||||
pretrained_model = load_model(logger.experiment,
|
||||
trainer.checkpoint_callback.filepath)
|
||||
|
||||
# test model preds
|
||||
for dataloader in trainer.get_test_dataloaders():
|
||||
testing_utils.run_prediction(dataloader, pretrained_model)
|
||||
run_prediction(dataloader, pretrained_model)
|
||||
|
||||
if trainer.use_ddp:
|
||||
# on hpc this would work fine... but need to hack it for the purpose of the test
|
||||
|
|
@ -158,7 +161,7 @@ def test_amp_gpu_ddp_slurm_managed():
|
|||
model.freeze()
|
||||
model.unfreeze()
|
||||
|
||||
testing_utils.clear_save_dir()
|
||||
clear_save_dir()
|
||||
|
||||
|
||||
def test_cpu_model_with_amp():
|
||||
|
|
@ -166,21 +169,21 @@ def test_cpu_model_with_amp():
|
|||
Make sure model trains on CPU
|
||||
:return:
|
||||
"""
|
||||
testing_utils.reset_seed()
|
||||
reset_seed()
|
||||
|
||||
trainer_options = dict(
|
||||
show_progress_bar=False,
|
||||
logger=testing_utils.get_test_tube_logger(),
|
||||
logger=get_test_tube_logger(),
|
||||
max_nb_epochs=1,
|
||||
train_percent_check=0.4,
|
||||
val_percent_check=0.4,
|
||||
use_amp=True
|
||||
)
|
||||
|
||||
model, hparams = testing_utils.get_model()
|
||||
model, hparams = get_model()
|
||||
|
||||
with pytest.raises((MisconfigurationException, ModuleNotFoundError)):
|
||||
testing_utils.run_model_test(trainer_options, model, hparams, on_gpu=False)
|
||||
run_model_test(trainer_options, model, hparams, on_gpu=False)
|
||||
|
||||
|
||||
def test_amp_gpu_dp():
|
||||
|
|
@ -188,12 +191,12 @@ def test_amp_gpu_dp():
|
|||
Make sure DP + AMP work
|
||||
:return:
|
||||
"""
|
||||
testing_utils.reset_seed()
|
||||
reset_seed()
|
||||
|
||||
if not testing_utils.can_run_gpu_test():
|
||||
if not can_run_gpu_test():
|
||||
return
|
||||
|
||||
model, hparams = testing_utils.get_model()
|
||||
model, hparams = get_model()
|
||||
trainer_options = dict(
|
||||
max_nb_epochs=1,
|
||||
gpus='0, 1', # test init with gpu string
|
||||
|
|
@ -201,7 +204,7 @@ def test_amp_gpu_dp():
|
|||
use_amp=True
|
||||
)
|
||||
with pytest.raises(MisconfigurationException):
|
||||
testing_utils.run_model_test(trainer_options, model, hparams)
|
||||
run_model_test(trainer_options, model, hparams)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
|
|
|||
Loading…
Reference in New Issue