import random from abc import ABC from collections import OrderedDict import torch from pytorch_lightning import EvalResult class TestStepVariations(ABC): """ Houses all variations of test steps """ def test_step(self, batch, batch_idx, *args, **kwargs): """ Default, baseline test_step :param batch: :return: """ self.test_step_called = True x, y = batch x = x.view(x.size(0), -1) y_hat = self(x) loss_test = self.loss(y, y_hat) # acc labels_hat = torch.argmax(y_hat, dim=1) test_acc = torch.sum(y == labels_hat).item() / (len(y) * 1.0) test_acc = torch.tensor(test_acc) test_acc = test_acc.type_as(x) # alternate possible outputs to test if batch_idx % 1 == 0: output = OrderedDict({'test_loss': loss_test, 'test_acc': test_acc}) return output if batch_idx % 2 == 0: return test_acc if batch_idx % 3 == 0: output = OrderedDict({'test_loss': loss_test, 'test_acc': test_acc, 'test_dic': {'test_loss_a': loss_test}}) return output def test_step_result_obj(self, batch, batch_idx, *args, **kwargs): """ Default, baseline test_step :param batch: :return: """ x, y = batch x = x.view(x.size(0), -1) y_hat = self(x) loss_test = self.loss(y, y_hat) # acc labels_hat = torch.argmax(y_hat, dim=1) test_acc = torch.sum(y == labels_hat).item() / (len(y) * 1.0) test_acc = torch.tensor(test_acc) test_acc = test_acc.type_as(x) result = EvalResult() # alternate possible outputs to test if batch_idx % 1 == 0: result.log_dict({'test_loss': loss_test, 'test_acc': test_acc}) return result if batch_idx % 2 == 0: return test_acc if batch_idx % 3 == 0: result.log_dict({'test_loss': loss_test, 'test_acc': test_acc}) result.test_dic = {'test_loss_a': loss_test} return result def test_step__multiple_dataloaders(self, batch, batch_idx, dataloader_idx, **kwargs): """ Default, baseline test_step :param batch: :return: """ x, y = batch x = x.view(x.size(0), -1) y_hat = self(x) loss_test = self.loss(y, y_hat) # acc labels_hat = torch.argmax(y_hat, dim=1) test_acc = torch.sum(y == labels_hat).item() / (len(y) * 1.0) test_acc = torch.tensor(test_acc) test_acc = test_acc.type_as(x) # alternate possible outputs to test if batch_idx % 1 == 0: output = OrderedDict({'test_loss': loss_test, 'test_acc': test_acc}) return output if batch_idx % 2 == 0: return test_acc if batch_idx % 3 == 0: output = OrderedDict({ 'test_loss': loss_test, 'test_acc': test_acc, 'test_dic': {'test_loss_a': loss_test} }) return output if batch_idx % 5 == 0: output = OrderedDict({f'test_loss_{dataloader_idx}': loss_test, f'test_acc_{dataloader_idx}': test_acc}) return output def test_step__empty(self, batch, batch_idx, *args, **kwargs): return {} def test_step_result_preds(self, batch, batch_idx, optimizer_idx=None): x, y = batch x = x.view(x.size(0), -1) y_hat = self(x) loss_test = self.loss(y, y_hat) # acc labels_hat = torch.argmax(y_hat, dim=1) test_acc = torch.sum(y == labels_hat).item() / (len(y) * 1.0) test_acc = torch.tensor(test_acc) test_acc = test_acc.type_as(x) # Do regular EvalResult Logging result = EvalResult(checkpoint_on=loss_test) result.log('test_loss', loss_test) result.log('test_acc', test_acc) batch_size = x.size(0) lst_of_str = [random.choice(['dog', 'cat']) for i in range(batch_size)] lst_of_int = [random.randint(500, 1000) for i in range(batch_size)] lst_of_lst = [[x] for x in lst_of_int] lst_of_dict = [{k: v} for k, v in zip(lst_of_str, lst_of_int)] # This is passed in from pytest via parameterization option = getattr(self, 'test_option', 0) prediction_file = getattr(self, 'prediction_file', 'predictions.pt') lazy_ids = torch.arange(batch_idx * self.batch_size, batch_idx * self.batch_size + x.size(0)) # Base if option == 0: self.write_prediction('idxs', lazy_ids, prediction_file) self.write_prediction('preds', labels_hat, prediction_file) # Check mismatching tensor len elif option == 1: self.write_prediction('idxs', torch.cat((lazy_ids, lazy_ids)), prediction_file) self.write_prediction('preds', labels_hat, prediction_file) # write multi-dimension elif option == 2: self.write_prediction('idxs', lazy_ids, prediction_file) self.write_prediction('preds', labels_hat, prediction_file) self.write_prediction('x', x, prediction_file) # write str list elif option == 3: self.write_prediction('idxs', lazy_ids, prediction_file) self.write_prediction('vals', lst_of_str, prediction_file) # write int list elif option == 4: self.write_prediction('idxs', lazy_ids, prediction_file) self.write_prediction('vals', lst_of_int, prediction_file) # write nested list elif option == 5: self.write_prediction('idxs', lazy_ids, prediction_file) self.write_prediction('vals', lst_of_lst, prediction_file) # write dict list elif option == 6: self.write_prediction('idxs', lazy_ids, prediction_file) self.write_prediction('vals', lst_of_dict, prediction_file) elif option == 7: self.write_prediction_dict({'idxs': lazy_ids, 'preds': labels_hat}, prediction_file) return result