diff --git a/docs/source/experiment_logging.rst b/docs/source/experiment_logging.rst deleted file mode 100644 index 4ccad84ef2..0000000000 --- a/docs/source/experiment_logging.rst +++ /dev/null @@ -1,242 +0,0 @@ -.. testsetup:: * - - from pytorch_lightning.trainer.trainer import Trainer - from pytorch_lightning.core.lightning import LightningModule - -.. _experiment_logging: - -Experiment Logging -================== - -Comet.ml -^^^^^^^^ - -`Comet.ml `_ is a third-party logger. -To use :class:`~pytorch_lightning.loggers.CometLogger` as your logger do the following. -First, install the package: - -.. code-block:: bash - - pip install comet-ml - -Then configure the logger and pass it to the :class:`~pytorch_lightning.trainer.trainer.Trainer`: - -.. testcode:: - - import os - from pytorch_lightning.loggers import CometLogger - comet_logger = CometLogger( - api_key=os.environ.get('COMET_API_KEY'), - workspace=os.environ.get('COMET_WORKSPACE'), # Optional - save_dir='.', # Optional - project_name='default_project', # Optional - rest_api_key=os.environ.get('COMET_REST_API_KEY'), # Optional - experiment_name='default' # Optional - ) - trainer = Trainer(logger=comet_logger) - -The :class:`~pytorch_lightning.loggers.CometLogger` is available anywhere except ``__init__`` in your -:class:`~pytorch_lightning.core.lightning.LightningModule`. - -.. testcode:: - - class MyModule(LightningModule): - def any_lightning_module_function_or_hook(self): - some_img = fake_image() - self.logger.experiment.add_image('generated_images', some_img, 0) - -.. seealso:: - :class:`~pytorch_lightning.loggers.CometLogger` docs. - ----------------- - -MLflow -^^^^^^ - -`MLflow `_ is a third-party logger. -To use :class:`~pytorch_lightning.loggers.MLFlowLogger` as your logger do the following. -First, install the package: - -.. code-block:: bash - - pip install mlflow - -Then configure the logger and pass it to the :class:`~pytorch_lightning.trainer.trainer.Trainer`: - -.. testcode:: - - from pytorch_lightning.loggers import MLFlowLogger - mlf_logger = MLFlowLogger( - experiment_name="default", - tracking_uri="file:./ml-runs" - ) - trainer = Trainer(logger=mlf_logger) - -.. seealso:: - :class:`~pytorch_lightning.loggers.MLFlowLogger` docs. - ----------------- - -Neptune.ai -^^^^^^^^^^ - -`Neptune.ai `_ is a third-party logger. -To use :class:`~pytorch_lightning.loggers.NeptuneLogger` as your logger do the following. -First, install the package: - -.. code-block:: bash - - pip install neptune-client - -Then configure the logger and pass it to the :class:`~pytorch_lightning.trainer.trainer.Trainer`: - -.. testcode:: - - from pytorch_lightning.loggers import NeptuneLogger - - neptune_logger = NeptuneLogger( - api_key='ANONYMOUS', # replace with your own - project_name='shared/pytorch-lightning-integration', - experiment_name='default', # Optional, - params={'max_epochs': 10}, # Optional, - tags=['pytorch-lightning', 'mlp'], # Optional, - ) - trainer = Trainer(logger=neptune_logger) - -The :class:`~pytorch_lightning.loggers.NeptuneLogger` is available anywhere except ``__init__`` in your -:class:`~pytorch_lightning.core.lightning.LightningModule`. - -.. testcode:: - - class MyModule(LightningModule): - def any_lightning_module_function_or_hook(self): - some_img = fake_image() - self.logger.experiment.add_image('generated_images', some_img, 0) - -.. seealso:: - :class:`~pytorch_lightning.loggers.NeptuneLogger` docs. - ----------------- - -Tensorboard -^^^^^^^^^^^ - -To use `TensorBoard `_ as your logger do the following. - -.. testcode:: - - from pytorch_lightning.loggers import TensorBoardLogger - logger = TensorBoardLogger('tb_logs', name='my_model') - trainer = Trainer(logger=logger) - -The :class:`~pytorch_lightning.loggers.TensorBoardLogger` is available anywhere except ``__init__`` in your -:class:`~pytorch_lightning.core.lightning.LightningModule`. - -.. testcode:: - - class MyModule(LightningModule): - def any_lightning_module_function_or_hook(self): - some_img = fake_image() - self.logger.experiment.add_image('generated_images', some_img, 0) - -.. seealso:: - :class:`~pytorch_lightning.loggers.TensorBoardLogger` docs. - ----------------- - -Test Tube -^^^^^^^^^ - -`Test Tube `_ is a -`TensorBoard `_ logger but with nicer file structure. -To use :class:`~pytorch_lightning.loggers.TestTubeLogger` as your logger do the following. -First, install the package: - -.. code-block:: bash - - pip install test_tube - -Then configure the logger and pass it to the :class:`~pytorch_lightning.trainer.trainer.Trainer`: - -.. testcode:: - - from pytorch_lightning.loggers import TestTubeLogger - logger = TestTubeLogger('tb_logs', name='my_model') - trainer = Trainer(logger=logger) - -The :class:`~pytorch_lightning.loggers.TestTubeLogger` is available anywhere except ``__init__`` in your -:class:`~pytorch_lightning.core.lightning.LightningModule`. - -.. testcode:: - - class MyModule(LightningModule): - def any_lightning_module_function_or_hook(self): - some_img = fake_image() - self.logger.experiment.add_image('generated_images', some_img, 0) - -.. seealso:: - :class:`~pytorch_lightning.loggers.TestTubeLogger` docs. - ----------------- - -Weights and Biases -^^^^^^^^^^^^^^^^^^ - -`Weights and Biases `_ is a third-party logger. -To use :class:`~pytorch_lightning.loggers.WandbLogger` as your logger do the following. -First, install the package: - -.. code-block:: bash - - pip install wandb - -Then configure the logger and pass it to the :class:`~pytorch_lightning.trainer.trainer.Trainer`: - -.. code-block:: python - - from pytorch_lightning.loggers import WandbLogger - wandb_logger = WandbLogger(offline=True) - trainer = Trainer(logger=wandb_logger) - -The :class:`~pytorch_lightning.loggers.WandbLogger` is available anywhere except ``__init__`` in your -:class:`~pytorch_lightning.core.lightning.LightningModule`. - -.. testcode:: - - class MyModule(LightningModule): - def any_lightning_module_function_or_hook(self): - some_img = fake_image() - self.logger.experiment.log({ - "generated_images": [wandb.Image(some_img, caption="...")] - }) - -.. seealso:: - :class:`~pytorch_lightning.loggers.WandbLogger` docs. - ----------------- - -Multiple Loggers -^^^^^^^^^^^^^^^^ - -Lightning supports the use of multiple loggers, just pass a list to the -:class:`~pytorch_lightning.trainer.trainer.Trainer`. - -.. testcode:: - - from pytorch_lightning.loggers import TensorBoardLogger, TestTubeLogger - logger1 = TensorBoardLogger('tb_logs', name='my_model') - logger2 = TestTubeLogger('tb_logs', name='my_model') - trainer = Trainer(logger=[logger1, logger2]) - -The loggers are available as a list anywhere except ``__init__`` in your -:class:`~pytorch_lightning.core.lightning.LightningModule`. - -.. testcode:: - - class MyModule(LightningModule): - def any_lightning_module_function_or_hook(self): - some_img = fake_image() - # Option 1 - self.logger.experiment[0].add_image('generated_images', some_img, 0) - # Option 2 - self.logger[0].experiment.add_image('generated_images', some_img, 0) diff --git a/docs/source/experiment_reporting.rst b/docs/source/experiment_reporting.rst deleted file mode 100644 index 4b6f0bb1ef..0000000000 --- a/docs/source/experiment_reporting.rst +++ /dev/null @@ -1,168 +0,0 @@ -.. testsetup:: * - - from pytorch_lightning.trainer.trainer import Trainer - -.. _experiment_reporting: - -Experiment Reporting -===================== - -Lightning supports many different experiment loggers. These loggers allow you to monitor losses, images, text, etc... -as training progresses. They usually provide a GUI to visualize and can sometimes even snapshot hyperparameters -used in each experiment. - ----------- - -Control logging frequency -^^^^^^^^^^^^^^^^^^^^^^^^^ - -It may slow training down to log every single batch. Trainer has an option to log every k batches instead. - -.. testcode:: - - k = 10 - trainer = Trainer(row_log_interval=k) - ----------- - -Control log writing frequency -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Writing to a logger can be expensive. In Lightning you can set the interval at which you -want to save logs to the filesystem using this trainer flag. - -.. testcode:: - - k = 100 - trainer = Trainer(log_save_interval=k) - -Unlike the `row_log_interval`, this argument does not apply to all loggers. -The example shown here works with :class:`~pytorch_lightning.loggers.tensorboard.TensorBoardLogger`, -which is the default logger in Lightning. - ----------- - -Log metrics -^^^^^^^^^^^ - -To plot metrics into whatever logger you passed in (tensorboard, comet, neptune, etc...) - -1. training_epoch_end, validation_epoch_end, test_epoch_end will all log anything in the "log" key of the return dict. - -.. testcode:: - - def training_epoch_end(self, outputs): - loss = some_loss() - ... - - logs = {'train_loss': loss} - results = {'log': logs} - return results - - def validation_epoch_end(self, outputs): - loss = some_loss() - ... - - logs = {'val_loss': loss} - results = {'log': logs} - return results - - def test_epoch_end(self, outputs): - loss = some_loss() - ... - - logs = {'test_loss': loss} - results = {'log': logs} - return results - -2. In addition, you can also use any arbitrary functionality from a particular logger from within your LightningModule. -For instance, here we log images using tensorboard. - -.. testcode:: - :skipif: not TORCHVISION_AVAILABLE - - def training_step(self, batch, batch_idx): - self.generated_imgs = self.decoder.generate() - - sample_imgs = self.generated_imgs[:6] - grid = torchvision.utils.make_grid(sample_imgs) - self.logger.experiment.add_image('generated_images', grid, 0) - - ... - return results - ----------- - -Modify progress bar -^^^^^^^^^^^^^^^^^^^ - -Each return dict from the -:meth:`~pytorch_lightning.core.lightning.LightningModule.training_step`, -:meth:`~pytorch_lightning.core.lightning.LightningModule.training_epoch_end`, -:meth:`~pytorch_lightning.core.lightning.LightningModule.validation_epoch_end` and -:meth:`~pytorch_lightning.core.lightning.LightningModule.test_epoch_end` -can also contain a key called `progress_bar`. - -Here we show the validation loss in the progress bar: - -.. testcode:: - - def validation_epoch_end(self, outputs): - loss = some_loss() - ... - - logs = {'val_loss': loss} - results = {'progress_bar': logs} - return results - -The progress bar by default already includes the training loss and version number of the experiment -if you are using a logger. These defaults can be customized by overriding the -:meth:`~pytorch_lightning.core.lightning.LightningModule.get_progress_bar_dict` hook in your module. - - ----------- - -Configure console logging -^^^^^^^^^^^^^^^^^^^^^^^^^ - -Lightning logs useful information about the training process and user warnings to the console. -You can retrieve the Lightning logger and change it to your liking. For example, increase the logging level -to see fewer messages like so: - -.. code-block:: python - - import logging - logging.getLogger("lightning").setLevel(logging.ERROR) - -Read more about custom Python logging `here `_. - - ----------- - -Snapshot hyperparameters -^^^^^^^^^^^^^^^^^^^^^^^^ - -When training a model, it's useful to know what hyperparams went into that model. -When Lightning creates a checkpoint, it stores a key "hparams" with the hyperparams. - -.. code-block:: python - - lightning_checkpoint = torch.load(filepath, map_location=lambda storage, loc: storage) - hyperparams = lightning_checkpoint['hparams'] - -Some loggers also allow logging the hyperparams used in the experiment. For instance, -when using the TestTubeLogger or the TensorBoardLogger, all hyperparams will show -in the `hparams tab `_. - ----------- - -Snapshot code -^^^^^^^^^^^^^ - -Loggers also allow you to snapshot a copy of the code used in this experiment. -For example, TestTubeLogger does this with a flag: - -.. testcode:: - - from pytorch_lightning.loggers import TestTubeLogger - logger = TestTubeLogger('.', create_git_tag=True) diff --git a/docs/source/index.rst b/docs/source/index.rst index d2d41c2768..c683f44a43 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -37,7 +37,7 @@ PyTorch Lightning Documentation callbacks datamodules - loggers + logging metrics .. toctree:: @@ -87,8 +87,7 @@ PyTorch Lightning Documentation slurm child_modules debugging - experiment_logging - experiment_reporting + loggers early_stopping fast_training hooks diff --git a/docs/source/loggers.rst b/docs/source/loggers.rst index 93b2e1cdac..0fae0f88ca 100644 --- a/docs/source/loggers.rst +++ b/docs/source/loggers.rst @@ -1,211 +1,253 @@ .. testsetup:: * - from pytorch_lightning.core.lightning import LightningModule from pytorch_lightning.trainer.trainer import Trainer - from pytorch_lightning import loggers as pl_loggers + from pytorch_lightning.core.lightning import LightningModule -.. role:: hidden - :class: hidden-section - .. _loggers: +******* Loggers -=========== -Lightning supports the most popular logging frameworks (TensorBoard, Comet, etc...). -To use a logger, simply pass it into the :class:`~pytorch_lightning.trainer.trainer.Trainer`. -Lightning uses TensorBoard by default. +******* -.. testcode:: - - from pytorch_lightning import loggers as pl_loggers - - tb_logger = pl_loggers.TensorBoardLogger('logs/') - trainer = Trainer(logger=tb_logger) - -Choose from any of the others such as MLflow, Comet, Neptune, WandB, ... - -.. testcode:: - - comet_logger = pl_loggers.CometLogger(save_dir='logs/') - trainer = Trainer(logger=comet_logger) - -To use multiple loggers, simply pass in a ``list`` or ``tuple`` of loggers ... - -.. testcode:: - - tb_logger = pl_loggers.TensorBoardLogger('logs/') - comet_logger = pl_loggers.CometLogger(save_dir='logs/') - trainer = Trainer(logger=[tb_logger, comet_logger]) +Lightning supports the most popular logging frameworks (TensorBoard, Comet, etc...). TensorBoard is used by default, +but you can pass to the :class:`~pytorch_lightning.trainer.trainer.Trainer` any combintation of the following loggers. .. note:: All loggers log by default to `os.getcwd()`. To change the path without creating a logger set `Trainer(default_root_dir='/your/path/to/save/checkpoints')` ----------- +Read more about :ref:`logging` options. -Logging from a LightningModule ------------------------------- -Interact with loggers in two ways, automatically and/or manually. +Comet.ml +======== -Automatic logging -^^^^^^^^^^^^^^^^^ -Use the :func:`~~pytorch_lightning.core.lightning.LightningModule.log` method to log from anywhere in a LightningModule. - -.. code-block:: python - - def training_step(self, batch, batch_idx): - self.log('my_metric', x) - -The :func:`~~pytorch_lightning.core.lightning.LightningModule.log` method has a few options: - -- on_step (logs the metric at that step in training) -- on_epoch (automatically accumulates and logs at the end of the epoch) -- prog_bar (logs to the progress bar) -- logger (logs to the logger like Tensorboard) - -Depending on where log is called from, Lightning auto-determines the correct mode for you. But of course -you can override the default behavior by manually setting the flags - -.. note:: Setting on_epoch=True will accumulate your logged values over the full training epoch. - -.. code-block:: python - - def training_step(self, batch, batch_idx): - self.log('my_loss', loss, on_step=True, on_epoch=True, prog_bar=True, logger=True) - -Once your training starts, you can view the logs by using your favorite logger or booting up the Tensorboard logs: +`Comet.ml `_ is a third-party logger. +To use :class:`~pytorch_lightning.loggers.CometLogger` as your logger do the following. +First, install the package: .. code-block:: bash - tensorboard --logdir ./lightning_logs + pip install comet-ml - -Manual logging -^^^^^^^^^^^^^^ -For certain things like histograms, text, images, etc... you may need to use the logger object directly. - -.. code-block:: python - - def training_step(...): - ... - # the logger you used (in this case tensorboard) - tensorboard = self.logger.experiment - tensorboard.add_histogram(...) - tensorboard.add_figure(...) - ----------- - -Logging from a Callback ------------------------ -To log from a callback, the :func:`~~pytorch_lightning.core.lightning.LightningModule.log` -method of the LightningModule. - -.. code-block:: python - - class MyCallback(Callback): - - def on_train_epoch_end(self, trainer, pl_module): - pl_module.log('something', x) - -or access the logger object directly - -.. code-block:: python - - class MyCallback(Callback): - - def on_train_epoch_end(self, trainer, pl_module): - tensorboard = pl_module.logger.experiment - tensorboard.add_histogram(...) - tensorboard.add_figure(...) - ----------- - -Make a Custom Logger --------------------- - -You can implement your own logger by writing a class that inherits from -:class:`LightningLoggerBase`. Use the :func:`~pytorch_lightning.loggers.base.rank_zero_only` -decorator to make sure that only the first process in DDP training logs data. +Then configure the logger and pass it to the :class:`~pytorch_lightning.trainer.trainer.Trainer`: .. testcode:: - from pytorch_lightning.utilities import rank_zero_only - from pytorch_lightning.loggers import LightningLoggerBase + import os + from pytorch_lightning.loggers import CometLogger + comet_logger = CometLogger( + api_key=os.environ.get('COMET_API_KEY'), + workspace=os.environ.get('COMET_WORKSPACE'), # Optional + save_dir='.', # Optional + project_name='default_project', # Optional + rest_api_key=os.environ.get('COMET_REST_API_KEY'), # Optional + experiment_name='default' # Optional + ) + trainer = Trainer(logger=comet_logger) - class MyLogger(LightningLoggerBase): +The :class:`~pytorch_lightning.loggers.CometLogger` is available anywhere except ``__init__`` in your +:class:`~pytorch_lightning.core.lightning.LightningModule`. - @rank_zero_only - def log_hyperparams(self, params): - # params is an argparse.Namespace - # your code to record hyperparameters goes here - pass +.. testcode:: - @rank_zero_only - def log_metrics(self, metrics, step): - # metrics is a dictionary of metric names and values - # your code to record metrics goes here - pass + class MyModule(LightningModule): + def any_lightning_module_function_or_hook(self): + some_img = fake_image() + self.logger.experiment.add_image('generated_images', some_img, 0) - def save(self): - # Optional. Any code necessary to save logger data goes here - # If you implement this, remember to call `super().save()` - # at the start of the method (important for aggregation of metrics) - super().save() +.. seealso:: + :class:`~pytorch_lightning.loggers.CometLogger` docs. - @rank_zero_only - def finalize(self, status): - # Optional. Any code that needs to be run after training - # finishes goes here - pass +---------------- -If you write a logger that may be useful to others, please send -a pull request to add it to Lightning! +MLflow +====== ----------- +`MLflow `_ is a third-party logger. +To use :class:`~pytorch_lightning.loggers.MLFlowLogger` as your logger do the following. +First, install the package: -Supported Loggers ------------------ -The following are loggers we support +.. code-block:: bash -Comet -^^^^^ + pip install mlflow -.. autoclass:: pytorch_lightning.loggers.comet.CometLogger - :noindex: +Then configure the logger and pass it to the :class:`~pytorch_lightning.trainer.trainer.Trainer`: -CSVLogger -^^^^^^^^^ +.. testcode:: -.. autoclass:: pytorch_lightning.loggers.csv_logs.CSVLogger - :noindex: + from pytorch_lightning.loggers import MLFlowLogger + mlf_logger = MLFlowLogger( + experiment_name="default", + tracking_uri="file:./ml-runs" + ) + trainer = Trainer(logger=mlf_logger) -MLFlow -^^^^^^ +.. seealso:: + :class:`~pytorch_lightning.loggers.MLFlowLogger` docs. -.. autoclass:: pytorch_lightning.loggers.mlflow.MLFlowLogger - :noindex: +---------------- -Neptune -^^^^^^^ +Neptune.ai +========== -.. autoclass:: pytorch_lightning.loggers.neptune.NeptuneLogger - :noindex: +`Neptune.ai `_ is a third-party logger. +To use :class:`~pytorch_lightning.loggers.NeptuneLogger` as your logger do the following. +First, install the package: + +.. code-block:: bash + + pip install neptune-client + +Then configure the logger and pass it to the :class:`~pytorch_lightning.trainer.trainer.Trainer`: + +.. testcode:: + + from pytorch_lightning.loggers import NeptuneLogger + + neptune_logger = NeptuneLogger( + api_key='ANONYMOUS', # replace with your own + project_name='shared/pytorch-lightning-integration', + experiment_name='default', # Optional, + params={'max_epochs': 10}, # Optional, + tags=['pytorch-lightning', 'mlp'], # Optional, + ) + trainer = Trainer(logger=neptune_logger) + +The :class:`~pytorch_lightning.loggers.NeptuneLogger` is available anywhere except ``__init__`` in your +:class:`~pytorch_lightning.core.lightning.LightningModule`. + +.. testcode:: + + class MyModule(LightningModule): + def any_lightning_module_function_or_hook(self): + some_img = fake_image() + self.logger.experiment.add_image('generated_images', some_img, 0) + +.. seealso:: + :class:`~pytorch_lightning.loggers.NeptuneLogger` docs. + +---------------- Tensorboard -^^^^^^^^^^^^ +=========== -.. autoclass:: pytorch_lightning.loggers.tensorboard.TensorBoardLogger - :noindex: +To use `TensorBoard `_ as your logger do the following. -Test-tube -^^^^^^^^^ +.. testcode:: -.. autoclass:: pytorch_lightning.loggers.test_tube.TestTubeLogger - :noindex: + from pytorch_lightning.loggers import TensorBoardLogger + logger = TensorBoardLogger('tb_logs', name='my_model') + trainer = Trainer(logger=logger) + +The :class:`~pytorch_lightning.loggers.TensorBoardLogger` is available anywhere except ``__init__`` in your +:class:`~pytorch_lightning.core.lightning.LightningModule`. + +.. testcode:: + + class MyModule(LightningModule): + def any_lightning_module_function_or_hook(self): + some_img = fake_image() + self.logger.experiment.add_image('generated_images', some_img, 0) + +.. seealso:: + :class:`~pytorch_lightning.loggers.TensorBoardLogger` docs. + +---------------- + +Test Tube +========= + +`Test Tube `_ is a +`TensorBoard `_ logger but with nicer file structure. +To use :class:`~pytorch_lightning.loggers.TestTubeLogger` as your logger do the following. +First, install the package: + +.. code-block:: bash + + pip install test_tube + +Then configure the logger and pass it to the :class:`~pytorch_lightning.trainer.trainer.Trainer`: + +.. testcode:: + + from pytorch_lightning.loggers import TestTubeLogger + logger = TestTubeLogger('tb_logs', name='my_model') + trainer = Trainer(logger=logger) + +The :class:`~pytorch_lightning.loggers.TestTubeLogger` is available anywhere except ``__init__`` in your +:class:`~pytorch_lightning.core.lightning.LightningModule`. + +.. testcode:: + + class MyModule(LightningModule): + def any_lightning_module_function_or_hook(self): + some_img = fake_image() + self.logger.experiment.add_image('generated_images', some_img, 0) + +.. seealso:: + :class:`~pytorch_lightning.loggers.TestTubeLogger` docs. + +---------------- Weights and Biases -^^^^^^^^^^^^^^^^^^ +================== -.. autoclass:: pytorch_lightning.loggers.wandb.WandbLogger - :noindex: +`Weights and Biases `_ is a third-party logger. +To use :class:`~pytorch_lightning.loggers.WandbLogger` as your logger do the following. +First, install the package: + +.. code-block:: bash + + pip install wandb + +Then configure the logger and pass it to the :class:`~pytorch_lightning.trainer.trainer.Trainer`: + +.. code-block:: python + + from pytorch_lightning.loggers import WandbLogger + wandb_logger = WandbLogger(offline=True) + trainer = Trainer(logger=wandb_logger) + +The :class:`~pytorch_lightning.loggers.WandbLogger` is available anywhere except ``__init__`` in your +:class:`~pytorch_lightning.core.lightning.LightningModule`. + +.. testcode:: + + class MyModule(LightningModule): + def any_lightning_module_function_or_hook(self): + some_img = fake_image() + self.logger.experiment.log({ + "generated_images": [wandb.Image(some_img, caption="...")] + }) + +.. seealso:: + :class:`~pytorch_lightning.loggers.WandbLogger` docs. + +---------------- + +Multiple Loggers +================ + +Lightning supports the use of multiple loggers, just pass a list to the +:class:`~pytorch_lightning.trainer.trainer.Trainer`. + +.. testcode:: + + from pytorch_lightning.loggers import TensorBoardLogger, TestTubeLogger + logger1 = TensorBoardLogger('tb_logs', name='my_model') + logger2 = TestTubeLogger('tb_logs', name='my_model') + trainer = Trainer(logger=[logger1, logger2]) + +The loggers are available as a list anywhere except ``__init__`` in your +:class:`~pytorch_lightning.core.lightning.LightningModule`. + +.. testcode:: + + class MyModule(LightningModule): + def any_lightning_module_function_or_hook(self): + some_img = fake_image() + # Option 1 + self.logger.experiment[0].add_image('generated_images', some_img, 0) + # Option 2 + self.logger[0].experiment.add_image('generated_images', some_img, 0) diff --git a/docs/source/logging.rst b/docs/source/logging.rst new file mode 100644 index 0000000000..8dff05a4b0 --- /dev/null +++ b/docs/source/logging.rst @@ -0,0 +1,362 @@ +.. testsetup:: * + + from pytorch_lightning.core.lightning import LightningModule + from pytorch_lightning.trainer.trainer import Trainer + from pytorch_lightning import loggers as pl_loggers + +.. role:: hidden + :class: hidden-section + +.. _logging: + + +####### +Logging +####### + +Lightning supports the most popular logging frameworks (TensorBoard, Comet, etc...). +To use a logger, simply pass it into the :class:`~pytorch_lightning.trainer.trainer.Trainer`. +Lightning uses TensorBoard by default. + +.. testcode:: + + from pytorch_lightning import loggers as pl_loggers + + tb_logger = pl_loggers.TensorBoardLogger('logs/') + trainer = Trainer(logger=tb_logger) + +Choose from any of the others such as MLflow, Comet, Neptune, WandB, ... + +.. testcode:: + + comet_logger = pl_loggers.CometLogger(save_dir='logs/') + trainer = Trainer(logger=comet_logger) + +To use multiple loggers, simply pass in a ``list`` or ``tuple`` of loggers ... + +.. testcode:: + + tb_logger = pl_loggers.TensorBoardLogger('logs/') + comet_logger = pl_loggers.CometLogger(save_dir='logs/') + trainer = Trainer(logger=[tb_logger, comet_logger]) + +.. note:: + + By default, lightning logs every 50 steps. Use Trainer flags to :ref:`logging_frequency`. + +.. note:: + + All loggers log by default to `os.getcwd()`. To change the path without creating a logger set + `Trainer(default_root_dir='/your/path/to/save/checkpoints')` + +---------- + +****************************** +Logging from a LightningModule +****************************** + +Lightning offers automatic log functionalities for logging scalars, or manual logging for anything else. + +Automatic logging +================= +Use the :func:`~~pytorch_lightning.core.lightning.LightningModule.log` method to log from anywhere in a :class:`~pytorch_lightning.core.LightningModule`. + +.. code-block:: python + + def training_step(self, batch, batch_idx): + self.log('my_metric', x) + +Depending on where log is called from, Lightning auto-determines the correct logging mode for you.\ +But of course you can override the default behavior by manually setting the :func:`~~pytorch_lightning.core.lightning.LightningModule.log` parameters. + +.. code-block:: python + + def training_step(self, batch, batch_idx): + self.log('my_loss', loss, on_step=True, on_epoch=True, prog_bar=True, logger=True) + +The :func:`~~pytorch_lightning.core.lightning.LightningModule.log` method has a few options: + +* on_step: Logs the metric at the current step. Defaults to True in :func:`~~pytorch_lightning.core.lightning.LightningModule.training_step`, and :func:`~pytorch_lightning.core.lightning.LightningModule.training_step_end`. + +* on_epoch: Automatically accumulates and logs at the end of the epoch. Defaults to True anywhere in validation or test loops, and in :func:`~~pytorch_lightning.core.lightning.LightningModule.training_epoch_end`. + +* prog_bar: Logs to the progress bar. + +* logger: Logs to the logger like Tensorboard, or any other custom logger passed to the :class:`~pytorch_lightning.trainer.trainer.Trainer`. + + +.. note:: Setting on_epoch=True will accumulate your logged values over the full training epoch. + + +Manual logging +============== +If you want to log anything that is not a scalar, like histograms, text, images, etc... you may need to use the logger object directly. + +.. code-block:: python + + def training_step(...): + ... + # the logger you used (in this case tensorboard) + tensorboard = self.logger.experiment + tensorboard.add_image() + tensorboard.add_histogram(...) + tensorboard.add_figure(...) + + +Access your logs +================ +Once your training starts, you can view the logs by using your favorite logger or booting up the Tensorboard logs: + +.. code-block:: bash + + tensorboard --logdir ./lightning_logs + +---------- + +*********************** +Logging from a Callback +*********************** +To log from a callback, use the :func:`~~pytorch_lightning.core.lightning.LightningModule.log` +method of the :class:`~pytorch_lightning.core.LightningModule`. + +.. code-block:: python + + class MyCallback(Callback): + + def on_train_epoch_end(self, trainer, pl_module): + pl_module.log('something', x) + +or access the logger object directly for manual logging + +.. code-block:: python + + class MyCallback(Callback): + + def on_train_epoch_end(self, trainer, pl_module): + tensorboard = pl_module.logger.experiment + tensorboard.add_histogram(...) + tensorboard.add_figure(...) + +---------- + +******************** +Make a custom logger +******************** + +You can implement your own logger by writing a class that inherits from +:class:`LightningLoggerBase`. Use the :func:`~pytorch_lightning.loggers.base.rank_zero_only` +decorator to make sure that only the first process in DDP training logs data. + +.. testcode:: + + from pytorch_lightning.utilities import rank_zero_only + from pytorch_lightning.loggers import LightningLoggerBase + + class MyLogger(LightningLoggerBase): + + def name(self): + return 'MyLogger' + + def experiment(self): + # Return the experiment object associated with this logger. + pass + + def version(self): + # Return the experiment version, int or str. + return '0.1' + + @rank_zero_only + def log_hyperparams(self, params): + # params is an argparse.Namespace + # your code to record hyperparameters goes here + pass + + @rank_zero_only + def log_metrics(self, metrics, step): + # metrics is a dictionary of metric names and values + # your code to record metrics goes here + pass + + def save(self): + # Optional. Any code necessary to save logger data goes here + # If you implement this, remember to call `super().save()` + # at the start of the method (important for aggregation of metrics) + super().save() + + @rank_zero_only + def finalize(self, status): + # Optional. Any code that needs to be run after training + # finishes goes here + pass + +If you write a logger that may be useful to others, please send +a pull request to add it to Lightning! + +---------- + +.. _logging_frequency: + + +************************* +Control logging frequency +************************* + +Logging frequency +================= + +It may slow training down to log every single batch. By default, Lightning logs every 50 rows, or 50 training steps. +To change this behaviour, set the `log_every_n_steps` :class:`~pytorch_lightning.trainer.trainer.Trainer` flag. + +.. testcode:: + + k = 10 + trainer = Trainer(log_every_n_steps=k) + + + +Log writing frequency +===================== + +Writing to a logger can be expensive, so by default Lightning write logs to disc or to the given logger every 100 training steps. +To change this behaviour, set the interval at which you wish to flush logs to the filesystem using `log_every_n_steps` :class:`~pytorch_lightning.trainer.trainer.Trainer` flag. + +.. testcode:: + + k = 100 + trainer = Trainer(flush_logs_every_n_steps=k) + +Unlike the `log_every_n_steps`, this argument does not apply to all loggers. +The example shown here works with :class:`~pytorch_lightning.loggers.tensorboard.TensorBoardLogger`, +which is the default logger in Lightning. + +---------- + +************ +Progress Bar +************ +You can add any metric to the progress bar using :func:`~~pytorch_lightning.core.lightning.LightningModule.log` +method, setting `prog_bar=True`. + + +.. code-block:: python + + def training_step(self, batch, batch_idx): + self.log('my_loss', loss, prog_bar=True) + + +Modifying the progress bar +========================== + +The progress bar by default already includes the training loss and version number of the experiment +if you are using a logger. These defaults can be customized by overriding the +:func:`~pytorch_lightning.core.lightning.LightningModule.get_progress_bar_dict` hook in your module. + +.. code-block:: python + + def get_progress_bar_dict(self): + # don't show the version number + items = super().get_progress_bar_dict() + items.pop("v_num", None) + return items + + +---------- + + +************************* +Configure console logging +************************* + +Lightning logs useful information about the training process and user warnings to the console. +You can retrieve the Lightning logger and change it to your liking. For example, increase the logging level +to see fewer messages like so: + +.. code-block:: python + + import logging + logging.getLogger("lightning").setLevel(logging.ERROR) + +Read more about custom Python logging `here `_. + + +---------- + +*********************** +Logging hyperparameters +*********************** + +When training a model, it's useful to know what hyperparams went into that model. +When Lightning creates a checkpoint, it stores a key "hparams" with the hyperparams. + +.. code-block:: python + + lightning_checkpoint = torch.load(filepath, map_location=lambda storage, loc: storage) + hyperparams = lightning_checkpoint['hparams'] + +Some loggers also allow logging the hyperparams used in the experiment. For instance, +when using the TestTubeLogger or the TensorBoardLogger, all hyperparams will show +in the `hparams tab `_. + +---------- + +************* +Snapshot code +************* + +Loggers also allow you to snapshot a copy of the code used in this experiment. +For example, TestTubeLogger does this with a flag: + +.. testcode:: + + from pytorch_lightning.loggers import TestTubeLogger + logger = TestTubeLogger('.', create_git_tag=True) + +---------- + +***************** +Supported Loggers +***************** + +The following are loggers we support + +Comet +===== + +.. autoclass:: pytorch_lightning.loggers.comet.CometLogger + :noindex: + +CSVLogger +========= + +.. autoclass:: pytorch_lightning.loggers.csv_logs.CSVLogger + :noindex: + +MLFlow +====== + +.. autoclass:: pytorch_lightning.loggers.mlflow.MLFlowLogger + :noindex: + +Neptune +======= + +.. autoclass:: pytorch_lightning.loggers.neptune.NeptuneLogger + :noindex: + +Tensorboard +============ + +.. autoclass:: pytorch_lightning.loggers.tensorboard.TensorBoardLogger + :noindex: + +Test-tube +========= + +.. autoclass:: pytorch_lightning.loggers.test_tube.TestTubeLogger + :noindex: + +Weights and Biases +================== + +.. autoclass:: pytorch_lightning.loggers.wandb.WandbLogger + :noindex: diff --git a/pytorch_lightning/trainer/__init__.py b/pytorch_lightning/trainer/__init__.py index ca723709fb..f29a68e9f4 100644 --- a/pytorch_lightning/trainer/__init__.py +++ b/pytorch_lightning/trainer/__init__.py @@ -620,7 +620,7 @@ Writes logs to disk this often. trainer = Trainer(flush_logs_every_n_steps=100) See Also: - - :ref:`Experiment Reporting ` + - :ref:`logging` logger ^^^^^^ @@ -955,7 +955,7 @@ How often to add logging rows (does not write to disk) trainer = Trainer(log_every_n_steps=50) See Also: - - :ref:`Experiment Reporting ` + - :ref:`logging` sync_batchnorm