344 lines
14 KiB
Python
344 lines
14 KiB
Python
"""
|
||
Log using `neptune-logger <https://neptune.ai>`_
|
||
|
||
.. _neptune:
|
||
|
||
NeptuneLogger
|
||
--------------
|
||
"""
|
||
from argparse import Namespace
|
||
from typing import Optional, List, Dict, Any, Union, Iterable
|
||
|
||
try:
|
||
import neptune
|
||
from neptune.experiments import Experiment
|
||
except ImportError: # pragma: no-cover
|
||
raise ImportError('You want to use `neptune` logger which is not installed yet,' # pragma: no-cover
|
||
' install it with `pip install neptune-client`.')
|
||
|
||
import torch
|
||
from torch import is_tensor
|
||
|
||
from pytorch_lightning import _logger as log
|
||
from pytorch_lightning.loggers.base import LightningLoggerBase, rank_zero_only
|
||
|
||
|
||
class NeptuneLogger(LightningLoggerBase):
|
||
r"""
|
||
Neptune logger can be used in the online mode or offline (silent) mode.
|
||
To log experiment data in online mode, NeptuneLogger requries an API key:
|
||
"""
|
||
|
||
def __init__(self, api_key: Optional[str] = None, project_name: Optional[str] = None,
|
||
close_after_fit: Optional[bool] = True, offline_mode: bool = False,
|
||
experiment_name: Optional[str] = None,
|
||
upload_source_files: Optional[List[str]] = None, params: Optional[Dict[str, Any]] = None,
|
||
properties: Optional[Dict[str, Any]] = None, tags: Optional[List[str]] = None, **kwargs):
|
||
r"""
|
||
|
||
Initialize a neptune.ai logger.
|
||
|
||
.. note:: Requires either an API Key (online mode) or a local directory path (offline mode)
|
||
|
||
.. code-block:: python
|
||
|
||
# ONLINE MODE
|
||
from pytorch_lightning.loggers import NeptuneLogger
|
||
# arguments made to NeptuneLogger are passed on to the neptune.experiments.Experiment class
|
||
# We are using an api_key for the anonymous user "neptuner" but you can use your own.
|
||
|
||
neptune_logger = NeptuneLogger(
|
||
api_key="ANONYMOUS"
|
||
project_name="shared/pytorch-lightning-integration",
|
||
experiment_name="default", # Optional,
|
||
params={"max_epochs": 10}, # Optional,
|
||
tags=["pytorch-lightning","mlp"] # Optional,
|
||
)
|
||
trainer = Trainer(max_epochs=10, logger=neptune_logger)
|
||
|
||
.. code-block:: python
|
||
|
||
# OFFLINE MODE
|
||
from pytorch_lightning.loggers import NeptuneLogger
|
||
# arguments made to NeptuneLogger are passed on to the neptune.experiments.Experiment class
|
||
|
||
neptune_logger = NeptuneLogger(
|
||
project_name="USER_NAME/PROJECT_NAME",
|
||
experiment_name="default", # Optional,
|
||
params={"max_epochs": 10}, # Optional,
|
||
tags=["pytorch-lightning","mlp"] # Optional,
|
||
)
|
||
trainer = Trainer(max_epochs=10, logger=neptune_logger)
|
||
|
||
Use the logger anywhere in you LightningModule as follows:
|
||
|
||
.. code-block:: python
|
||
|
||
def train_step(...):
|
||
# example
|
||
self.logger.experiment.log_metric("acc_train", acc_train) # log metrics
|
||
self.logger.experiment.log_image("worse_predictions", prediction_image) # log images
|
||
self.logger.experiment.log_artifact("model_checkpoint.pt", prediction_image) # log model checkpoint
|
||
self.logger.experiment.whatever_neptune_supports(...)
|
||
|
||
def any_lightning_module_function_or_hook(...):
|
||
self.logger.experiment.log_metric("acc_train", acc_train) # log metrics
|
||
self.logger.experiment.log_image("worse_predictions", prediction_image) # log images
|
||
self.logger.experiment.log_artifact("model_checkpoint.pt", prediction_image) # log model checkpoint
|
||
self.logger.experiment.whatever_neptune_supports(...)
|
||
|
||
If you want to log objects after the training is finished use close_after_train=False:
|
||
|
||
.. code-block:: python
|
||
|
||
neptune_logger = NeptuneLogger(
|
||
...
|
||
close_after_fit=False,
|
||
...)
|
||
trainer = Trainer(logger=neptune_logger)
|
||
trainer.fit()
|
||
|
||
# Log test metrics
|
||
trainer.test(model)
|
||
|
||
# Log additional metrics
|
||
from sklearn.metrics import accuracy_score
|
||
|
||
accuracy = accuracy_score(y_true, y_pred)
|
||
neptune_logger.experiment.log_metric('test_accuracy', accuracy)
|
||
|
||
# Log charts
|
||
from scikitplot.metrics import plot_confusion_matrix
|
||
import matplotlib.pyplot as plt
|
||
|
||
fig, ax = plt.subplots(figsize=(16, 12))
|
||
plot_confusion_matrix(y_true, y_pred, ax=ax)
|
||
neptune_logger.experiment.log_image('confusion_matrix', fig)
|
||
|
||
# Save checkpoints folder
|
||
neptune_logger.experiment.log_artifact('my/checkpoints')
|
||
|
||
# When you are done, stop the experiment
|
||
neptune_logger.experiment.stop()
|
||
|
||
You can go and see an example experiment here:
|
||
https://ui.neptune.ai/o/shared/org/pytorch-lightning-integration/e/PYTOR-66/charts
|
||
|
||
Args:
|
||
api_key: Required in online mode.
|
||
Neputne API token, found on https://neptune.ai
|
||
Read how to get your API key
|
||
https://docs.neptune.ai/python-api/tutorials/get-started.html#copy-api-token.
|
||
It is recommended to keep it in the `NEPTUNE_API_TOKEN`
|
||
environment variable and then you can leave `api_key=None`
|
||
project_name: Required in online mode. Qualified name of a project in a form of
|
||
"namespace/project_name" for example "tom/minst-classification".
|
||
If None, the value of NEPTUNE_PROJECT environment variable will be taken.
|
||
You need to create the project in https://neptune.ai first.
|
||
offline_mode: Optional default False. If offline_mode=True no logs will be send
|
||
to neptune. Usually used for debug purposes.
|
||
close_after_fit: Optional default True. If close_after_fit=False the experiment
|
||
will not be closed after training and additional metrics,
|
||
images or artifacts can be logged. Also, remember to close the experiment explicitly
|
||
by running neptune_logger.experiment.stop().
|
||
experiment_name: Optional. Editable name of the experiment.
|
||
Name is displayed in the experiment’s Details (Metadata section) and
|
||
in experiments view as a column.
|
||
upload_source_files: Optional. List of source files to be uploaded.
|
||
Must be list of str or single str. Uploaded sources are displayed
|
||
in the experiment’s Source code tab.
|
||
If None is passed, Python file from which experiment was created will be uploaded.
|
||
Pass empty list ([]) to upload no files.
|
||
Unix style pathname pattern expansion is supported.
|
||
For example, you can pass '\*.py'
|
||
to upload all python source files from the current directory.
|
||
For recursion lookup use '\**/\*.py' (for Python 3.5 and later).
|
||
For more information see glob library.
|
||
params: Optional. Parameters of the experiment.
|
||
After experiment creation params are read-only.
|
||
Parameters are displayed in the experiment’s Parameters section and
|
||
each key-value pair can be viewed in experiments view as a column.
|
||
properties: Optional default is {}. Properties of the experiment.
|
||
They are editable after experiment is created.
|
||
Properties are displayed in the experiment’s Details and
|
||
each key-value pair can be viewed in experiments view as a column.
|
||
tags: Optional default []. Must be list of str. Tags of the experiment.
|
||
They are editable after experiment is created (see: append_tag() and remove_tag()).
|
||
Tags are displayed in the experiment’s Details and can be viewed
|
||
in experiments view as a column.
|
||
"""
|
||
super().__init__()
|
||
self.api_key = api_key
|
||
self.project_name = project_name
|
||
self.offline_mode = offline_mode
|
||
self.close_after_fit = close_after_fit
|
||
self.experiment_name = experiment_name
|
||
self.upload_source_files = upload_source_files
|
||
self.params = params
|
||
self.properties = properties
|
||
self.tags = tags
|
||
self._experiment = None
|
||
self._kwargs = kwargs
|
||
|
||
if offline_mode:
|
||
self.mode = 'offline'
|
||
neptune.init(project_qualified_name='dry-run/project',
|
||
backend=neptune.OfflineBackend())
|
||
else:
|
||
self.mode = 'online'
|
||
neptune.init(api_token=self.api_key,
|
||
project_qualified_name=self.project_name)
|
||
|
||
log.info(f'NeptuneLogger was initialized in {self.mode} mode')
|
||
|
||
def __getstate__(self):
|
||
state = self.__dict__.copy()
|
||
# cannot be pickled
|
||
state['_experiment'] = None
|
||
return state
|
||
|
||
@property
|
||
def experiment(self) -> Experiment:
|
||
r"""
|
||
|
||
Actual neptune object. To use neptune features do the following.
|
||
|
||
Example::
|
||
|
||
self.logger.experiment.some_neptune_function()
|
||
|
||
"""
|
||
|
||
if self._experiment is None:
|
||
self._experiment = neptune.create_experiment(
|
||
name=self.experiment_name,
|
||
params=self.params,
|
||
properties=self.properties,
|
||
tags=self.tags,
|
||
upload_source_files=self.upload_source_files,
|
||
**self._kwargs)
|
||
return self._experiment
|
||
|
||
@rank_zero_only
|
||
def log_hyperparams(self, params: Union[Dict[str, Any], Namespace]) -> None:
|
||
params = self._convert_params(params)
|
||
params = self._flatten_dict(params)
|
||
for key, val in params.items():
|
||
self.experiment.set_property(f'param__{key}', val)
|
||
|
||
@rank_zero_only
|
||
def log_metrics(
|
||
self,
|
||
metrics: Dict[str, Union[torch.Tensor, float]],
|
||
step: Optional[int] = None
|
||
) -> None:
|
||
"""Log metrics (numeric values) in Neptune experiments
|
||
|
||
Args:
|
||
metrics: Dictionary with metric names as keys and measured quantities as values
|
||
step: Step number at which the metrics should be recorded, must be strictly increasing
|
||
"""
|
||
for key, val in metrics.items():
|
||
self.log_metric(key, val, step=step)
|
||
|
||
@rank_zero_only
|
||
def finalize(self, status: str) -> None:
|
||
if self.close_after_fit:
|
||
self.experiment.stop()
|
||
|
||
@property
|
||
def name(self) -> str:
|
||
if self.mode == 'offline':
|
||
return 'offline-name'
|
||
else:
|
||
return self.experiment.name
|
||
|
||
@property
|
||
def version(self) -> str:
|
||
if self.mode == 'offline':
|
||
return 'offline-id-1234'
|
||
else:
|
||
return self.experiment.id
|
||
|
||
@rank_zero_only
|
||
def log_metric(
|
||
self,
|
||
metric_name: str,
|
||
metric_value: Union[torch.Tensor, float, str],
|
||
step: Optional[int] = None
|
||
) -> None:
|
||
"""Log metrics (numeric values) in Neptune experiments
|
||
|
||
Args:
|
||
metric_name: The name of log, i.e. mse, loss, accuracy.
|
||
metric_value: The value of the log (data-point).
|
||
step: Step number at which the metrics should be recorded, must be strictly increasing
|
||
"""
|
||
if is_tensor(metric_value):
|
||
metric_value = metric_value.cpu().detach()
|
||
|
||
if step is None:
|
||
self.experiment.log_metric(metric_name, metric_value)
|
||
else:
|
||
self.experiment.log_metric(metric_name, x=step, y=metric_value)
|
||
|
||
@rank_zero_only
|
||
def log_text(self, log_name: str, text: str, step: Optional[int] = None) -> None:
|
||
"""Log text data in Neptune experiment
|
||
|
||
Args:
|
||
log_name: The name of log, i.e. mse, my_text_data, timing_info.
|
||
text: The value of the log (data-point).
|
||
step: Step number at which the metrics should be recorded, must be strictly increasing
|
||
"""
|
||
self.log_metric(log_name, text, step=step)
|
||
|
||
@rank_zero_only
|
||
def log_image(self, log_name: str, image: Union[str, Any], step: Optional[int] = None) -> None:
|
||
"""Log image data in Neptune experiment
|
||
|
||
Args:
|
||
log_name: The name of log, i.e. bboxes, visualisations, sample_images.
|
||
image (str|PIL.Image|matplotlib.figure.Figure): The value of the log (data-point).
|
||
Can be one of the following types: PIL image, matplotlib.figure.Figure, path to image file (str)
|
||
step: Step number at which the metrics should be recorded, must be strictly increasing
|
||
"""
|
||
if step is None:
|
||
self.experiment.log_image(log_name, image)
|
||
else:
|
||
self.experiment.log_image(log_name, x=step, y=image)
|
||
|
||
@rank_zero_only
|
||
def log_artifact(self, artifact: str, destination: Optional[str] = None) -> None:
|
||
"""Save an artifact (file) in Neptune experiment storage.
|
||
|
||
Args:
|
||
artifact: A path to the file in local filesystem.
|
||
destination: Optional default None. A destination path.
|
||
If None is passed, an artifact file name will be used.
|
||
"""
|
||
self.experiment.log_artifact(artifact, destination)
|
||
|
||
@rank_zero_only
|
||
def set_property(self, key: str, value: Any) -> None:
|
||
"""Set key-value pair as Neptune experiment property.
|
||
|
||
Args:
|
||
key: Property key.
|
||
value: New value of a property.
|
||
"""
|
||
self.experiment.set_property(key, value)
|
||
|
||
@rank_zero_only
|
||
def append_tags(self, tags: Union[str, Iterable[str]]) -> None:
|
||
"""appends tags to neptune experiment
|
||
|
||
Args:
|
||
tags: Tags to add to the current experiment. If str is passed, singe tag is added.
|
||
If multiple - comma separated - str are passed, all of them are added as tags.
|
||
If list of str is passed, all elements of the list are added as tags.
|
||
"""
|
||
if str(tags) == tags:
|
||
tags = [tags] # make it as an iterable is if it is not yet
|
||
self.experiment.append_tags(*tags)
|