Rooba
/
lightning
mirror of https://github.com/Lightning-AI/lightning.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Remove deprecated ClustertEnvironment methods (#13458) 

* Remove deprecated ClustertEnvironment methods

* update changelog

* ignore typing error

Co-authored-by: Akihiro Nitta <nitta@akihironitta.com>
This commit is contained in:
Adrian Wälchli 2022-07-13 21:53:46 +02:00 committed by GitHub
parent feb8e7d344
commit daf7cec01e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 11 additions and 140 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
src/pytorch_lightning/CHANGELOG.md
View File

@ -281,6 +281,12 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/).
- Removed support for the `DDP2Strategy` ([#12705](https://github.com/PyTorchLightning/pytorch-lightning/pull/12705))
- Removed deprecated ClusterEnvironment properties `master_address` and `master_port` in favor of `main_address` and `main_port` ([#13458](https://github.com/PyTorchLightning/pytorch-lightning/pull/13458))
- Removed deprecated ClusterEnvironment methods `KubeflowEnvironment.is_using_kubelfow()`, `LSFEnvironment.is_using_lsf()` and `TorchElasticEnvironment.is_using_torchelastic()` in favor of the `detect()` method ([#13458](https://github.com/PyTorchLightning/pytorch-lightning/pull/13458))
- Removed deprecated `Callback.on_keyboard_interrupt` ([#13438](https://github.com/Lightning-AI/lightning/pull/13438))

21
src/pytorch_lightning/plugins/environments/cluster_environment.py
View File

@ -12,19 +12,11 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from abc import ABC, abstractmethod
from typing import Any, Type
from pytorch_lightning.utilities import rank_zero_deprecation
class ClusterEnvironment(ABC):
"""Specification of a cluster environment."""
def __new__(cls, *args: Any, **kwargs: Any) -> "ClusterEnvironment":
# TODO: remove in 1.7
_check_for_deprecated_methods(cls)
return super().__new__(cls)
@property
@abstractmethod
def creates_processes_externally(self) -> bool:
@ -72,16 +64,3 @@ class ClusterEnvironment(ABC):
def teardown(self) -> None:
"""Clean up any state set after execution finishes."""
pass
def _check_for_deprecated_methods(cls: Type[ClusterEnvironment]) -> None:
if hasattr(cls, "master_address") and callable(cls.master_address):
rank_zero_deprecation(
f"`{cls.__name__}.master_address` has been deprecated in v1.6 and will be removed in v1.7."
" Implement the property `main_address` instead (do not forget to add the `@property` decorator)."
)
if hasattr(cls, "master_port") and callable(cls.master_port):
rank_zero_deprecation(
f"`{cls.__name__}.master_port` has been deprecated in v1.6 and will be removed in v1.7."
" Implement the property `main_port` instead (do not forget to add the `@property` decorator)."
)

11
src/pytorch_lightning/plugins/environments/kubeflow_environment.py
View File

@ -16,7 +16,6 @@ import logging
import os
from pytorch_lightning.plugins.environments.cluster_environment import ClusterEnvironment
from pytorch_lightning.utilities import rank_zero_deprecation
log = logging.getLogger(__name__)
@ -28,16 +27,6 @@ class KubeflowEnvironment(ClusterEnvironment):
.. _Kubeflow: https://www.kubeflow.org
"""
def __init__(self) -> None:
super().__init__()
# TODO: remove in 1.7
if hasattr(self, "is_using_kubeflow") and callable(self.is_using_kubeflow):
rank_zero_deprecation(
f"`{self.__class__.__name__}.is_using_kubeflow` has been deprecated in v1.6 and will be removed in"
f" v1.7. Implement the static method `detect()` instead (do not forget to add the `@staticmethod`"
f" decorator)."
)
@property
def creates_processes_externally(self) -> bool:
return True

7
src/pytorch_lightning/plugins/environments/lsf_environment.py
View File

@ -18,7 +18,6 @@ from typing import Dict, List
from pytorch_lightning import _logger as log
from pytorch_lightning.plugins.environments import ClusterEnvironment
from pytorch_lightning.utilities import rank_zero_deprecation
from pytorch_lightning.utilities.cloud_io import get_filesystem
@ -48,12 +47,6 @@ class LSFEnvironment(ClusterEnvironment):
def __init__(self) -> None:
super().__init__()
# TODO: remove in 1.7
if hasattr(self, "is_using_lsf") and callable(self.is_using_lsf):
rank_zero_deprecation(
f"`{self.__class__.__name__}.is_using_lsf` has been deprecated in v1.6 and will be removed in v1.7."
" Implement the static method `detect()` instead (do not forget to add the `@staticmethod` decorator)."
)
self._main_address = self._get_main_address()
self._main_port = self._get_main_port()
self._node_rank = self._get_node_rank()

12
src/pytorch_lightning/plugins/environments/torchelastic_environment.py
View File

@ -19,7 +19,7 @@ import torch.distributed
from pytorch_lightning.plugins.environments.cluster_environment import ClusterEnvironment
from pytorch_lightning.utilities.imports import _TORCH_GREATER_EQUAL_1_9_1
from pytorch_lightning.utilities.rank_zero import rank_zero_deprecation, rank_zero_warn
from pytorch_lightning.utilities.rank_zero import rank_zero_warn
log = logging.getLogger(__name__)
@ -27,16 +27,6 @@ log = logging.getLogger(__name__)
class TorchElasticEnvironment(ClusterEnvironment):
"""Environment for fault-tolerant and elastic training with `torchelastic <https://pytorch.org/elastic/>`_"""
def __init__(self) -> None:
super().__init__()
# TODO: remove in 1.7
if hasattr(self, "is_using_torchelastic") and callable(self.is_using_torchelastic):
rank_zero_deprecation(
f"`{self.__class__.__name__}.is_using_torchelastic` has been deprecated in v1.6 and will be removed in"
" v1.7. Implement the static method `detect()` instead (do not forget to add the `@staticmethod`"
" decorator)."
)
@property
def creates_processes_externally(self) -> bool:
return True

3
src/pytorch_lightning/trainer/connectors/accelerator_connector.py
View File

@ -546,7 +546,8 @@ class AcceleratorConnector:
return SLURMEnvironment()
for env_type in (BaguaEnvironment, TorchElasticEnvironment, KubeflowEnvironment, LSFEnvironment):
if env_type.detect():
return env_type()
# Ignore type error because it is a false positive: https://github.com/python/mypy/issues/13044
return env_type() # type: ignore[abstract]
return LightningEnvironment()
def _is_slurm_managing_tasks(self) -> bool:

87
tests/tests_pytorch/deprecated_api/test_remove_1-7.py
View File

@ -12,23 +12,13 @@
# See the License for the specific language governing permissions and
# limitations under the License.
"""Test deprecated functionality which will be removed in v1.7.0."""
import os
from re import escape
from unittest import mock
import pytest
import torch
from pytorch_lightning import Trainer
from pytorch_lightning.plugins.environments import (
KubeflowEnvironment,
LightningEnvironment,
LSFEnvironment,
SLURMEnvironment,
TorchElasticEnvironment,
)
from pytorch_lightning.strategies import SingleDeviceStrategy
from tests_pytorch.plugins.environments.test_lsf_environment import _make_rankfile
def test_v1_7_0_deprecate_lightning_distributed(tmpdir):
@ -47,83 +37,6 @@ def test_v1_7_0_deprecated_max_steps_none(tmpdir):
trainer.fit_loop.max_steps = None
@pytest.mark.parametrize(
"cls",
[
KubeflowEnvironment,
LightningEnvironment,
SLURMEnvironment,
TorchElasticEnvironment,
],
)
def test_v1_7_0_cluster_environment_master_address(cls):
class MyClusterEnvironment(cls):
def master_address(self):
pass
with pytest.deprecated_call(
match="MyClusterEnvironment.master_address` has been deprecated in v1.6 and will be removed in v1.7"
):
MyClusterEnvironment()
@pytest.mark.parametrize(
"cls",
[
KubeflowEnvironment,
LightningEnvironment,
SLURMEnvironment,
TorchElasticEnvironment,
],
)
def test_v1_7_0_cluster_environment_master_port(cls):
class MyClusterEnvironment(cls):
def master_port(self):
pass
with pytest.deprecated_call(
match="MyClusterEnvironment.master_port` has been deprecated in v1.6 and will be removed in v1.7"
):
MyClusterEnvironment()
@pytest.mark.parametrize(
"cls,method_name",
[
(KubeflowEnvironment, "is_using_kubeflow"),
(LSFEnvironment, "is_using_lsf"),
(TorchElasticEnvironment, "is_using_torchelastic"),
],
)
def test_v1_7_0_cluster_environment_detection(cls, method_name, tmp_path):
class MyClusterEnvironment(cls):
@staticmethod
def is_using_kubeflow():
pass
@staticmethod
def is_using_lsf():
pass
@staticmethod
def is_using_torchelastic():
pass
environ = {
"LSB_DJOB_RANKFILE": _make_rankfile(tmp_path),
"LSB_JOBID": "1234",
"JSM_NAMESPACE_SIZE": "4",
"JSM_NAMESPACE_RANK": "3",
"JSM_NAMESPACE_LOCAL_RANK": "1",
}
with mock.patch.dict(os.environ, environ):
with mock.patch("socket.gethostname", return_value="10.10.10.2"):
with pytest.deprecated_call(
match=f"MyClusterEnvironment.{method_name}` has been deprecated in v1.6 and will be removed in v1.7"
):
MyClusterEnvironment()
def test_v1_7_0_post_dispatch_hook():
class CustomPlugin(SingleDeviceStrategy):
def post_dispatch(self, trainer):

2
tests/tests_pytorch/plugins/environments/test_lightning_environment.py
View File

@ -66,7 +66,7 @@ def test_node_rank_from_group_rank():
@mock.patch.dict(os.environ, {}, clear=True)
def test_random_master_port():
def test_random_main_port():
"""Test randomly chosen main port when no main port was given by user."""
env = LightningEnvironment()
port = env.main_port

2
tests/tests_pytorch/plugins/environments/test_slurm_environment.py
View File

@ -83,7 +83,7 @@ def test_attributes_from_environment_variables(caplog):
"slurm_node_list,expected",
[("alpha,beta,gamma", "alpha"), ("alpha beta gamma", "alpha"), ("1.2.3.[100-110]", "1.2.3.100")],
)
def test_master_address_from_slurm_node_list(slurm_node_list, expected):
def test_main_address_from_slurm_node_list(slurm_node_list, expected):
"""Test extracting the main node from different formats for the SLURM_NODELIST."""
with mock.patch.dict(os.environ, {"SLURM_NODELIST": slurm_node_list}):
env = SLURMEnvironment()