diff --git a/src/pytorch_lightning/CHANGELOG.md b/src/pytorch_lightning/CHANGELOG.md index f014739e49..e5081e4993 100644 --- a/src/pytorch_lightning/CHANGELOG.md +++ b/src/pytorch_lightning/CHANGELOG.md @@ -281,6 +281,12 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). - Removed support for the `DDP2Strategy` ([#12705](https://github.com/PyTorchLightning/pytorch-lightning/pull/12705)) +- Removed deprecated ClusterEnvironment properties `master_address` and `master_port` in favor of `main_address` and `main_port` ([#13458](https://github.com/PyTorchLightning/pytorch-lightning/pull/13458)) + + +- Removed deprecated ClusterEnvironment methods `KubeflowEnvironment.is_using_kubelfow()`, `LSFEnvironment.is_using_lsf()` and `TorchElasticEnvironment.is_using_torchelastic()` in favor of the `detect()` method ([#13458](https://github.com/PyTorchLightning/pytorch-lightning/pull/13458)) + + - Removed deprecated `Callback.on_keyboard_interrupt` ([#13438](https://github.com/Lightning-AI/lightning/pull/13438)) diff --git a/src/pytorch_lightning/plugins/environments/cluster_environment.py b/src/pytorch_lightning/plugins/environments/cluster_environment.py index 1871f0afdf..9660708653 100644 --- a/src/pytorch_lightning/plugins/environments/cluster_environment.py +++ b/src/pytorch_lightning/plugins/environments/cluster_environment.py @@ -12,19 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. from abc import ABC, abstractmethod -from typing import Any, Type - -from pytorch_lightning.utilities import rank_zero_deprecation class ClusterEnvironment(ABC): """Specification of a cluster environment.""" - def __new__(cls, *args: Any, **kwargs: Any) -> "ClusterEnvironment": - # TODO: remove in 1.7 - _check_for_deprecated_methods(cls) - return super().__new__(cls) - @property @abstractmethod def creates_processes_externally(self) -> bool: @@ -72,16 +64,3 @@ class ClusterEnvironment(ABC): def teardown(self) -> None: """Clean up any state set after execution finishes.""" pass - - -def _check_for_deprecated_methods(cls: Type[ClusterEnvironment]) -> None: - if hasattr(cls, "master_address") and callable(cls.master_address): - rank_zero_deprecation( - f"`{cls.__name__}.master_address` has been deprecated in v1.6 and will be removed in v1.7." - " Implement the property `main_address` instead (do not forget to add the `@property` decorator)." - ) - if hasattr(cls, "master_port") and callable(cls.master_port): - rank_zero_deprecation( - f"`{cls.__name__}.master_port` has been deprecated in v1.6 and will be removed in v1.7." - " Implement the property `main_port` instead (do not forget to add the `@property` decorator)." - ) diff --git a/src/pytorch_lightning/plugins/environments/kubeflow_environment.py b/src/pytorch_lightning/plugins/environments/kubeflow_environment.py index 03dfdde9d7..4e38f6d082 100644 --- a/src/pytorch_lightning/plugins/environments/kubeflow_environment.py +++ b/src/pytorch_lightning/plugins/environments/kubeflow_environment.py @@ -16,7 +16,6 @@ import logging import os from pytorch_lightning.plugins.environments.cluster_environment import ClusterEnvironment -from pytorch_lightning.utilities import rank_zero_deprecation log = logging.getLogger(__name__) @@ -28,16 +27,6 @@ class KubeflowEnvironment(ClusterEnvironment): .. _Kubeflow: https://www.kubeflow.org """ - def __init__(self) -> None: - super().__init__() - # TODO: remove in 1.7 - if hasattr(self, "is_using_kubeflow") and callable(self.is_using_kubeflow): - rank_zero_deprecation( - f"`{self.__class__.__name__}.is_using_kubeflow` has been deprecated in v1.6 and will be removed in" - f" v1.7. Implement the static method `detect()` instead (do not forget to add the `@staticmethod`" - f" decorator)." - ) - @property def creates_processes_externally(self) -> bool: return True diff --git a/src/pytorch_lightning/plugins/environments/lsf_environment.py b/src/pytorch_lightning/plugins/environments/lsf_environment.py index 150328dbf7..359add5137 100644 --- a/src/pytorch_lightning/plugins/environments/lsf_environment.py +++ b/src/pytorch_lightning/plugins/environments/lsf_environment.py @@ -18,7 +18,6 @@ from typing import Dict, List from pytorch_lightning import _logger as log from pytorch_lightning.plugins.environments import ClusterEnvironment -from pytorch_lightning.utilities import rank_zero_deprecation from pytorch_lightning.utilities.cloud_io import get_filesystem @@ -48,12 +47,6 @@ class LSFEnvironment(ClusterEnvironment): def __init__(self) -> None: super().__init__() - # TODO: remove in 1.7 - if hasattr(self, "is_using_lsf") and callable(self.is_using_lsf): - rank_zero_deprecation( - f"`{self.__class__.__name__}.is_using_lsf` has been deprecated in v1.6 and will be removed in v1.7." - " Implement the static method `detect()` instead (do not forget to add the `@staticmethod` decorator)." - ) self._main_address = self._get_main_address() self._main_port = self._get_main_port() self._node_rank = self._get_node_rank() diff --git a/src/pytorch_lightning/plugins/environments/torchelastic_environment.py b/src/pytorch_lightning/plugins/environments/torchelastic_environment.py index 98cad39a0a..2cd3f408f4 100644 --- a/src/pytorch_lightning/plugins/environments/torchelastic_environment.py +++ b/src/pytorch_lightning/plugins/environments/torchelastic_environment.py @@ -19,7 +19,7 @@ import torch.distributed from pytorch_lightning.plugins.environments.cluster_environment import ClusterEnvironment from pytorch_lightning.utilities.imports import _TORCH_GREATER_EQUAL_1_9_1 -from pytorch_lightning.utilities.rank_zero import rank_zero_deprecation, rank_zero_warn +from pytorch_lightning.utilities.rank_zero import rank_zero_warn log = logging.getLogger(__name__) @@ -27,16 +27,6 @@ log = logging.getLogger(__name__) class TorchElasticEnvironment(ClusterEnvironment): """Environment for fault-tolerant and elastic training with `torchelastic `_""" - def __init__(self) -> None: - super().__init__() - # TODO: remove in 1.7 - if hasattr(self, "is_using_torchelastic") and callable(self.is_using_torchelastic): - rank_zero_deprecation( - f"`{self.__class__.__name__}.is_using_torchelastic` has been deprecated in v1.6 and will be removed in" - " v1.7. Implement the static method `detect()` instead (do not forget to add the `@staticmethod`" - " decorator)." - ) - @property def creates_processes_externally(self) -> bool: return True diff --git a/src/pytorch_lightning/trainer/connectors/accelerator_connector.py b/src/pytorch_lightning/trainer/connectors/accelerator_connector.py index f1accaa29e..2e112c754c 100644 --- a/src/pytorch_lightning/trainer/connectors/accelerator_connector.py +++ b/src/pytorch_lightning/trainer/connectors/accelerator_connector.py @@ -546,7 +546,8 @@ class AcceleratorConnector: return SLURMEnvironment() for env_type in (BaguaEnvironment, TorchElasticEnvironment, KubeflowEnvironment, LSFEnvironment): if env_type.detect(): - return env_type() + # Ignore type error because it is a false positive: https://github.com/python/mypy/issues/13044 + return env_type() # type: ignore[abstract] return LightningEnvironment() def _is_slurm_managing_tasks(self) -> bool: diff --git a/tests/tests_pytorch/deprecated_api/test_remove_1-7.py b/tests/tests_pytorch/deprecated_api/test_remove_1-7.py index bd642d9dbf..2af4e4c8e2 100644 --- a/tests/tests_pytorch/deprecated_api/test_remove_1-7.py +++ b/tests/tests_pytorch/deprecated_api/test_remove_1-7.py @@ -12,23 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. """Test deprecated functionality which will be removed in v1.7.0.""" -import os from re import escape -from unittest import mock import pytest import torch from pytorch_lightning import Trainer -from pytorch_lightning.plugins.environments import ( - KubeflowEnvironment, - LightningEnvironment, - LSFEnvironment, - SLURMEnvironment, - TorchElasticEnvironment, -) from pytorch_lightning.strategies import SingleDeviceStrategy -from tests_pytorch.plugins.environments.test_lsf_environment import _make_rankfile def test_v1_7_0_deprecate_lightning_distributed(tmpdir): @@ -47,83 +37,6 @@ def test_v1_7_0_deprecated_max_steps_none(tmpdir): trainer.fit_loop.max_steps = None -@pytest.mark.parametrize( - "cls", - [ - KubeflowEnvironment, - LightningEnvironment, - SLURMEnvironment, - TorchElasticEnvironment, - ], -) -def test_v1_7_0_cluster_environment_master_address(cls): - class MyClusterEnvironment(cls): - def master_address(self): - pass - - with pytest.deprecated_call( - match="MyClusterEnvironment.master_address` has been deprecated in v1.6 and will be removed in v1.7" - ): - MyClusterEnvironment() - - -@pytest.mark.parametrize( - "cls", - [ - KubeflowEnvironment, - LightningEnvironment, - SLURMEnvironment, - TorchElasticEnvironment, - ], -) -def test_v1_7_0_cluster_environment_master_port(cls): - class MyClusterEnvironment(cls): - def master_port(self): - pass - - with pytest.deprecated_call( - match="MyClusterEnvironment.master_port` has been deprecated in v1.6 and will be removed in v1.7" - ): - MyClusterEnvironment() - - -@pytest.mark.parametrize( - "cls,method_name", - [ - (KubeflowEnvironment, "is_using_kubeflow"), - (LSFEnvironment, "is_using_lsf"), - (TorchElasticEnvironment, "is_using_torchelastic"), - ], -) -def test_v1_7_0_cluster_environment_detection(cls, method_name, tmp_path): - class MyClusterEnvironment(cls): - @staticmethod - def is_using_kubeflow(): - pass - - @staticmethod - def is_using_lsf(): - pass - - @staticmethod - def is_using_torchelastic(): - pass - - environ = { - "LSB_DJOB_RANKFILE": _make_rankfile(tmp_path), - "LSB_JOBID": "1234", - "JSM_NAMESPACE_SIZE": "4", - "JSM_NAMESPACE_RANK": "3", - "JSM_NAMESPACE_LOCAL_RANK": "1", - } - with mock.patch.dict(os.environ, environ): - with mock.patch("socket.gethostname", return_value="10.10.10.2"): - with pytest.deprecated_call( - match=f"MyClusterEnvironment.{method_name}` has been deprecated in v1.6 and will be removed in v1.7" - ): - MyClusterEnvironment() - - def test_v1_7_0_post_dispatch_hook(): class CustomPlugin(SingleDeviceStrategy): def post_dispatch(self, trainer): diff --git a/tests/tests_pytorch/plugins/environments/test_lightning_environment.py b/tests/tests_pytorch/plugins/environments/test_lightning_environment.py index 9197e51583..0ad05e0bfa 100644 --- a/tests/tests_pytorch/plugins/environments/test_lightning_environment.py +++ b/tests/tests_pytorch/plugins/environments/test_lightning_environment.py @@ -66,7 +66,7 @@ def test_node_rank_from_group_rank(): @mock.patch.dict(os.environ, {}, clear=True) -def test_random_master_port(): +def test_random_main_port(): """Test randomly chosen main port when no main port was given by user.""" env = LightningEnvironment() port = env.main_port diff --git a/tests/tests_pytorch/plugins/environments/test_slurm_environment.py b/tests/tests_pytorch/plugins/environments/test_slurm_environment.py index 61ef404dd1..44b0dd97e3 100644 --- a/tests/tests_pytorch/plugins/environments/test_slurm_environment.py +++ b/tests/tests_pytorch/plugins/environments/test_slurm_environment.py @@ -83,7 +83,7 @@ def test_attributes_from_environment_variables(caplog): "slurm_node_list,expected", [("alpha,beta,gamma", "alpha"), ("alpha beta gamma", "alpha"), ("1.2.3.[100-110]", "1.2.3.100")], ) -def test_master_address_from_slurm_node_list(slurm_node_list, expected): +def test_main_address_from_slurm_node_list(slurm_node_list, expected): """Test extracting the main node from different formats for the SLURM_NODELIST.""" with mock.patch.dict(os.environ, {"SLURM_NODELIST": slurm_node_list}): env = SLURMEnvironment()