From 104290efa5b4f300a562c3f345fdea103f391798 Mon Sep 17 00:00:00 2001 From: thomas chaton Date: Tue, 14 Feb 2023 14:12:51 +0000 Subject: [PATCH] [App] Add rm one level below project level (#16740) Co-authored-by: Ethan Harris Co-authored-by: Justus Schock <12886177+justusschock@users.noreply.github.com> Co-authored-by: thomas --- examples/app_commands_and_api/app.py | 4 +- src/lightning/app/CHANGELOG.md | 3 + src/lightning/app/cli/commands/cp.py | 7 +- src/lightning/app/cli/commands/ls.py | 3 +- src/lightning/app/cli/commands/rm.py | 104 +++++++++++++++++ src/lightning/app/cli/lightning_cli.py | 2 + src/lightning/app/testing/testing.py | 5 +- src/lightning/app/utilities/cli_helpers.py | 2 +- .../public/test_commands_and_api.py | 2 +- tests/tests_app/cli/test_rm.py | 109 ++++++++++++++++++ tests/tests_app/utilities/test_commands.py | 3 +- 11 files changed, 236 insertions(+), 8 deletions(-) create mode 100644 src/lightning/app/cli/commands/rm.py create mode 100644 tests/tests_app/cli/test_rm.py diff --git a/examples/app_commands_and_api/app.py b/examples/app_commands_and_api/app.py index 8c62510dea..a661663e8d 100644 --- a/examples/app_commands_and_api/app.py +++ b/examples/app_commands_and_api/app.py @@ -1,8 +1,8 @@ from command import CustomCommand, CustomConfig from lightning import LightningFlow -from lightning_app.api import Get, Post -from lightning_app.core.app import LightningApp +from lightning.app.api import Get, Post +from lightning.app.core.app import LightningApp async def handler(): diff --git a/src/lightning/app/CHANGELOG.md b/src/lightning/app/CHANGELOG.md index 00038be348..76545f0b31 100644 --- a/src/lightning/app/CHANGELOG.md +++ b/src/lightning/app/CHANGELOG.md @@ -9,6 +9,9 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). ### Added +- Added Storage Commands ([#16740](https://github.com/Lightning-AI/lightning/pull/16740)) + * `rm`: Delete files from your Cloud Platform Filesystem + - Added `lightning connect data` to register data connection to private s3 buckets ([#16738](https://github.com/Lightning-AI/lightning/pull/16738)) diff --git a/src/lightning/app/cli/commands/cp.py b/src/lightning/app/cli/commands/cp.py index 6cba363c9f..766f35b255 100644 --- a/src/lightning/app/cli/commands/cp.py +++ b/src/lightning/app/cli/commands/cp.py @@ -57,7 +57,7 @@ def cp(src_path: str, dst_path: str, r: bool = False, recursive: bool = False) - if pwd == "/" or len(pwd.split("/")) == 1: return _error_and_exit("Uploading files at the project level isn't allowed yet.") - client = LightningClient() + client = LightningClient(retry=False) src_path, src_remote = _sanitize_path(src_path, pwd) dst_path, dst_remote = _sanitize_path(dst_path, pwd) @@ -87,6 +87,9 @@ def _upload_files(live, client: LightningClient, local_src: str, remote_dst: str else: project_id = _get_project_id_from_name(remote_dst) + if len(remote_splits) > 2: + remote_dst = os.path.join(*remote_splits[2:]) + local_src = Path(local_src).resolve() upload_paths = [] @@ -101,6 +104,8 @@ def _upload_files(live, client: LightningClient, local_src: str, remote_dst: str clusters = client.projects_service_list_project_cluster_bindings(project_id) + live.stop() + for upload_path in upload_paths: for cluster in clusters.clusters: filename = str(upload_path).replace(str(os.getcwd()), "")[1:] diff --git a/src/lightning/app/cli/commands/ls.py b/src/lightning/app/cli/commands/ls.py index 96f09c6fc4..8af8f22cd1 100644 --- a/src/lightning/app/cli/commands/ls.py +++ b/src/lightning/app/cli/commands/ls.py @@ -18,6 +18,7 @@ from contextlib import nullcontext from typing import Generator, List, Optional import click +import lightning_cloud import rich from lightning_cloud.openapi import Externalv1LightningappInstance from rich.console import Console @@ -255,7 +256,7 @@ def _collect_artifacts( page_token=response.next_page_token, tokens=tokens, ) - except Exception: + except lightning_cloud.openapi.rest.ApiException: # Note: This is triggered when the request is wrong. # This is currently happening due to looping through the user clusters. pass diff --git a/src/lightning/app/cli/commands/rm.py b/src/lightning/app/cli/commands/rm.py new file mode 100644 index 0000000000..9127515713 --- /dev/null +++ b/src/lightning/app/cli/commands/rm.py @@ -0,0 +1,104 @@ +# Copyright The Lightning AI team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import click +import lightning_cloud +import rich + +from lightning.app.cli.commands.ls import _add_colors, _get_prefix +from lightning.app.cli.commands.pwd import _pwd +from lightning.app.utilities.app_helpers import Logger +from lightning.app.utilities.cli_helpers import _error_and_exit +from lightning.app.utilities.network import LightningClient + +logger = Logger(__name__) + + +@click.argument("rm_path", required=True) +@click.option("-r", required=False, hidden=True) +@click.option("--recursive", required=False, hidden=True) +def rm(rm_path: str, r: bool = False, recursive: bool = False) -> None: + """Delete files on the Lightning Cloud filesystem.""" + + root = _pwd() + + if rm_path in (".", ".."): + return _error_and_exit('rm "." and ".." may not be removed') + + if ".." in rm_path: + return _error_and_exit('rm ".." or higher may not be removed') + + root = os.path.join(root, rm_path) + splits = [split for split in root.split("/") if split != ""] + + if root == "/" or len(splits) == 1: + return _error_and_exit("rm at the project level isn't supported") + + client = LightningClient(retry=False) + projects = client.projects_service_list_memberships() + + project = [project for project in projects.memberships if project.name == splits[0]] + + # This happens if the user changes cluster and the project doesn't exist. + if len(project) == 0: + return _error_and_exit( + f"There isn't any Lightning Project matching the name {splits[0]}." " HINT: Use `lightning cd`." + ) + + project_id = project[0].project_id + + # Parallelise calls + lit_apps = client.lightningapp_instance_service_list_lightningapp_instances(project_id=project_id, async_req=True) + lit_cloud_spaces = client.cloud_space_service_list_cloud_spaces(project_id=project_id, async_req=True) + + lit_apps = lit_apps.get().lightningapps + lit_cloud_spaces = lit_cloud_spaces.get().cloudspaces + + lit_ressources = [lit_resource for lit_resource in lit_cloud_spaces if lit_resource.name == splits[1]] + + if len(lit_ressources) == 0: + + lit_ressources = [lit_resource for lit_resource in lit_apps if lit_resource.name == splits[1]] + + if len(lit_ressources) == 0: + _error_and_exit(f"There isn't any Lightning Ressource matching the name {splits[1]}.") + + lit_resource = lit_ressources[0] + + prefix = "/".join(splits[2:]) + prefix = _get_prefix(prefix, lit_resource) + + clusters = client.projects_service_list_project_cluster_bindings(project_id) + succeeded = False + + for cluster in clusters.clusters: + try: + client.lightningapp_instance_service_delete_project_artifact( + project_id=project_id, + cluster_id=cluster.cluster_id, + filename=prefix, + ) + succeeded = True + break + except lightning_cloud.openapi.rest.ApiException: + pass + + prefix = os.path.join(*splits) + + if succeeded: + rich.print(_add_colors(f"Successfuly deleted `{prefix}`.", color="green")) + else: + return _error_and_exit(f"No file or folder named `{prefix}` was found.") diff --git a/src/lightning/app/cli/lightning_cli.py b/src/lightning/app/cli/lightning_cli.py index 5b81253481..482dbdaf5a 100644 --- a/src/lightning/app/cli/lightning_cli.py +++ b/src/lightning/app/cli/lightning_cli.py @@ -38,6 +38,7 @@ from lightning.app.cli.commands.cp import cp from lightning.app.cli.commands.logs import logs from lightning.app.cli.commands.ls import ls from lightning.app.cli.commands.pwd import pwd +from lightning.app.cli.commands.rm import rm from lightning.app.cli.connect.app import ( _list_app_commands, _retrieve_connection_to_an_app, @@ -144,6 +145,7 @@ _main.command(hidden=True)(ls) _main.command(hidden=True)(cd) _main.command(hidden=True)(cp) _main.command(hidden=True)(pwd) +_main.command(hidden=True)(rm) show.command()(logs) diff --git a/src/lightning/app/testing/testing.py b/src/lightning/app/testing/testing.py index f37e4b9853..dce3a0ac24 100644 --- a/src/lightning/app/testing/testing.py +++ b/src/lightning/app/testing/testing.py @@ -282,7 +282,7 @@ def run_app_in_cloud( token = res.json()["token"] # 3. Disconnect from the App if any. - Popen("lightning disconnect", shell=True).wait() + Popen("lightning logout", shell=True).wait() # 4. Launch the application in the cloud from the Lightning CLI. with tempfile.TemporaryDirectory() as tmpdir: @@ -392,6 +392,9 @@ def run_app_in_cloud( admin_page.locator(f'[data-cy="{name}"]').click() + app_url = admin_page.url + admin_page.goto(app_url + "/logs") + client = LightningClient() project_id = _get_project(client).project_id diff --git a/src/lightning/app/utilities/cli_helpers.py b/src/lightning/app/utilities/cli_helpers.py index 97aff7e3d8..270b6e0953 100644 --- a/src/lightning/app/utilities/cli_helpers.py +++ b/src/lightning/app/utilities/cli_helpers.py @@ -357,6 +357,6 @@ def _check_environment_and_redirect(): return -def _error_and_exit(msg: str) -> str: +def _error_and_exit(msg: str) -> None: rich.print(f"[red]ERROR[/red]: {msg}") sys.exit(0) diff --git a/tests/integrations_app/public/test_commands_and_api.py b/tests/integrations_app/public/test_commands_and_api.py index 16e9b071b8..226b80bfd3 100644 --- a/tests/integrations_app/public/test_commands_and_api.py +++ b/tests/integrations_app/public/test_commands_and_api.py @@ -23,7 +23,7 @@ def test_commands_and_api_example_cloud() -> None: cmd_2 = "python -m lightning command with client --name=this" cmd_3 = "python -m lightning command without client --name=is" cmd_4 = "python -m lightning command without client --name=awesome" - cmd_5 = "lightning disconnect app" + cmd_5 = "lightning logout" process = Popen(" && ".join([cmd_1, cmd_2, cmd_3, cmd_4, cmd_5]), shell=True) process.wait() diff --git a/tests/tests_app/cli/test_rm.py b/tests/tests_app/cli/test_rm.py new file mode 100644 index 0000000000..6d9dcfa90b --- /dev/null +++ b/tests/tests_app/cli/test_rm.py @@ -0,0 +1,109 @@ +import os +import sys +from unittest.mock import MagicMock + +import pytest +from lightning_cloud.openapi import ( + Externalv1LightningappInstance, + V1LightningappInstanceArtifact, + V1ListCloudSpacesResponse, + V1ListLightningappInstanceArtifactsResponse, + V1ListLightningappInstancesResponse, + V1ListMembershipsResponse, + V1Membership, +) + +from lightning.app.cli.commands import cd, ls, rm + + +@pytest.mark.skipif(sys.platform == "win32", reason="not supported on windows yet") +def test_rm(monkeypatch): + """This test validates rm behaves as expected.""" + + if os.path.exists(cd._CD_FILE): + os.remove(cd._CD_FILE) + + client = MagicMock() + client.projects_service_list_memberships.return_value = V1ListMembershipsResponse( + memberships=[ + V1Membership(name="project-0", project_id="project-id-0"), + V1Membership(name="project-1", project_id="project-id-1"), + V1Membership(name="project 2", project_id="project-id-2"), + ] + ) + + client.lightningapp_instance_service_list_lightningapp_instances().get.return_value = ( + V1ListLightningappInstancesResponse( + lightningapps=[ + Externalv1LightningappInstance( + name="app-name-0", + id="app-id-0", + ), + Externalv1LightningappInstance( + name="app-name-1", + id="app-id-1", + ), + Externalv1LightningappInstance( + name="app name 2", + id="app-id-1", + ), + ] + ) + ) + + client.cloud_space_service_list_cloud_spaces().get.return_value = V1ListCloudSpacesResponse(cloudspaces=[]) + + clusters = MagicMock() + clusters.clusters = [MagicMock()] + client.projects_service_list_project_cluster_bindings.return_value = clusters + + def fn(*args, prefix, **kwargs): + splits = [split for split in prefix.split("/") if split != ""] + if len(splits) == 2: + return V1ListLightningappInstanceArtifactsResponse( + artifacts=[ + V1LightningappInstanceArtifact(filename="file_1.txt"), + V1LightningappInstanceArtifact(filename="folder_1/file_2.txt"), + V1LightningappInstanceArtifact(filename="folder_2/folder_3/file_3.txt"), + V1LightningappInstanceArtifact(filename="folder_2/file_4.txt"), + ] + ) + elif splits[-1] == "folder_1": + return V1ListLightningappInstanceArtifactsResponse( + artifacts=[V1LightningappInstanceArtifact(filename="file_2.txt")] + ) + elif splits[-1] == "folder_2": + return V1ListLightningappInstanceArtifactsResponse( + artifacts=[ + V1LightningappInstanceArtifact(filename="folder_3/file_3.txt"), + V1LightningappInstanceArtifact(filename="file_4.txt"), + ] + ) + elif splits[-1] == "folder_3": + return V1ListLightningappInstanceArtifactsResponse( + artifacts=[ + V1LightningappInstanceArtifact(filename="file_3.txt"), + ] + ) + + client.lightningapp_instance_service_list_project_artifacts = fn + + client.lightningapp_instance_service_delete_project_artifact = MagicMock() + + monkeypatch.setattr(rm, "LightningClient", MagicMock(return_value=client)) + monkeypatch.setattr(ls, "LightningClient", MagicMock(return_value=client)) + + assert ls.ls() == ["project-0", "project-1", "project 2"] + assert "/project-0" == cd.cd("project-0", verify=False) + + assert f"/project-0{os.sep}app-name-1" == cd.cd("app-name-1", verify=False) + + assert f"/project-0{os.sep}app-name-1{os.sep}folder_1" == cd.cd("folder_1", verify=False) + + rm.rm("file_2.txt") + + kwargs = client.lightningapp_instance_service_delete_project_artifact._mock_call_args.kwargs + assert kwargs["project_id"] == "project-id-0" + assert kwargs["filename"] == "/lightningapps/app-id-1/folder_1/file_2.txt" + + os.remove(cd._CD_FILE) diff --git a/tests/tests_app/utilities/test_commands.py b/tests/tests_app/utilities/test_commands.py index 3a96261413..0d743c849e 100644 --- a/tests/tests_app/utilities/test_commands.py +++ b/tests/tests_app/utilities/test_commands.py @@ -47,6 +47,7 @@ class FlowCommands(LightningFlow): self.stop() def trigger_method(self, name: str): + print(name) self.names.append(name) def sweep(self, config: SweepConfig): @@ -147,7 +148,7 @@ def test_configure_commands(monkeypatch): monkeypatch.setattr(sys, "argv", ["lightning", "user", "command", "--name=something"]) connect_app("localhost") _run_app_command("localhost", None) - sleep(0.5) + sleep(2) state = AppState() state._request_state() assert state.names == ["something"]