diff --git a/.azure/app-cloud-e2e.yml b/.azure/app-cloud-e2e.yml
index 260c7fb4b2..e1b6711a20 100644
--- a/.azure/app-cloud-e2e.yml
+++ b/.azure/app-cloud-e2e.yml
@@ -194,6 +194,7 @@ jobs:
     - bash: |
         time python -c "from lightning.app import testing; testing.delete_cloud_lightning_apps()"
       condition: always()
+      continueOnError: "true"
       timeoutInMinutes: "3"
       env:
         #LAI_USER: $(LAI_USER)  # for STAGING
diff --git a/.github/workflows/ci-tests-app.yml b/.github/workflows/ci-tests-app.yml
index acfea532ae..e7d94d9434 100644
--- a/.github/workflows/ci-tests-app.yml
+++ b/.github/workflows/ci-tests-app.yml
@@ -47,7 +47,7 @@ jobs:
           - {os: "ubuntu-20.04", pkg-name: "app", python-version: "3.9", requires: "latest"}
           - {os: "windows-2022", pkg-name: "app", python-version: "3.8", requires: "latest"}
     # Timeout: https://stackoverflow.com/a/59076067/4521646
-    timeout-minutes: 40
+    timeout-minutes: 45
     env:
       PACKAGE_NAME: ${{ matrix.pkg-name }}
       FREEZE_REQUIREMENTS: ${{ ! (github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/heads/release/')) }}
@@ -80,6 +80,7 @@ jobs:
 
     - name: Install package & dependencies
       run: |
+        python -m pip install -q pip -U
         pip install -e .[dev] -U -f ${TORCH_URL} -f ${PYPI_CACHE_DIR} --prefer-binary
         pip list
     - name: Dump handy wheels
diff --git a/src/lightning/app/utilities/cli_helpers.py b/src/lightning/app/utilities/cli_helpers.py
index 99a76d3999..5397ed82d6 100644
--- a/src/lightning/app/utilities/cli_helpers.py
+++ b/src/lightning/app/utilities/cli_helpers.py
@@ -249,14 +249,6 @@ def _arrow_time_callback(
             raise click.ClickException(f"cannot parse time {value}")
 
 
-def _is_valid_release(release):
-    version, release = release
-    version = packaging.version.parse(version)
-    if any(r["yanked"] for r in release) or version.is_devrelease or version.is_prerelease:
-        return False
-    return True
-
-
 @functools.lru_cache(maxsize=1)
 def _get_newer_version() -> Optional[str]:
     """Check PyPI for newer versions of ``lightning``, returning the newest version if different from the current
@@ -265,16 +257,15 @@ def _get_newer_version() -> Optional[str]:
         return None
     try:
         response = requests.get(f"https://pypi.org/pypi/{__package_name__}/json")
-        releases = response.json()["releases"]
+        response_json = response.json()
+        releases = response_json["releases"]
         if __version__ not in releases:
             # Always return None if not installed from PyPI (e.g. dev versions)
             return None
-        releases = dict(filter(_is_valid_release, releases.items()))
-        sorted_releases = sorted(
-            releases.items(), key=lambda release: release[1][0]["upload_time_iso_8601"], reverse=True
-        )
-        latest_version = sorted_releases[0][0]
-        return None if __version__ == latest_version else latest_version
+        latest_version = response_json["info"]["version"]
+        parsed_version = packaging.version.parse(latest_version)
+        is_invalid = response_json["info"]["yanked"] or parsed_version.is_devrelease or parsed_version.is_prerelease
+        return None if __version__ == latest_version or is_invalid else latest_version
     except Exception:
         # Return None if any exception occurs
         return None
diff --git a/tests/tests_app/utilities/test_cli_helpers.py b/tests/tests_app/utilities/test_cli_helpers.py
index 248b46acd3..e82a6e1acf 100644
--- a/tests/tests_app/utilities/test_cli_helpers.py
+++ b/tests/tests_app/utilities/test_cli_helpers.py
@@ -73,46 +73,84 @@ def test_arrow_time_callback():
 
 
 @pytest.mark.parametrize(
-    "releases, current_version, newer_version",
+    "response, current_version, newer_version",
     [
         (
             {
-                "1.0.0": [{"upload_time_iso_8601": "2022-09-10", "yanked": False}],
-                "2.0.0": [{"upload_time_iso_8601": "2022-11-01", "yanked": False}],
+                "info": {
+                    "version": "2.0.0",
+                    "yanked": False,
+                },
+                "releases": {
+                    "1.0.0": {},
+                    "2.0.0": {},
+                },
             },
             "1.0.0",
             "2.0.0",
         ),
         (
             {
-                "1.0.0": [{"upload_time_iso_8601": "2022-09-10", "yanked": False}],
-                "2.0.0": [{"upload_time_iso_8601": "2022-11-01", "yanked": True}],
+                "info": {
+                    "version": "2.0.0",
+                    "yanked": True,
+                },
+                "releases": {
+                    "1.0.0": {},
+                    "2.0.0": {},
+                },
             },
             "1.0.0",
             None,
         ),
         (
             {
-                "1.0.0": [{"upload_time_iso_8601": "2022-09-10", "yanked": False}],
-                "2.0.0rc0": [{"upload_time_iso_8601": "2022-11-01", "yanked": False}],
+                "info": {
+                    "version": "1.0.0",
+                    "yanked": False,
+                },
+                "releases": {
+                    "1.0.0": {},
+                },
             },
             "1.0.0",
             None,
         ),
         (
             {
-                "2.0.0": [{"upload_time_iso_8601": "2022-11-01", "yanked": False}],
+                "info": {
+                    "version": "2.0.0rc0",
+                    "yanked": False,
+                },
+                "releases": {
+                    "1.0.0": {},
+                    "2.0.0": {},
+                },
+            },
+            "1.0.0",
+            None,
+        ),
+        (
+            {
+                "info": {
+                    "version": "2.0.0",
+                    "yanked": False,
+                },
+                "releases": {
+                    "1.0.0": {},
+                    "2.0.0": {},
+                },
             },
             "1.0.0dev",
             None,
         ),
-        ({"1.0.0": "this wil trigger an error"}, "1.0.0", None),
+        ({"this wil trigger an error": True}, "1.0.0", None),
         ({}, "1.0.0rc0", None),
     ],
 )
 @patch("lightning.app.utilities.cli_helpers.requests")
-def test_get_newer_version(mock_requests, releases, current_version, newer_version):
-    mock_requests.get().json.return_value = {"releases": releases}
+def test_get_newer_version(mock_requests, response, current_version, newer_version):
+    mock_requests.get().json.return_value = response
 
     lightning.app.utilities.cli_helpers.__version__ = current_version