Fix pip install with no PACKAGE_NAME or editable mode (#15853)

Co-authored-by: Jirka Borovec <6035284+Borda@users.noreply.github.com>
Co-authored-by: Akihiro Nitta <nitta@akihironitta.com>

.github/actions/pkg-check/action.yml

@@ -19,15 +19,16 @@ runs:
     run: pip install "twine==4.0.1" setuptools wheel flake8
     shell: bash
 
+  - name: Set PACKAGE_NAME envvar
+    if: ${{ inputs.pkg-name != 'notset' }}
+    run: echo "PACKAGE_NAME=${{ inputs.pkg-name }}" >> $GITHUB_ENV
+    shell: bash
+
   - name: Source check
-    env:
-      PACKAGE_NAME: ${{ inputs.pkg-name }}
     run: python setup.py check --metadata --strict
     shell: bash
 
   - name: Create package
-    env:
-      PACKAGE_NAME: ${{ inputs.pkg-name }}
     run: python setup.py sdist bdist_wheel
     shell: bash
 
@@ -39,13 +40,13 @@ runs:
     shell: bash
 
   - name: Unzip packages
-    if: ${{ inputs.pkg-name != 'lightning' }}
+    if: ${{ inputs.pkg-name != 'lightning' && inputs.pkg-name != 'notset' }}
     working-directory: dist
     run: for file in `ls *.gz`; do tar -xzf $file; done
     shell: bash
 
   - name: Check single pkg/folder
-    if: ${{ inputs.pkg-name != 'lightning' }}
+    if: ${{ inputs.pkg-name != 'lightning' && inputs.pkg-name != 'notset' }}
     working-directory: dist
     run: |
       import os, glob, pathlib, shutil

.github/actions/pkg-install/action.yml

@@ -15,7 +15,7 @@ runs:
   steps:
   - name: Choose package import
     run: |
-      python -c "print('PKG_IMPORT=' + {'app': 'lightning_app', 'lite': 'lightning_lite', 'pytorch': 'pytorch_lightning', 'lightning': 'lightning'}['${{matrix.pkg-name}}'])" >> $GITHUB_ENV
+      python -c "print('PKG_IMPORT=' + {'app': 'lightning_app', 'lite': 'lightning_lite', 'pytorch': 'pytorch_lightning', 'lightning': 'lightning', 'notset': 'lightning'}['${{matrix.pkg-name}}'])" >> $GITHUB_ENV
     shell: bash
 
   - name: Install package - archive

.github/checkgroup.yml

@@ -331,6 +331,8 @@ subprojects:
       - "install-pkg (ubuntu-22.04, pytorch, 3.10)"
       - "install-pkg (ubuntu-22.04, lightning, 3.7)"
       - "install-pkg (ubuntu-22.04, lightning, 3.10)"
+      - "install-pkg (ubuntu-22.04, notset, 3.7)"
+      - "install-pkg (ubuntu-22.04, notset, 3.10)"
       - "install-pkg (macOS-12, app, 3.7)"
       - "install-pkg (macOS-12, app, 3.10)"
       - "install-pkg (macOS-12, lite, 3.7)"
@@ -339,6 +341,8 @@ subprojects:
       - "install-pkg (macOS-12, pytorch, 3.10)"
       - "install-pkg (macOS-12, lightning, 3.7)"
       - "install-pkg (macOS-12, lightning, 3.10)"
+      - "install-pkg (macOS-12, notset, 3.7)"
+      - "install-pkg (macOS-12, notset, 3.10)"
       - "install-pkg (windows-2022, app, 3.7)"
       - "install-pkg (windows-2022, app, 3.10)"
       - "install-pkg (windows-2022, lite, 3.7)"
@@ -347,3 +351,5 @@ subprojects:
       - "install-pkg (windows-2022, pytorch, 3.10)"
       - "install-pkg (windows-2022, lightning, 3.7)"
       - "install-pkg (windows-2022, lightning, 3.10)"
+      - "install-pkg (windows-2022, notset, 3.7)"
+      - "install-pkg (windows-2022, notset, 3.10)"

.github/workflows/ci-pkg-install.yml

@@ -34,7 +34,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-22.04, macOS-12, windows-2022]
-        pkg-name: ["app", "lite", "pytorch", "lightning"]
+        pkg-name: ["app", "lite", "pytorch", "lightning", "notset"]
         python-version: ["3.7" , "3.10"]
     steps:
       - uses: actions/checkout@v3
@@ -60,11 +60,14 @@ jobs:
         with:
           pkg-name: ${{ matrix.pkg-name }}
 
-      - name: Run CLI
-        # todo: add testing for `lightning_app`
-        if: ${{ matrix.pkg-name == 'lightning' }}
+      - name: Run CLI (via python)
+        if: ${{ matrix.pkg-name == 'lightning' || matrix.pkg-name == 'notset' }}
         run: python -m lightning --version
 
+      - name: Run CLI (direct bash)
+        if: ${{ matrix.pkg-name == 'lightning' || matrix.pkg-name == 'app' }}
+        run: lightning --version
+
       - name: DocTest package
         env:
           LIGHTING_TESTING: 1  # path for require wrapper
@@ -72,5 +75,5 @@ jobs:
         run: |
           pip install -q "pytest-doctestplus>=0.9.0"
           pip list
-          PKG_NAME=$(python -c "print({'app': 'lightning_app', 'lite': 'lightning_lite', 'pytorch': 'pytorch_lightning', 'lightning': 'lightning'}['${{matrix.pkg-name}}'])")
+          PKG_NAME=$(python -c "print({'app': 'lightning_app', 'lite': 'lightning_lite', 'pytorch': 'pytorch_lightning', 'lightning': 'lightning', 'notset': 'lightning'}['${{matrix.pkg-name}}'])")
           python -m pytest src/${PKG_NAME} --ignore-glob="**/cli/*-template/**"

setup.py

@@ -49,7 +49,7 @@ from typing import Generator, Optional
 import setuptools
 import setuptools.command.egg_info
 
-_PACKAGE_NAME = os.environ.get("PACKAGE_NAME")
+_PACKAGE_NAME = os.environ.get("PACKAGE_NAME", "lightning")
 _PACKAGE_MAPPING = {
     "lightning": "lightning",
     "pytorch": "pytorch_lightning",
@@ -90,7 +90,15 @@ def _set_manifest_path(manifest_dir: str, aggregate: bool = False) -> Generator:
         del mapping["lightning"]
         lines = ["include src/lightning/version.info\n"]
         for pkg in mapping.values():
-            with open(os.path.join(_PATH_SRC, pkg, "MANIFEST.in")) as fh:
+            pkg_path = os.path.join(_PATH_SRC, pkg)
+            if not os.path.exists(pkg_path):
+                # this function is getting called with `pip install .` and `pip install *.tar.gz`, however, it only
+                # should be called with the former. i haven't found a way to differentiate the two so this is the hacky
+                # solution to avoid an error
+                print(f"{pkg_path!r} does not exist")
+                yield
+                return
+            with open(os.path.join(pkg_path, "MANIFEST.in")) as fh:
                 lines.extend(fh.readlines())
         # convert lightning_foo to lightning/foo
         for new, old in mapping.items():
@@ -143,11 +151,12 @@ if __name__ == "__main__":
             print(f"{pkg_setup} exists. Running `setuptools.setup`")
             setup_module = _load_py_module(name=f"{pkg}_setup", location=pkg_setup)
             setup_args = setup_module._setup_args()
-            if is_wheel_install:
+            is_main_pkg = pkg == "lightning"
+            if is_wheel_install and not is_main_pkg:
                 setuptools.setup(**setup_args)
             else:
                 # we are installing from source, set the correct manifest path
-                with _set_manifest_path(pkg_path, aggregate=pkg == "lightning"):
+                with _set_manifest_path(pkg_path, aggregate=is_main_pkg):
                     setuptools.setup(**setup_args)
             break
     else: