From 9a778fc65957a56285d2dcf0a36553caa7846533 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Mochol=C3=AD?= Date: Mon, 12 Dec 2022 00:01:06 +0100 Subject: [PATCH] Fix pip install with no PACKAGE_NAME or editable mode (#15853) Co-authored-by: Jirka Borovec <6035284+Borda@users.noreply.github.com> Co-authored-by: Akihiro Nitta --- .github/actions/pkg-check/action.yml | 13 +++++++------ .github/actions/pkg-install/action.yml | 2 +- .github/checkgroup.yml | 6 ++++++ .github/workflows/ci-pkg-install.yml | 13 ++++++++----- setup.py | 17 +++++++++++++---- 5 files changed, 35 insertions(+), 16 deletions(-) diff --git a/.github/actions/pkg-check/action.yml b/.github/actions/pkg-check/action.yml index 4f3dddbe73..b0822acd86 100644 --- a/.github/actions/pkg-check/action.yml +++ b/.github/actions/pkg-check/action.yml @@ -19,15 +19,16 @@ runs: run: pip install "twine==4.0.1" setuptools wheel flake8 shell: bash + - name: Set PACKAGE_NAME envvar + if: ${{ inputs.pkg-name != 'notset' }} + run: echo "PACKAGE_NAME=${{ inputs.pkg-name }}" >> $GITHUB_ENV + shell: bash + - name: Source check - env: - PACKAGE_NAME: ${{ inputs.pkg-name }} run: python setup.py check --metadata --strict shell: bash - name: Create package - env: - PACKAGE_NAME: ${{ inputs.pkg-name }} run: python setup.py sdist bdist_wheel shell: bash @@ -39,13 +40,13 @@ runs: shell: bash - name: Unzip packages - if: ${{ inputs.pkg-name != 'lightning' }} + if: ${{ inputs.pkg-name != 'lightning' && inputs.pkg-name != 'notset' }} working-directory: dist run: for file in `ls *.gz`; do tar -xzf $file; done shell: bash - name: Check single pkg/folder - if: ${{ inputs.pkg-name != 'lightning' }} + if: ${{ inputs.pkg-name != 'lightning' && inputs.pkg-name != 'notset' }} working-directory: dist run: | import os, glob, pathlib, shutil diff --git a/.github/actions/pkg-install/action.yml b/.github/actions/pkg-install/action.yml index 0e0751c217..2a227fb678 100644 --- a/.github/actions/pkg-install/action.yml +++ b/.github/actions/pkg-install/action.yml @@ -15,7 +15,7 @@ runs: steps: - name: Choose package import run: | - python -c "print('PKG_IMPORT=' + {'app': 'lightning_app', 'lite': 'lightning_lite', 'pytorch': 'pytorch_lightning', 'lightning': 'lightning'}['${{matrix.pkg-name}}'])" >> $GITHUB_ENV + python -c "print('PKG_IMPORT=' + {'app': 'lightning_app', 'lite': 'lightning_lite', 'pytorch': 'pytorch_lightning', 'lightning': 'lightning', 'notset': 'lightning'}['${{matrix.pkg-name}}'])" >> $GITHUB_ENV shell: bash - name: Install package - archive diff --git a/.github/checkgroup.yml b/.github/checkgroup.yml index dac1e77bfe..a2e34a7021 100644 --- a/.github/checkgroup.yml +++ b/.github/checkgroup.yml @@ -331,6 +331,8 @@ subprojects: - "install-pkg (ubuntu-22.04, pytorch, 3.10)" - "install-pkg (ubuntu-22.04, lightning, 3.7)" - "install-pkg (ubuntu-22.04, lightning, 3.10)" + - "install-pkg (ubuntu-22.04, notset, 3.7)" + - "install-pkg (ubuntu-22.04, notset, 3.10)" - "install-pkg (macOS-12, app, 3.7)" - "install-pkg (macOS-12, app, 3.10)" - "install-pkg (macOS-12, lite, 3.7)" @@ -339,6 +341,8 @@ subprojects: - "install-pkg (macOS-12, pytorch, 3.10)" - "install-pkg (macOS-12, lightning, 3.7)" - "install-pkg (macOS-12, lightning, 3.10)" + - "install-pkg (macOS-12, notset, 3.7)" + - "install-pkg (macOS-12, notset, 3.10)" - "install-pkg (windows-2022, app, 3.7)" - "install-pkg (windows-2022, app, 3.10)" - "install-pkg (windows-2022, lite, 3.7)" @@ -347,3 +351,5 @@ subprojects: - "install-pkg (windows-2022, pytorch, 3.10)" - "install-pkg (windows-2022, lightning, 3.7)" - "install-pkg (windows-2022, lightning, 3.10)" + - "install-pkg (windows-2022, notset, 3.7)" + - "install-pkg (windows-2022, notset, 3.10)" diff --git a/.github/workflows/ci-pkg-install.yml b/.github/workflows/ci-pkg-install.yml index d9474edb98..eab62d2151 100644 --- a/.github/workflows/ci-pkg-install.yml +++ b/.github/workflows/ci-pkg-install.yml @@ -34,7 +34,7 @@ jobs: fail-fast: false matrix: os: [ubuntu-22.04, macOS-12, windows-2022] - pkg-name: ["app", "lite", "pytorch", "lightning"] + pkg-name: ["app", "lite", "pytorch", "lightning", "notset"] python-version: ["3.7" , "3.10"] steps: - uses: actions/checkout@v3 @@ -60,11 +60,14 @@ jobs: with: pkg-name: ${{ matrix.pkg-name }} - - name: Run CLI - # todo: add testing for `lightning_app` - if: ${{ matrix.pkg-name == 'lightning' }} + - name: Run CLI (via python) + if: ${{ matrix.pkg-name == 'lightning' || matrix.pkg-name == 'notset' }} run: python -m lightning --version + - name: Run CLI (direct bash) + if: ${{ matrix.pkg-name == 'lightning' || matrix.pkg-name == 'app' }} + run: lightning --version + - name: DocTest package env: LIGHTING_TESTING: 1 # path for require wrapper @@ -72,5 +75,5 @@ jobs: run: | pip install -q "pytest-doctestplus>=0.9.0" pip list - PKG_NAME=$(python -c "print({'app': 'lightning_app', 'lite': 'lightning_lite', 'pytorch': 'pytorch_lightning', 'lightning': 'lightning'}['${{matrix.pkg-name}}'])") + PKG_NAME=$(python -c "print({'app': 'lightning_app', 'lite': 'lightning_lite', 'pytorch': 'pytorch_lightning', 'lightning': 'lightning', 'notset': 'lightning'}['${{matrix.pkg-name}}'])") python -m pytest src/${PKG_NAME} --ignore-glob="**/cli/*-template/**" diff --git a/setup.py b/setup.py index 7e16a4496d..fe09537ee3 100755 --- a/setup.py +++ b/setup.py @@ -49,7 +49,7 @@ from typing import Generator, Optional import setuptools import setuptools.command.egg_info -_PACKAGE_NAME = os.environ.get("PACKAGE_NAME") +_PACKAGE_NAME = os.environ.get("PACKAGE_NAME", "lightning") _PACKAGE_MAPPING = { "lightning": "lightning", "pytorch": "pytorch_lightning", @@ -90,7 +90,15 @@ def _set_manifest_path(manifest_dir: str, aggregate: bool = False) -> Generator: del mapping["lightning"] lines = ["include src/lightning/version.info\n"] for pkg in mapping.values(): - with open(os.path.join(_PATH_SRC, pkg, "MANIFEST.in")) as fh: + pkg_path = os.path.join(_PATH_SRC, pkg) + if not os.path.exists(pkg_path): + # this function is getting called with `pip install .` and `pip install *.tar.gz`, however, it only + # should be called with the former. i haven't found a way to differentiate the two so this is the hacky + # solution to avoid an error + print(f"{pkg_path!r} does not exist") + yield + return + with open(os.path.join(pkg_path, "MANIFEST.in")) as fh: lines.extend(fh.readlines()) # convert lightning_foo to lightning/foo for new, old in mapping.items(): @@ -143,11 +151,12 @@ if __name__ == "__main__": print(f"{pkg_setup} exists. Running `setuptools.setup`") setup_module = _load_py_module(name=f"{pkg}_setup", location=pkg_setup) setup_args = setup_module._setup_args() - if is_wheel_install: + is_main_pkg = pkg == "lightning" + if is_wheel_install and not is_main_pkg: setuptools.setup(**setup_args) else: # we are installing from source, set the correct manifest path - with _set_manifest_path(pkg_path, aggregate=pkg == "lightning"): + with _set_manifest_path(pkg_path, aggregate=is_main_pkg): setuptools.setup(**setup_args) break else: