From 4e3273a81f35e3c89988423dc1d15515c34fa8ee Mon Sep 17 00:00:00 2001 From: Jirka Borovec <6035284+Borda@users.noreply.github.com> Date: Sun, 5 Mar 2023 11:41:00 +0100 Subject: [PATCH] docs: deploy all (#16951) --- .github/checkgroup.yml | 6 +- .../{docs-checks.yml => docs-build.yml} | 67 +++++++++---- .github/workflows/docs-deploy.yml | 93 ------------------- .gitignore | 3 + 4 files changed, 57 insertions(+), 112 deletions(-) rename .github/workflows/{docs-checks.yml => docs-build.yml} (66%) delete mode 100644 .github/workflows/docs-deploy.yml diff --git a/.github/checkgroup.yml b/.github/checkgroup.yml index b8a0e228db..2bad505b1d 100644 --- a/.github/checkgroup.yml +++ b/.github/checkgroup.yml @@ -140,7 +140,7 @@ subprojects: - "src/lightning/fabric/**" - "src/lightning_fabric/*" - "docs/source-fabric/**" - - ".github/workflows/docs-checks.yml" + - ".github/workflows/docs-build.yml" - "requirements/docs.txt" - "requirements/fabric/**" - "setup.py" @@ -157,7 +157,7 @@ subprojects: - "src/pytorch_lightning/*" - "docs/source-pytorch/**" - ".actions/**" - - ".github/workflows/docs-checks.yml" + - ".github/workflows/docs-build.yml" - "requirements/docs.txt" - "requirements/pytorch/**" - "setup.py" @@ -339,7 +339,7 @@ subprojects: - "src/lightning/app/**" - "src/lightning_app/*" - "docs/source-app/**" - - ".github/workflows/docs-checks.yml" + - ".github/workflows/docs-build.yml" - "requirements/docs.txt" - "requirements/app/**" - "setup.py" diff --git a/.github/workflows/docs-checks.yml b/.github/workflows/docs-build.yml similarity index 66% rename from .github/workflows/docs-checks.yml rename to .github/workflows/docs-build.yml index ecaf1a7623..05db9c7f3a 100644 --- a/.github/workflows/docs-checks.yml +++ b/.github/workflows/docs-build.yml @@ -10,7 +10,7 @@ on: branches: ["master", "release/*"] paths: - ".actions/**" - - ".github/workflows/docs-checks.yml" + - ".github/workflows/docs-build.yml" - "requirements/**" - "docs/**" - "src/**" @@ -30,6 +30,7 @@ defaults: env: FREEZE_REQUIREMENTS: "1" TORCH_URL: "https://download.pytorch.org/whl/cpu/torch_stable.html" + PYPI_CACHE: "_pip-wheels" jobs: make-doctest: @@ -56,17 +57,16 @@ jobs: - run: aws s3 sync s3://sphinx-packages/ pypi/ if: ${{ matrix.pkg-name != 'pytorch' }} - # Note: This uses an internal pip API and may not always work - # https://github.com/actions/cache/blob/master/examples.md#multiple-oss-in-a-workflow - - name: Cache pip - uses: actions/cache@v3 + - name: pip wheels cache + uses: actions/cache/restore@v3 with: - path: ~/.cache/pip - key: docs-test-${{ matrix.pkg-name }}-${{ hashFiles('requirements/${{ matrix.pkg-name }}/*.txt') }} - restore-keys: docs-test-${{ matrix.pkg-name }}- + path: ${{ env.PYPI_CACHE }} + key: pypi_wheels - - name: Install this package + - name: Install package run: | + mkdir -p $PYPI_CACHE + ls -lh $PYPI_CACHE pip install -e .[extra,cloud,ui] -U -r requirements/${{ matrix.pkg-name }}/docs.txt -f pypi -f ${TORCH_URL} pip list @@ -78,6 +78,7 @@ jobs: make doctest make coverage + make-html: runs-on: ubuntu-latest container: @@ -102,17 +103,16 @@ jobs: - run: aws s3 sync s3://sphinx-packages/ pypi/ if: ${{ matrix.pkg-name != 'pytorch' }} - # Note: This uses an internal pip API and may not always work - # https://github.com/actions/cache/blob/master/examples.md#multiple-oss-in-a-workflow - - name: Cache pip - uses: actions/cache@v3 + - name: pip wheels cache + uses: actions/cache/restore@v3 with: - path: ~/.cache/pip - key: docs-make-${{ hashFiles('requirements/${{ matrix.pkg-name }}/*.txt') }} - restore-keys: docs-make- + path: ${{ env.PYPI_CACHE }} + key: pypi_wheels - name: Install package & dependencies run: | + mkdir -p $PYPI_CACHE + ls -lh $PYPI_CACHE pip --version pip install -e . -U -r requirements/${{ matrix.pkg-name }}/docs.txt -f pypi -f ${TORCH_URL} pip list @@ -132,3 +132,38 @@ jobs: with: name: docs-${{ matrix.pkg-name }}-${{ github.sha }} path: docs/build/html/ + + + deploy-docs: + needs: [make-doctest, make-html] + if: github.repository_owner == 'Lightning-AI' && github.event_name == 'push' + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + pkg-name: ["app", "fabric", "pytorch"] + steps: + - uses: actions/download-artifact@v3 + with: + name: docs-${{ matrix.pkg-name }}-${{ github.sha }} + path: docs/build/html/ + + - name: Authenticate to Google Cloud + uses: google-github-actions/auth@v1 + with: + credentials_json: ${{ secrets.GCS_SA_KEY }} + + - name: Setup gcloud + uses: google-github-actions/setup-gcloud@v1 + with: + project_id: ${{ secrets.GCS_PROJECT }} + + # Uploading docs to GCS, so they can be served on lightning.ai + - name: Upload docs/stable to GCS 🪣 + if: startsWith(github.ref, 'refs/heads/release/') + run: gsutil -m rsync -d -R docs/build/html/ gs://lightning-docs-${{ matrix.pkg-name }}/stable + + # Uploading docs to GCS, so they can be served on lightning.ai + - name: Upload docs/latest to GCS 🪣 + if: github.ref == 'refs/heads/master' + run: gsutil -m rsync -d -R docs/build/html/ gs://lightning-docs-${{ matrix.pkg-name }}/latest diff --git a/.github/workflows/docs-deploy.yml b/.github/workflows/docs-deploy.yml deleted file mode 100644 index e8e329e67e..0000000000 --- a/.github/workflows/docs-deploy.yml +++ /dev/null @@ -1,93 +0,0 @@ -name: "Deploy Docs" -on: - push: - branches: ["master", "release/stable"] - pull_request: - branches: ["master", "release/*"] - paths: - - ".github/workflows/docs-deploy.yml" - # TODO: this workflow is just for debugging. extend the paths that should trigger it - -defaults: - run: - shell: bash - -jobs: - # https://github.com/marketplace/actions/deploy-to-github-pages - build-docs-deploy: - if: github.repository_owner == 'Lightning-AI' - runs-on: ubuntu-latest - container: - image: pytorchlightning/pytorch_lightning:docs - steps: - - name: Checkout 🛎️ - uses: actions/checkout@v3 - # If you're using actions/checkout@v3 you must set persist-credentials to false in most cases for the deployment to work correctly. - with: - persist-credentials: false - - - uses: aws-actions/configure-aws-credentials@v1 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_KEY_ID }} - aws-region: us-east-1 - - run: aws s3 sync s3://sphinx-packages/ pypi/ - - # Note: This uses an internal pip API and may not always work - # https://github.com/actions/cache/blob/master/examples.md#multiple-oss-in-a-workflow - - name: Cache pip - uses: actions/cache@v3 - with: - path: ~/.cache/pip - key: deploy-docs-pip-${{ hashFiles('requirements/**/*.txt') }} - restore-keys: deploy-docs-pip- - - - name: Install package & dependencies - env: - FREEZE_REQUIREMENTS: 1 - TORCH_URL: "https://download.pytorch.org/whl/cpu/torch_stable.html" - run: | - pip --version - pip install -e . -r requirements/app/docs.txt \ - -f ${TORCH_URL} -f pypi - pip list - - - name: Make Documentation - working-directory: ./docs/source-app - run: | - # First run the same pipeline as Read-The-Docs - make clean - make html --jobs 2 - - - name: Deploy 🚀 - uses: JamesIves/github-pages-deploy-action@v4.4.1 - with: - token: ${{ secrets.GITHUB_TOKEN }} - branch: gh-pages # The branch the action should deploy to. - folder: docs/build/html # The folder the action should deploy. - clean: true # Automatically remove deleted files from deploy branch - target-folder: docs # If you'd like to push the contents of the deployment folder into a specific directory - single-commit: true # you'd prefer to have a single commit on the deployment branch instead of full history - if: github.event_name == 'push' && github.ref == 'refs/heads/master' - - - name: Authenticate to Google Cloud - if: github.event_name == 'push' - uses: google-github-actions/auth@v1 - with: - credentials_json: ${{ secrets.GCS_SA_KEY }} - - - name: Setup gcloud - if: github.event_name == 'push' - uses: google-github-actions/setup-gcloud@v1 - with: - project_id: ${{ secrets.GCS_PROJECT }} - - # Uploading docs to GCS, so they can be served on lightning.ai - - name: Upload docs/stable to GCS 🪣 - if: github.event_name == 'push' && startsWith(github.ref, 'refs/heads/release/') - run: gsutil -m rsync -d -R docs/build/html/ gs://lightning-docs-stable - - # Uploading docs to GCS, so they can be served on lightning.ai - - name: Upload docs/latest to GCS 🪣 - if: github.event_name == 'push' && github.ref == 'refs/heads/master' - run: gsutil -m rsync -d -R docs/build/html/ gs://lightning-docs-latest diff --git a/.gitignore b/.gitignore index 1ab84c850d..f5864e578d 100644 --- a/.gitignore +++ b/.gitignore @@ -195,3 +195,6 @@ requirements/base.txt # CI .wheels/ + +# sourced notebooks from tutorials +_notebooks/.notebooks/