From d1514e189813245f41adb0d31a747db8ce8ac371 Mon Sep 17 00:00:00 2001 From: Vitalii Koshura Date: Tue, 29 Mar 2022 00:18:29 +0200 Subject: [PATCH] [CI] Simplify AWS S3 usage for vcpkg cache Signed-off-by: Vitalii Koshura --- .github/workflows/android.yml | 25 +++++------ .github/workflows/flutter-ubuntu.yml | 25 +++++------ .github/workflows/flutter-windows.yml | 20 +++++---- .github/workflows/linux.yml | 25 +++++------ .github/workflows/maintenance.yml | 29 ------------- .github/workflows/mingw.yml | 15 +++++++ .github/workflows/windows.yml | 42 +++++++++---------- android/buildAndroidBOINC-CI.sh | 1 - linux/update_vcpkg_apps.sh | 2 - linux/update_vcpkg_client.sh | 2 - linux/update_vcpkg_libs.sh | 2 - mingw/update_vcpkg.sh | 2 - ...vcpkg_3rdparty_dependencies_vs2019.vcxproj | 6 +-- 13 files changed, 83 insertions(+), 113 deletions(-) delete mode 100644 .github/workflows/maintenance.yml diff --git a/.github/workflows/android.yml b/.github/workflows/android.yml index d481153557..607f6c6486 100644 --- a/.github/workflows/android.yml +++ b/.github/workflows/android.yml @@ -7,6 +7,11 @@ on: schedule: - cron: '5 12 * * 0' +env: + AWS_ACCESS_KEY_ID: ${{ secrets.S3_ACCESS_KEY }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_SECRET_KEY }} + AWS_DEFAULT_REGION: us-west-2 + jobs: build: name: ${{ matrix.type }}-build @@ -36,15 +41,15 @@ jobs: !3rdParty/buildCache/android/vcpkgcache/ key: android-${{ matrix.type }}-${{ hashFiles('android/*.sh', '.github/workflows/android.yml') }} - - name: Configure Python - if: ${{ success() && contains(matrix.type, 'vcpkg') }} + - name: Check if build is running from origin repo + if: ${{ success() && env.AWS_ACCESS_KEY_ID != 0 && env.AWS_SECRET_ACCESS_KEY != 0 }} run: | - pip install boto3 + echo "VCPKG_BINARY_SOURCES=clear;x-aws,s3://vcpkg.cache.boinc/,readwrite" >> $GITHUB_ENV - - name: Download vcpkg binary cache - if: ${{ success() && contains(matrix.type, 'vcpkg') }} + - name: Check if build is running from fork + if: ${{ success() && (env.AWS_ACCESS_KEY_ID == 0 || env.AWS_SECRET_ACCESS_KEY == 0) }} run: | - python ./deploy/manage_vcpkg_archive_cache.py download 3rdParty/buildCache/android/vcpkgcache/vcpkg/archives/ android "edu.berkeley.boinc.github.actions.build.vcpkg.binary.cache" + echo "VCPKG_BINARY_SOURCES=clear;x-aws-config,no-sign-request;x-aws,s3://vcpkg.cache.boinc/,read" >> $GITHUB_ENV - name: Build manager if: ${{ success() && matrix.type == 'manager' }} @@ -118,11 +123,3 @@ jobs: with: fail_ci_if_error: true verbose: false - - - name: Upload vcpkg binary cache - if: ${{ success() && contains(matrix.type, 'vcpkg') }} - env: - ACCESS_KEY: ${{ secrets.S3_ACCESS_KEY }} - SECRET_KEY: ${{ secrets.S3_SECRET_KEY }} - run: | - python ./deploy/manage_vcpkg_archive_cache.py upload 3rdParty/buildCache/android/vcpkgcache/vcpkg/archives/ android "edu.berkeley.boinc.github.actions.build.vcpkg.binary.cache" "$ACCESS_KEY" "$SECRET_KEY" diff --git a/.github/workflows/flutter-ubuntu.yml b/.github/workflows/flutter-ubuntu.yml index 7383a11871..c2bc54c756 100644 --- a/.github/workflows/flutter-ubuntu.yml +++ b/.github/workflows/flutter-ubuntu.yml @@ -7,6 +7,11 @@ on: schedule: - cron: '25 12 * * 0' +env: + AWS_ACCESS_KEY_ID: ${{ secrets.S3_ACCESS_KEY }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_SECRET_KEY }} + AWS_DEFAULT_REGION: us-west-2 + jobs: build: name: ${{ matrix.type }}-flutter-build @@ -45,10 +50,6 @@ jobs: sudo apt update sudo apt install -y clang cmake ninja-build pkg-config libgtk-3-dev - - name: Configure Python - run: | - pip install boto3 - - name: Cache dependencies uses: actions/cache@v2 if: matrix.type == 'linux' @@ -58,10 +59,15 @@ jobs: !3rdParty/buildCache/linux/vcpkgcache/ key: linux-flutter-client-vcpkg-${{ hashFiles('3rdParty/*Linux*.sh', 'linux/*.sh', '.github/workflows/flutter-ubuntu.yml') }} - - name: Download vcpkg binary cache - if: matrix.type == 'linux' + - name: Check if build is running from origin repo + if: ${{ success() && env.AWS_ACCESS_KEY_ID != 0 && env.AWS_SECRET_ACCESS_KEY != 0 }} run: | - python ./deploy/manage_vcpkg_archive_cache.py download 3rdParty/buildCache/linux/vcpkgcache/vcpkg/archives/ linux "edu.berkeley.boinc.github.actions.build.vcpkg.binary.cache" + echo "VCPKG_BINARY_SOURCES=clear;x-aws,s3://vcpkg.cache.boinc/,readwrite" >> $GITHUB_ENV + + - name: Check if build is running from fork + if: ${{ success() && (env.AWS_ACCESS_KEY_ID == 0 || env.AWS_SECRET_ACCESS_KEY == 0) }} + run: | + echo "VCPKG_BINARY_SOURCES=clear;x-aws-config,no-sign-request;x-aws,s3://vcpkg.cache.boinc/,read" >> $GITHUB_ENV - name: Build linux client if: matrix.type == 'linux' @@ -76,11 +82,6 @@ jobs: !3rdParty/buildCache/android/vcpkgcache/ key: android-manager-vcpkg-${{ hashFiles('android/*.sh') }} - - name: Download vcpkg binary cache - if: matrix.type == 'android' - run: | - python ./deploy/manage_vcpkg_archive_cache.py download 3rdParty/buildCache/android/vcpkgcache/vcpkg/archives/ android "edu.berkeley.boinc.github.actions.build.vcpkg.binary.cache" - - name: Build android client if: matrix.type == 'android' run: samples/flutter/ci_build_android.sh diff --git a/.github/workflows/flutter-windows.yml b/.github/workflows/flutter-windows.yml index c633c11338..e96ae057c2 100644 --- a/.github/workflows/flutter-windows.yml +++ b/.github/workflows/flutter-windows.yml @@ -7,6 +7,11 @@ on: schedule: - cron: '25 12 * * 0' +env: + AWS_ACCESS_KEY_ID: ${{ secrets.S3_ACCESS_KEY }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_SECRET_KEY }} + AWS_DEFAULT_REGION: us-west-2 + jobs: build: name: ${{ matrix.type }}-flutter-build @@ -44,18 +49,15 @@ jobs: ${{ github.workspace }}\3rdParty\Windows\cuda\ key: windows-flutter-x64-Release-${{ hashFiles('win_build/vcpkg_3rdparty_dependencies_vs2019.vcxproj', '.github/workflows/flutter-windows.yml') }} - - name: Configure Python - if: matrix.type == 'windows' - shell: cmd + - name: Check if build is running from origin repo + if: ${{ success() && env.AWS_ACCESS_KEY_ID != 0 && env.AWS_SECRET_ACCESS_KEY != 0 }} run: | - pip install boto3 + echo "VCPKG_BINARY_SOURCES=clear;x-aws,s3://vcpkg.cache.boinc/,readwrite" >> $GITHUB_ENV - - name: Download vcpkg binary cache - if: matrix.type == 'windows' - shell: cmd + - name: Check if build is running from fork + if: ${{ success() && (env.AWS_ACCESS_KEY_ID == 0 || env.AWS_SECRET_ACCESS_KEY == 0) }} run: | - python deploy\manage_vcpkg_archive_cache.py download ${{ github.workspace }}\3rdParty\buildCache\windows\vcpkgcache\ windows_x64 "edu.berkeley.boinc.github.actions.build.vcpkg.binary.cache" - + echo "VCPKG_BINARY_SOURCES=clear;x-aws-config,no-sign-request;x-aws,s3://vcpkg.cache.boinc/,read" >> $GITHUB_ENV - name: Build windows client if: matrix.type == 'windows' diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 21ef60823a..a380fa60e8 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -7,6 +7,11 @@ on: schedule: - cron: '10 12 * * 0' +env: + AWS_ACCESS_KEY_ID: ${{ secrets.S3_ACCESS_KEY }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_SECRET_KEY }} + AWS_DEFAULT_REGION: us-west-2 + jobs: build: name: ${{ matrix.type }}-build @@ -41,15 +46,15 @@ jobs: !3rdParty/buildCache/linux/vcpkgcache/ key: linux-${{ matrix.type }}-${{ hashFiles('3rdParty/*Linux*.sh', 'linux/*.sh', '.github/workflows/linux.yml') }} - - name: Configure Python - if: ${{ success() && contains(matrix.type, 'vcpkg') }} + - name: Check if build is running from origin repo + if: ${{ success() && env.AWS_ACCESS_KEY_ID != 0 && env.AWS_SECRET_ACCESS_KEY != 0 }} run: | - pip install boto3==1.19.12 + echo "VCPKG_BINARY_SOURCES=clear;x-aws,s3://vcpkg.cache.boinc/,readwrite" >> $GITHUB_ENV - - name: Download vcpkg binary cache - if: ${{ success() && contains(matrix.type, 'vcpkg') }} + - name: Check if build is running from fork + if: ${{ success() && (env.AWS_ACCESS_KEY_ID == 0 || env.AWS_SECRET_ACCESS_KEY == 0) }} run: | - python ./deploy/manage_vcpkg_archive_cache.py download 3rdParty/buildCache/linux/vcpkgcache/vcpkg/archives/ linux "edu.berkeley.boinc.github.actions.build.vcpkg.binary.cache" + echo "VCPKG_BINARY_SOURCES=clear;x-aws-config,no-sign-request;x-aws,s3://vcpkg.cache.boinc/,read" >> $GITHUB_ENV - name: Automake if: success() @@ -146,11 +151,3 @@ jobs: with: fail_ci_if_error: true verbose: false - - - name: Upload vcpkg binary cache - if: ${{ success() && contains(matrix.type, 'vcpkg') }} - env: - ACCESS_KEY: ${{ secrets.S3_ACCESS_KEY }} - SECRET_KEY: ${{ secrets.S3_SECRET_KEY }} - run: | - python ./deploy/manage_vcpkg_archive_cache.py upload 3rdParty/buildCache/linux/vcpkgcache/vcpkg/archives/ linux "edu.berkeley.boinc.github.actions.build.vcpkg.binary.cache" "$ACCESS_KEY" "$SECRET_KEY" diff --git a/.github/workflows/maintenance.yml b/.github/workflows/maintenance.yml deleted file mode 100644 index 1b022a618b..0000000000 --- a/.github/workflows/maintenance.yml +++ /dev/null @@ -1,29 +0,0 @@ -name: Maintenance -on: - schedule: - - cron: '0 15 * * 0' - -jobs: - build: - name: ${{ matrix.type }} - runs-on: ubuntu-latest - strategy: - matrix: - type: [maintenance] - fail-fast: false - steps: - - uses: actions/checkout@v2 - with: - fetch-depth: 2 - - - name: Configure Python - run: | - pip install boto3==1.19.12 - - - name: Cleanup outdated binary cache - if: ${{ success() }} - env: - ACCESS_KEY: ${{ secrets.S3_ACCESS_KEY }} - SECRET_KEY: ${{ secrets.S3_SECRET_KEY }} - run: | - python ./deploy/cleanup_vcpkg_archive_cache.py ./vcpkgcache "edu.berkeley.boinc.github.actions.build.vcpkg.binary.cache" "$ACCESS_KEY" "$SECRET_KEY" diff --git a/.github/workflows/mingw.yml b/.github/workflows/mingw.yml index 2a510a4441..d432b447cb 100644 --- a/.github/workflows/mingw.yml +++ b/.github/workflows/mingw.yml @@ -7,6 +7,11 @@ on: schedule: - cron: '0 12 * * 0' +env: + AWS_ACCESS_KEY_ID: ${{ secrets.S3_ACCESS_KEY }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_SECRET_KEY }} + AWS_DEFAULT_REGION: us-west-2 + jobs: build: name: ${{ matrix.type }}-build @@ -18,6 +23,16 @@ jobs: steps: - uses: actions/checkout@v2 + - name: Check if build is running from origin repo + if: ${{ success() && env.AWS_ACCESS_KEY_ID != 0 && env.AWS_SECRET_ACCESS_KEY != 0 }} + run: | + echo "VCPKG_BINARY_SOURCES=clear;x-aws,s3://vcpkg.cache.boinc/,readwrite" >> $GITHUB_ENV + + - name: Check if build is running from fork + if: ${{ success() && (env.AWS_ACCESS_KEY_ID == 0 || env.AWS_SECRET_ACCESS_KEY == 0) }} + run: | + echo "VCPKG_BINARY_SOURCES=clear;x-aws-config,no-sign-request;x-aws,s3://vcpkg.cache.boinc/,read" >> $GITHUB_ENV + - name: Install dependencies run: | sudo apt-get -qq update diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index 63fb6312ce..1fbcbcd2a2 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -7,6 +7,11 @@ on: schedule: - cron: '15 12 * * 0' +env: + AWS_ACCESS_KEY_ID: ${{ secrets.S3_ACCESS_KEY }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_SECRET_KEY }} + AWS_DEFAULT_REGION: us-west-2 + jobs: build: name: ${{ matrix.configuration }}-${{ matrix.platform }}-build @@ -16,8 +21,6 @@ jobs: platform: [x64, ARM64] configuration: [Release] fail-fast: false - env: - VCPKG_BINARY_SOURCES: 'clear;files,${{ github.workspace }}\3rdParty\buildCache\windows\vcpkgcache\,readwrite' steps: # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - uses: actions/checkout@v2 @@ -42,6 +45,16 @@ jobs: - name: Setup msbuild uses: microsoft/setup-msbuild@v1 + - name: Check if build is running from origin repo + if: ${{ success() && env.AWS_ACCESS_KEY_ID != 0 && env.AWS_SECRET_ACCESS_KEY != 0 }} + run: | + echo "VCPKG_BINARY_SOURCES=clear;x-aws,s3://vcpkg.cache.boinc/,readwrite" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf-8 -Append + + - name: Check if build is running from fork + if: ${{ success() && (env.AWS_ACCESS_KEY_ID == 0 || env.AWS_SECRET_ACCESS_KEY == 0) }} + run: | + echo "VCPKG_BINARY_SOURCES=clear;x-aws-config,no-sign-request;x-aws,s3://vcpkg.cache.boinc/,read" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf-8 -Append + - name: Fix vcpkg run: vcpkg.exe integrate remove @@ -52,18 +65,6 @@ jobs: ${{ github.workspace }}\3rdParty\Windows\cuda\ key: windows-${{ matrix.platform }}-${{ matrix.configuration }}-${{ hashFiles('win_build/vcpkg_3rdparty_dependencies_vs2019.vcxproj', '.github/workflows/windows.yml') }} - - name: Configure Python - if: ${{ success() }} - shell: cmd - run: | - pip install boto3 - - - name: Download vcpkg binary cache - if: ${{ success() }} - shell: cmd - run: | - python deploy\manage_vcpkg_archive_cache.py download ${{ github.workspace }}\3rdParty\buildCache\windows\vcpkgcache\ windows_${{ matrix.platform }} "edu.berkeley.boinc.github.actions.build.vcpkg.binary.cache" - - name: Build run: msbuild win_build\boinc_vs2019.sln -p:Configuration=${{ matrix.configuration }} -p:Platform=${{ matrix.platform }} -p:VcpkgTripletConfig=ci -p:BuildConfig=ci -m @@ -91,21 +92,25 @@ jobs: python deploy\prepare_deployment.py win_apps python deploy\prepare_deployment.py win_client python deploy\prepare_deployment.py win_manager + - name: Upload apps uses: actions/upload-artifact@v2 with: name: win_apps_${{ matrix.platform }}_${{ github.event.pull_request.head.sha }} path: deploy/win_apps.7z + - name: Upload client uses: actions/upload-artifact@v2 with: name: win_client_${{ matrix.platform }}_${{ github.event.pull_request.head.sha }} path: deploy/win_client.7z + - name: Upload manager uses: actions/upload-artifact@v2 with: name: win_manager_${{ matrix.platform }}_${{ github.event.pull_request.head.sha }} path: deploy/win_manager.7z + - name: Upload symbol files uses: actions/upload-artifact@v2 with: @@ -125,12 +130,3 @@ jobs: with: fail_ci_if_error: true verbose: false - - - name: Upload vcpkg binary cache - if: ${{ success() }} - shell: cmd - env: - ACCESS_KEY: ${{ secrets.S3_ACCESS_KEY }} - SECRET_KEY: ${{ secrets.S3_SECRET_KEY }} - run: | - python deploy\manage_vcpkg_archive_cache.py upload ${{ github.workspace }}\3rdParty\buildCache\windows\vcpkgcache\ windows_${{ matrix.platform }} "edu.berkeley.boinc.github.actions.build.vcpkg.binary.cache" "%ACCESS_KEY%" "%SECRET_KEY%" diff --git a/android/buildAndroidBOINC-CI.sh b/android/buildAndroidBOINC-CI.sh index 99c78a06b8..f0a18fcd4a 100755 --- a/android/buildAndroidBOINC-CI.sh +++ b/android/buildAndroidBOINC-CI.sh @@ -253,7 +253,6 @@ getTripletName() } if [ $build_with_vcpkg = "yes" ]; then - export XDG_CACHE_HOME=$cache_dir/vcpkgcache/ if [ $ci = "yes" ]; then triplets_setup="ci" else diff --git a/linux/update_vcpkg_apps.sh b/linux/update_vcpkg_apps.sh index 6bfe90e2ca..119270bcda 100755 --- a/linux/update_vcpkg_apps.sh +++ b/linux/update_vcpkg_apps.sh @@ -11,8 +11,6 @@ BUILD_DIR="$PWD/3rdParty/linux" VCPKG_PORTS="$PWD/3rdParty/vcpkg_ports" VCPKG_ROOT="$BUILD_DIR/vcpkg" -export XDG_CACHE_HOME=$CACHE_DIR/vcpkgcache - if [ ! -d $VCPKG_ROOT ]; then mkdir -p $BUILD_DIR git -C $BUILD_DIR clone https://github.com/microsoft/vcpkg diff --git a/linux/update_vcpkg_client.sh b/linux/update_vcpkg_client.sh index 3b725340d0..9816aa2047 100755 --- a/linux/update_vcpkg_client.sh +++ b/linux/update_vcpkg_client.sh @@ -11,8 +11,6 @@ BUILD_DIR="$PWD/3rdParty/linux" VCPKG_PORTS="$PWD/3rdParty/vcpkg_ports" VCPKG_ROOT="$BUILD_DIR/vcpkg" -export XDG_CACHE_HOME=$CACHE_DIR/vcpkgcache - if [ ! -d $VCPKG_ROOT ]; then mkdir -p $BUILD_DIR git -C $BUILD_DIR clone https://github.com/microsoft/vcpkg diff --git a/linux/update_vcpkg_libs.sh b/linux/update_vcpkg_libs.sh index 3b725340d0..9816aa2047 100755 --- a/linux/update_vcpkg_libs.sh +++ b/linux/update_vcpkg_libs.sh @@ -11,8 +11,6 @@ BUILD_DIR="$PWD/3rdParty/linux" VCPKG_PORTS="$PWD/3rdParty/vcpkg_ports" VCPKG_ROOT="$BUILD_DIR/vcpkg" -export XDG_CACHE_HOME=$CACHE_DIR/vcpkgcache - if [ ! -d $VCPKG_ROOT ]; then mkdir -p $BUILD_DIR git -C $BUILD_DIR clone https://github.com/microsoft/vcpkg diff --git a/mingw/update_vcpkg.sh b/mingw/update_vcpkg.sh index 0f45379727..ee6258a5af 100755 --- a/mingw/update_vcpkg.sh +++ b/mingw/update_vcpkg.sh @@ -11,8 +11,6 @@ BUILD_DIR="$PWD/3rdParty/mingw" VCPKG_PORTS="$PWD/3rdParty/vcpkg_ports" VCPKG_ROOT="$BUILD_DIR/vcpkg" -export XDG_CACHE_HOME=$CACHE_DIR/vcpkgcache - if [ ! -d $VCPKG_ROOT ]; then mkdir -p $BUILD_DIR git -C $BUILD_DIR clone https://github.com/microsoft/vcpkg diff --git a/win_build/vcpkg_3rdparty_dependencies_vs2019.vcxproj b/win_build/vcpkg_3rdparty_dependencies_vs2019.vcxproj index 09dcaf9643..a60923b3bd 100644 --- a/win_build/vcpkg_3rdparty_dependencies_vs2019.vcxproj +++ b/win_build/vcpkg_3rdparty_dependencies_vs2019.vcxproj @@ -178,8 +178,8 @@ - - - + + +