Rooba
/
boinc
mirror of https://github.com/BOINC/boinc.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

[CI] Simplify AWS S3 usage for vcpkg cache 

Signed-off-by: Vitalii Koshura <lestat.de.lionkur@gmail.com>
This commit is contained in:
Vitalii Koshura 2022-03-29 00:18:29 +02:00
parent e1ef1c6348
commit d1514e1898
No known key found for this signature in database
GPG Key ID: CE0DB1726070A5A3
13 changed files with 83 additions and 113 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
.github/workflows/android.yml vendored
View File

@ -7,6 +7,11 @@ on:
schedule:
- cron: '5 12 * * 0'
env:
AWS_ACCESS_KEY_ID: ${{ secrets.S3_ACCESS_KEY }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_SECRET_KEY }}
AWS_DEFAULT_REGION: us-west-2
jobs:
build:
name: ${{ matrix.type }}-build
@ -36,15 +41,15 @@ jobs:
!3rdParty/buildCache/android/vcpkgcache/
key: android-${{ matrix.type }}-${{ hashFiles('android/*.sh', '.github/workflows/android.yml') }}
- name: Configure Python
if: ${{ success() && contains(matrix.type, 'vcpkg') }}
- name: Check if build is running from origin repo
if: ${{ success() && env.AWS_ACCESS_KEY_ID != 0 && env.AWS_SECRET_ACCESS_KEY != 0 }}
run: |
pip install boto3
echo "VCPKG_BINARY_SOURCES=clear;x-aws,s3://vcpkg.cache.boinc/,readwrite" >> $GITHUB_ENV
- name: Download vcpkg binary cache
if: ${{ success() && contains(matrix.type, 'vcpkg') }}
- name: Check if build is running from fork
if: ${{ success() && (env.AWS_ACCESS_KEY_ID == 0 || env.AWS_SECRET_ACCESS_KEY == 0) }}
run: |
python ./deploy/manage_vcpkg_archive_cache.py download 3rdParty/buildCache/android/vcpkgcache/vcpkg/archives/ android "edu.berkeley.boinc.github.actions.build.vcpkg.binary.cache"
echo "VCPKG_BINARY_SOURCES=clear;x-aws-config,no-sign-request;x-aws,s3://vcpkg.cache.boinc/,read" >> $GITHUB_ENV
- name: Build manager
if: ${{ success() && matrix.type == 'manager' }}
@ -118,11 +123,3 @@ jobs:
with:
fail_ci_if_error: true
verbose: false
- name: Upload vcpkg binary cache
if: ${{ success() && contains(matrix.type, 'vcpkg') }}
env:
ACCESS_KEY: ${{ secrets.S3_ACCESS_KEY }}
SECRET_KEY: ${{ secrets.S3_SECRET_KEY }}
run: |
python ./deploy/manage_vcpkg_archive_cache.py upload 3rdParty/buildCache/android/vcpkgcache/vcpkg/archives/ android "edu.berkeley.boinc.github.actions.build.vcpkg.binary.cache" "$ACCESS_KEY" "$SECRET_KEY"

25
.github/workflows/flutter-ubuntu.yml vendored
View File

@ -7,6 +7,11 @@ on:
schedule:
- cron: '25 12 * * 0'
env:
AWS_ACCESS_KEY_ID: ${{ secrets.S3_ACCESS_KEY }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_SECRET_KEY }}
AWS_DEFAULT_REGION: us-west-2
jobs:
build:
name: ${{ matrix.type }}-flutter-build
@ -45,10 +50,6 @@ jobs:
sudo apt update
sudo apt install -y clang cmake ninja-build pkg-config libgtk-3-dev
- name: Configure Python
run: |
pip install boto3
- name: Cache dependencies
uses: actions/cache@v2
if: matrix.type == 'linux'
@ -58,10 +59,15 @@ jobs:
!3rdParty/buildCache/linux/vcpkgcache/
key: linux-flutter-client-vcpkg-${{ hashFiles('3rdParty/*Linux*.sh', 'linux/*.sh', '.github/workflows/flutter-ubuntu.yml') }}
- name: Download vcpkg binary cache
if: matrix.type == 'linux'
- name: Check if build is running from origin repo
if: ${{ success() && env.AWS_ACCESS_KEY_ID != 0 && env.AWS_SECRET_ACCESS_KEY != 0 }}
run: |
python ./deploy/manage_vcpkg_archive_cache.py download 3rdParty/buildCache/linux/vcpkgcache/vcpkg/archives/ linux "edu.berkeley.boinc.github.actions.build.vcpkg.binary.cache"
echo "VCPKG_BINARY_SOURCES=clear;x-aws,s3://vcpkg.cache.boinc/,readwrite" >> $GITHUB_ENV
- name: Check if build is running from fork
if: ${{ success() && (env.AWS_ACCESS_KEY_ID == 0 || env.AWS_SECRET_ACCESS_KEY == 0) }}
run: |
echo "VCPKG_BINARY_SOURCES=clear;x-aws-config,no-sign-request;x-aws,s3://vcpkg.cache.boinc/,read" >> $GITHUB_ENV
- name: Build linux client
if: matrix.type == 'linux'
@ -76,11 +82,6 @@ jobs:
!3rdParty/buildCache/android/vcpkgcache/
key: android-manager-vcpkg-${{ hashFiles('android/*.sh') }}
- name: Download vcpkg binary cache
if: matrix.type == 'android'
run: |
python ./deploy/manage_vcpkg_archive_cache.py download 3rdParty/buildCache/android/vcpkgcache/vcpkg/archives/ android "edu.berkeley.boinc.github.actions.build.vcpkg.binary.cache"
- name: Build android client
if: matrix.type == 'android'
run: samples/flutter/ci_build_android.sh

20
.github/workflows/flutter-windows.yml vendored
View File

@ -7,6 +7,11 @@ on:
schedule:
- cron: '25 12 * * 0'
env:
AWS_ACCESS_KEY_ID: ${{ secrets.S3_ACCESS_KEY }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_SECRET_KEY }}
AWS_DEFAULT_REGION: us-west-2
jobs:
build:
name: ${{ matrix.type }}-flutter-build
@ -44,18 +49,15 @@ jobs:
${{ github.workspace }}\3rdParty\Windows\cuda\
key: windows-flutter-x64-Release-${{ hashFiles('win_build/vcpkg_3rdparty_dependencies_vs2019.vcxproj', '.github/workflows/flutter-windows.yml') }}
- name: Configure Python
if: matrix.type == 'windows'
shell: cmd
- name: Check if build is running from origin repo
if: ${{ success() && env.AWS_ACCESS_KEY_ID != 0 && env.AWS_SECRET_ACCESS_KEY != 0 }}
run: |
pip install boto3
echo "VCPKG_BINARY_SOURCES=clear;x-aws,s3://vcpkg.cache.boinc/,readwrite" >> $GITHUB_ENV
- name: Download vcpkg binary cache
if: matrix.type == 'windows'
shell: cmd
- name: Check if build is running from fork
if: ${{ success() && (env.AWS_ACCESS_KEY_ID == 0 || env.AWS_SECRET_ACCESS_KEY == 0) }}
run: |
python deploy\manage_vcpkg_archive_cache.py download ${{ github.workspace }}\3rdParty\buildCache\windows\vcpkgcache\ windows_x64 "edu.berkeley.boinc.github.actions.build.vcpkg.binary.cache"
echo "VCPKG_BINARY_SOURCES=clear;x-aws-config,no-sign-request;x-aws,s3://vcpkg.cache.boinc/,read" >> $GITHUB_ENV
- name: Build windows client
if: matrix.type == 'windows'

25
.github/workflows/linux.yml vendored
View File

@ -7,6 +7,11 @@ on:
schedule:
- cron: '10 12 * * 0'
env:
AWS_ACCESS_KEY_ID: ${{ secrets.S3_ACCESS_KEY }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_SECRET_KEY }}
AWS_DEFAULT_REGION: us-west-2
jobs:
build:
name: ${{ matrix.type }}-build
@ -41,15 +46,15 @@ jobs:
!3rdParty/buildCache/linux/vcpkgcache/
key: linux-${{ matrix.type }}-${{ hashFiles('3rdParty/*Linux*.sh', 'linux/*.sh', '.github/workflows/linux.yml') }}
- name: Configure Python
if: ${{ success() && contains(matrix.type, 'vcpkg') }}
- name: Check if build is running from origin repo
if: ${{ success() && env.AWS_ACCESS_KEY_ID != 0 && env.AWS_SECRET_ACCESS_KEY != 0 }}
run: |
pip install boto3==1.19.12
echo "VCPKG_BINARY_SOURCES=clear;x-aws,s3://vcpkg.cache.boinc/,readwrite" >> $GITHUB_ENV
- name: Download vcpkg binary cache
if: ${{ success() && contains(matrix.type, 'vcpkg') }}
- name: Check if build is running from fork
if: ${{ success() && (env.AWS_ACCESS_KEY_ID == 0 || env.AWS_SECRET_ACCESS_KEY == 0) }}
run: |
python ./deploy/manage_vcpkg_archive_cache.py download 3rdParty/buildCache/linux/vcpkgcache/vcpkg/archives/ linux "edu.berkeley.boinc.github.actions.build.vcpkg.binary.cache"
echo "VCPKG_BINARY_SOURCES=clear;x-aws-config,no-sign-request;x-aws,s3://vcpkg.cache.boinc/,read" >> $GITHUB_ENV
- name: Automake
if: success()
@ -146,11 +151,3 @@ jobs:
with:
fail_ci_if_error: true
verbose: false
- name: Upload vcpkg binary cache
if: ${{ success() && contains(matrix.type, 'vcpkg') }}
env:
ACCESS_KEY: ${{ secrets.S3_ACCESS_KEY }}
SECRET_KEY: ${{ secrets.S3_SECRET_KEY }}
run: |
python ./deploy/manage_vcpkg_archive_cache.py upload 3rdParty/buildCache/linux/vcpkgcache/vcpkg/archives/ linux "edu.berkeley.boinc.github.actions.build.vcpkg.binary.cache" "$ACCESS_KEY" "$SECRET_KEY"

29
.github/workflows/maintenance.yml vendored
View File

@ -1,29 +0,0 @@
name: Maintenance
on:
schedule:
- cron: '0 15 * * 0'
jobs:
build:
name: ${{ matrix.type }}
runs-on: ubuntu-latest
strategy:
matrix:
type: [maintenance]
fail-fast: false
steps:
- uses: actions/checkout@v2
with:
fetch-depth: 2
- name: Configure Python
run: |
pip install boto3==1.19.12
- name: Cleanup outdated binary cache
if: ${{ success() }}
env:
ACCESS_KEY: ${{ secrets.S3_ACCESS_KEY }}
SECRET_KEY: ${{ secrets.S3_SECRET_KEY }}
run: |
python ./deploy/cleanup_vcpkg_archive_cache.py ./vcpkgcache "edu.berkeley.boinc.github.actions.build.vcpkg.binary.cache" "$ACCESS_KEY" "$SECRET_KEY"

15
.github/workflows/mingw.yml vendored
View File

@ -7,6 +7,11 @@ on:
schedule:
- cron: '0 12 * * 0'
env:
AWS_ACCESS_KEY_ID: ${{ secrets.S3_ACCESS_KEY }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_SECRET_KEY }}
AWS_DEFAULT_REGION: us-west-2
jobs:
build:
name: ${{ matrix.type }}-build
@ -18,6 +23,16 @@ jobs:
steps:
- uses: actions/checkout@v2
- name: Check if build is running from origin repo
if: ${{ success() && env.AWS_ACCESS_KEY_ID != 0 && env.AWS_SECRET_ACCESS_KEY != 0 }}
run: |
echo "VCPKG_BINARY_SOURCES=clear;x-aws,s3://vcpkg.cache.boinc/,readwrite" >> $GITHUB_ENV
- name: Check if build is running from fork
if: ${{ success() && (env.AWS_ACCESS_KEY_ID == 0 || env.AWS_SECRET_ACCESS_KEY == 0) }}
run: |
echo "VCPKG_BINARY_SOURCES=clear;x-aws-config,no-sign-request;x-aws,s3://vcpkg.cache.boinc/,read" >> $GITHUB_ENV
- name: Install dependencies
run: |
sudo apt-get -qq update

42
.github/workflows/windows.yml vendored
View File

@ -7,6 +7,11 @@ on:
schedule:
- cron: '15 12 * * 0'
env:
AWS_ACCESS_KEY_ID: ${{ secrets.S3_ACCESS_KEY }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_SECRET_KEY }}
AWS_DEFAULT_REGION: us-west-2
jobs:
build:
name: ${{ matrix.configuration }}-${{ matrix.platform }}-build
@ -16,8 +21,6 @@ jobs:
platform: [x64, ARM64]
configuration: [Release]
fail-fast: false
env:
VCPKG_BINARY_SOURCES: 'clear;files,${{ github.workspace }}\3rdParty\buildCache\windows\vcpkgcache\,readwrite'
steps:
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
- uses: actions/checkout@v2
@ -42,6 +45,16 @@ jobs:
- name: Setup msbuild
uses: microsoft/setup-msbuild@v1
- name: Check if build is running from origin repo
if: ${{ success() && env.AWS_ACCESS_KEY_ID != 0 && env.AWS_SECRET_ACCESS_KEY != 0 }}
run: |
echo "VCPKG_BINARY_SOURCES=clear;x-aws,s3://vcpkg.cache.boinc/,readwrite" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf-8 -Append
- name: Check if build is running from fork
if: ${{ success() && (env.AWS_ACCESS_KEY_ID == 0 || env.AWS_SECRET_ACCESS_KEY == 0) }}
run: |
echo "VCPKG_BINARY_SOURCES=clear;x-aws-config,no-sign-request;x-aws,s3://vcpkg.cache.boinc/,read" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf-8 -Append
- name: Fix vcpkg
run: vcpkg.exe integrate remove
@ -52,18 +65,6 @@ jobs:
${{ github.workspace }}\3rdParty\Windows\cuda\
key: windows-${{ matrix.platform }}-${{ matrix.configuration }}-${{ hashFiles('win_build/vcpkg_3rdparty_dependencies_vs2019.vcxproj', '.github/workflows/windows.yml') }}
- name: Configure Python
if: ${{ success() }}
shell: cmd
run: |
pip install boto3
- name: Download vcpkg binary cache
if: ${{ success() }}
shell: cmd
run: |
python deploy\manage_vcpkg_archive_cache.py download ${{ github.workspace }}\3rdParty\buildCache\windows\vcpkgcache\ windows_${{ matrix.platform }} "edu.berkeley.boinc.github.actions.build.vcpkg.binary.cache"
- name: Build
run: msbuild win_build\boinc_vs2019.sln -p:Configuration=${{ matrix.configuration }} -p:Platform=${{ matrix.platform }} -p:VcpkgTripletConfig=ci -p:BuildConfig=ci -m
@ -91,21 +92,25 @@ jobs:
python deploy\prepare_deployment.py win_apps
python deploy\prepare_deployment.py win_client
python deploy\prepare_deployment.py win_manager
- name: Upload apps
uses: actions/upload-artifact@v2
with:
name: win_apps_${{ matrix.platform }}_${{ github.event.pull_request.head.sha }}
path: deploy/win_apps.7z
- name: Upload client
uses: actions/upload-artifact@v2
with:
name: win_client_${{ matrix.platform }}_${{ github.event.pull_request.head.sha }}
path: deploy/win_client.7z
- name: Upload manager
uses: actions/upload-artifact@v2
with:
name: win_manager_${{ matrix.platform }}_${{ github.event.pull_request.head.sha }}
path: deploy/win_manager.7z
- name: Upload symbol files
uses: actions/upload-artifact@v2
with:
@ -125,12 +130,3 @@ jobs:
with:
fail_ci_if_error: true
verbose: false
- name: Upload vcpkg binary cache
if: ${{ success() }}
shell: cmd
env:
ACCESS_KEY: ${{ secrets.S3_ACCESS_KEY }}
SECRET_KEY: ${{ secrets.S3_SECRET_KEY }}
run: |
python deploy\manage_vcpkg_archive_cache.py upload ${{ github.workspace }}\3rdParty\buildCache\windows\vcpkgcache\ windows_${{ matrix.platform }} "edu.berkeley.boinc.github.actions.build.vcpkg.binary.cache" "%ACCESS_KEY%" "%SECRET_KEY%"

1
android/buildAndroidBOINC-CI.sh
View File

@ -253,7 +253,6 @@ getTripletName()
}
if [ $build_with_vcpkg = "yes" ]; then
export XDG_CACHE_HOME=$cache_dir/vcpkgcache/
if [ $ci = "yes" ]; then
triplets_setup="ci"
else

2
linux/update_vcpkg_apps.sh
View File

@ -11,8 +11,6 @@ BUILD_DIR="$PWD/3rdParty/linux"
VCPKG_PORTS="$PWD/3rdParty/vcpkg_ports"
VCPKG_ROOT="$BUILD_DIR/vcpkg"
export XDG_CACHE_HOME=$CACHE_DIR/vcpkgcache
if [ ! -d $VCPKG_ROOT ]; then
mkdir -p $BUILD_DIR
git -C $BUILD_DIR clone https://github.com/microsoft/vcpkg

2
linux/update_vcpkg_client.sh
View File

@ -11,8 +11,6 @@ BUILD_DIR="$PWD/3rdParty/linux"
VCPKG_PORTS="$PWD/3rdParty/vcpkg_ports"
VCPKG_ROOT="$BUILD_DIR/vcpkg"
export XDG_CACHE_HOME=$CACHE_DIR/vcpkgcache
if [ ! -d $VCPKG_ROOT ]; then
mkdir -p $BUILD_DIR
git -C $BUILD_DIR clone https://github.com/microsoft/vcpkg

2
linux/update_vcpkg_libs.sh
View File

@ -11,8 +11,6 @@ BUILD_DIR="$PWD/3rdParty/linux"
VCPKG_PORTS="$PWD/3rdParty/vcpkg_ports"
VCPKG_ROOT="$BUILD_DIR/vcpkg"
export XDG_CACHE_HOME=$CACHE_DIR/vcpkgcache
if [ ! -d $VCPKG_ROOT ]; then
mkdir -p $BUILD_DIR
git -C $BUILD_DIR clone https://github.com/microsoft/vcpkg

2
mingw/update_vcpkg.sh
View File

@ -11,8 +11,6 @@ BUILD_DIR="$PWD/3rdParty/mingw"
VCPKG_PORTS="$PWD/3rdParty/vcpkg_ports"
VCPKG_ROOT="$BUILD_DIR/vcpkg"
export XDG_CACHE_HOME=$CACHE_DIR/vcpkgcache
if [ ! -d $VCPKG_ROOT ]; then
mkdir -p $BUILD_DIR
git -C $BUILD_DIR clone https://github.com/microsoft/vcpkg

6
win_build/vcpkg_3rdparty_dependencies_vs2019.vcxproj
View File

@ -178,8 +178,8 @@
<Exec Command="vcpkg.exe remove curl --overlay-triplets=../../vcpkg_ports/triplets/$(VcpkgTripletConfig) --triplet $(VcpkgTripletName)" WorkingDirectory="$(VcpkgRootDir)" ConsoleToMSBuild="true" Condition="$(OutdatedCurl) != ''" />
</Target>
<Target Name="Build3rdPartyLibraries" BeforeTargets="ClCompile" DependsOnTargets="RemoveOutdated3rdPartyLibraries" AfterTargets="RemoveOutdated3rdPartyLibraries">
<Exec Command="vcpkg.exe install openssl curl[core,schannel] wxwidgets opencl rappture gtest --overlay-triplets=../../vcpkg_ports/triplets/$(VcpkgTripletConfig) --triplet $(VcpkgTripletName) --recurse --clean-after-build" WorkingDirectory="$(VcpkgRootDir)" ConsoleToMSBuild="true" />
<Exec Command="vcpkg.exe install ftgl --overlay-triplets=../../vcpkg_ports/triplets/$(VcpkgTripletConfig) --triplet $(VcpkgTripletName) --clean-after-build" WorkingDirectory="$(VcpkgRootDir)" ConsoleToMSBuild="true" Condition="'$(Platform)' == 'x64'" />
<Exec Command="vcpkg.exe upgrade --no-dry-run --overlay-triplets=../../vcpkg_ports/triplets/$(VcpkgTripletConfig) --triplet $(VcpkgTripletName)" WorkingDirectory="$(VcpkgRootDir)" ConsoleToMSBuild="true" />
<Exec Command="vcpkg.exe install openssl curl[core,schannel] wxwidgets opencl rappture gtest --overlay-triplets=../../vcpkg_ports/triplets/$(VcpkgTripletConfig) --triplet $(VcpkgTripletName) --recurse --clean-after-build" WorkingDirectory="$(VcpkgRootDir)" ConsoleToMSBuild="true" ContinueOnError="WarnAndContinue" />
<Exec Command="vcpkg.exe install ftgl --overlay-triplets=../../vcpkg_ports/triplets/$(VcpkgTripletConfig) --triplet $(VcpkgTripletName) --clean-after-build" WorkingDirectory="$(VcpkgRootDir)" ConsoleToMSBuild="true" Condition="'$(Platform)' == 'x64'" ContinueOnError="WarnAndContinue" />
<Exec Command="vcpkg.exe upgrade --no-dry-run --overlay-triplets=../../vcpkg_ports/triplets/$(VcpkgTripletConfig) --triplet $(VcpkgTripletName)" WorkingDirectory="$(VcpkgRootDir)" ConsoleToMSBuild="true" ContinueOnError="WarnAndContinue" />
</Target>
</Project>