From e3710bc84a40dea2d7b0a1a99f390dbb6994fb43 Mon Sep 17 00:00:00 2001 From: Oliver Chang Date: Thu, 23 Jan 2025 19:56:56 +1100 Subject: [PATCH] chronos: Run build_cache_local.sh on cloud build (#12962) Also make some changes: 1. Support coverage builds by parameterising the sanitizer. 2. Build a total of 2 images: - us-central1-docker.pkg.dev/oss-fuzz/oss-fuzz-gen/${_PROJECT}-ofg-cached-${_SANITIZER} - us-central1-docker.pkg.dev/oss-fuzz/oss-fuzz-gen/${_PROJECT}-ofg-ccache-${_SANITIZER} The "cached" image will be different depending on whether replay worked or not. The "ccache" image will always use ccache. 3. Fix an incorrect exception path in bash_parser.py --- infra/base-images/base-builder/bash_parser.py | 2 +- infra/experimental/chronos/README.md | 2 +- .../experimental/chronos/build_cache_local.sh | 69 ++++++++------ infra/experimental/chronos/cloudbuild.yaml | 89 +++---------------- 4 files changed, 58 insertions(+), 104 deletions(-) diff --git a/infra/base-images/base-builder/bash_parser.py b/infra/base-images/base-builder/bash_parser.py index 3d27ca3fb..f383a3e39 100644 --- a/infra/base-images/base-builder/bash_parser.py +++ b/infra/base-images/base-builder/bash_parser.py @@ -196,7 +196,7 @@ def parse_script(bash_script, all_scripts) -> str: build_script = f.read() try: parts = bashlex.parse(build_script) - except bashlex.error.ParsingError: + except bashlex.errors.ParsingError: return '' for part in parts: new_script += handle_node(part, all_scripts, build_script) diff --git a/infra/experimental/chronos/README.md b/infra/experimental/chronos/README.md index 4dfe1baee..3c29d16c8 100644 --- a/infra/experimental/chronos/README.md +++ b/infra/experimental/chronos/README.md @@ -7,7 +7,7 @@ From the OSS-Fuzz root ```sh -$ RUN_ALL=1 ./infra/experimental/chronos/build_cache_local.sh htslib c +$ RUN_ALL=1 ./infra/experimental/chronos/build_cache_local.sh htslib c address ... ... Vanilla compile time: diff --git a/infra/experimental/chronos/build_cache_local.sh b/infra/experimental/chronos/build_cache_local.sh index df2edba31..64c203381 100755 --- a/infra/experimental/chronos/build_cache_local.sh +++ b/infra/experimental/chronos/build_cache_local.sh @@ -1,4 +1,4 @@ -#!/bin/bash -eux +#!/bin/bash -eu # Copyright 2024 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -17,9 +17,16 @@ _PROJECT=$1 _FUZZING_LANGUAGE=$2 +_SANITIZER=${3:-address} BASE=$PWD +# Final image is either ccache or replay script, depending on which worked. +FINAL_IMAGE_NAME=us-central1-docker.pkg.dev/oss-fuzz/oss-fuzz-gen/${_PROJECT}-ofg-cached-${_SANITIZER} + +# Always build an image with ccache. +CCACHE_IMAGE_NAME=us-central1-docker.pkg.dev/oss-fuzz/oss-fuzz-gen/${_PROJECT}-ofg-ccache-${_SANITIZER} + # Step 1: build the base image cd projects/${_PROJECT} docker build -t gcr.io/oss-fuzz/${_PROJECT} . @@ -31,13 +38,16 @@ cd ${BASE} mkdir -p ccaches/${_PROJECT} mkdir -p build/out/${_PROJECT} B_START=$SECONDS + +docker container rm -f ${_PROJECT}-origin-${_SANITIZER} + docker run \ --entrypoint=/bin/bash \ - --env=SANITIZER=address \ + --env=SANITIZER=${_SANITIZER} \ --env=CCACHE_DIR=/workspace/ccache \ --env=FUZZING_LANGUAGE=${_FUZZING_LANGUAGE} \ --env=CAPTURE_REPLAY_SCRIPT=1 \ - --name=${_PROJECT}-origin-asan \ + --name=${_PROJECT}-origin-${_SANITIZER} \ -v=$PWD/ccaches/${_PROJECT}/ccache:/workspace/ccache \ -v=$PWD/build/out/${_PROJECT}/:/out/ \ gcr.io/oss-fuzz/${_PROJECT} \ @@ -46,13 +56,12 @@ docker run \ B_TIME=$(($SECONDS - $B_START)) # Step 3: save (commit, locally) the cached container as an image -docker container commit ${_PROJECT}-origin-asan local/ossfuzz/${_PROJECT}-origin-asan - +docker container commit -c "ENV ENTRYPOINT=/bin/sh" -c "ENV REPLAY_ENABLED=1" ${_PROJECT}-origin-${_SANITIZER} $FINAL_IMAGE_NAME # Step 4: save the list of executables created from a vanilla build. This is # needed for validating if replay and ccaching works. # notes: run a shell the container with e.g. -# `docker run --entrypoint /bin/bash -it local/ossfuzz/htslib-origin-asan` +# `docker run --entrypoint /bin/bash -it local/ossfuzz/htslib-origin-address` executables_vanilla="$(find ./build/out/${_PROJECT} -executable -type f | sort)" @@ -62,13 +71,12 @@ executables_vanilla="$(find ./build/out/${_PROJECT} -executable -type f | sort)" # in the docker command. R_START=$SECONDS docker run \ - --entrypoint=/bin/bash \ - --env=SANITIZER=address \ - --env=REPLAY_ENABLED=1 \ + --rm \ + --env=SANITIZER=${_SANITIZER} \ --env=FUZZING_LANGUAGE=${_FUZZING_LANGUAGE} \ -v=$PWD/build/out/${_PROJECT}/:/out/ \ - --name=${_PROJECT}-origin-asan-replay-recached \ - local/ossfuzz/${_PROJECT}-origin-asan \ + --name=${_PROJECT}-origin-${_SANITIZER}-replay-recached \ + $FINAL_IMAGE_NAME \ -c \ "export PATH=/ccache/bin:\$PATH && rm -rf /out/* && compile" R_TIME=$(($SECONDS - $R_START)) @@ -83,17 +91,19 @@ echo "------------------------------------------------------" echo "Executables replay: " echo ${executables_replay} +REPLAY_WORKED= + # Step 7: match executables from vanilla builds and replay builds. # If this step is successful, then the process can exit as it's ready. if [[ "$executables_replay" == "$executables_vanilla" ]] then echo "Replay worked" - echo "Vanilla compile time:" - echo ${B_TIME} - echo "Replay compile time:" - echo ${R_TIME} + echo "Vanilla compile time: ${B_TIME}" + echo "Replay compile time: ${R_TIME}" - if [ -n "${RUN_ALL+1}" ]; then + REPLAY_WORKED=1 + + if [ -z "${RUN_ALL+1}" ]; then exit 0 fi else @@ -109,7 +119,7 @@ cd projects/${_PROJECT} # Step 9: Build an image with CCache's new items (modifications are done on the # dockerfile) -docker build -t us-central1-docker.pkg.dev/oss-fuzz/oss-fuzz-gen/${_PROJECT}-ofg-cached-address . +docker build -t $CCACHE_IMAGE_NAME . cd ${BASE} @@ -117,12 +127,13 @@ cd ${BASE} # Run the ccache build A_START=$SECONDS docker run \ + --rm \ --entrypoint=/bin/bash \ - --env=SANITIZER=address \ + --env=SANITIZER=${_SANITIZER} \ --env=FUZZING_LANGUAGE=${_FUZZING_LANGUAGE} \ - --name=${_PROJECT}-origin-asan-recached \ + --name=${_PROJECT}-origin-${_SANITIZER}-recached \ -v=$PWD/build/out/${_PROJECT}/:/out/ \ - us-central1-docker.pkg.dev/oss-fuzz/oss-fuzz-gen/${_PROJECT}-ofg-cached-address \ + $CCACHE_IMAGE_NAME \ -c \ "export PATH=/ccache/bin:\$PATH && rm -rf /out/* && compile" A_TIME=$(($SECONDS - $A_START)) @@ -134,21 +145,25 @@ executables_ccache="$(find ./build/out/${_PROJECT}/ -executable -type f | sort)" # Step 12: validate the ccache builds are successful if [[ "$executables_ccache" == "$executables_vanilla" ]] then - echo "Vanilla compile time:" - echo ${B_TIME} + echo "Vanilla compile time: ${B_TIME}" if [[ "$executables_replay" == "$executables_vanilla" ]] then echo "Replay worked" - echo "Replay compile time:" - echo ${R_TIME} + echo "Replay compile time: ${R_TIME}" fi - echo "Ccache compile time: " - echo ${A_TIME} + echo "Ccache compile time: ${A_TIME}" + + if [ -z "${REPLAY_WORKED}" ]; then + # Replay didn't work, so make the default "cached" image use the ccache one. + docker image tag \ + $CCACHE_IMAGE_NAME \ + $FINAL_IMAGE_NAME + fi exit 0 else echo "Replay and ccaching did not work." + exit 1 fi - diff --git a/infra/experimental/chronos/cloudbuild.yaml b/infra/experimental/chronos/cloudbuild.yaml index 2e4ae186b..7f584fc77 100644 --- a/infra/experimental/chronos/cloudbuild.yaml +++ b/infra/experimental/chronos/cloudbuild.yaml @@ -26,89 +26,28 @@ # TODO (David): add support for use of dedicated replay_build.sh steps: - name: 'gcr.io/cloud-builders/docker' + entrypoint: /bin/bash args: - - build - - -t - - gcr.io/oss-fuzz/${_PROJECT} - - . - dir: projects/${_PROJECT} -- name: ubuntu - args: - - mkdir - - /workspace/ccache -- name: 'gcr.io/cloud-builders/docker' - args: - - run - - --entrypoint=/bin/sh - - --env=SANITIZER=address - - --env=CCACHE_DIR=/workspace/ccache - - --env=FUZZING_LANGUAGE=${_FUZZING_LANGUAGE} - - --name=${_PROJECT}-origin-asan - - --volume=/workspace/ccache:/workspace/ccache - - gcr.io/oss-fuzz/${_PROJECT} - - -c - - "export PATH=/ccache/bin:$$PATH && compile && rm -rf /out/*" -- name: ubuntu - args: - - cp - - -rf - - /workspace/ccache - - projects/${_PROJECT}/ccache-cache -# Prepare for ccache environment -- name: ubuntu - args: - - /workspace/infra/experimental/chronos/prepare-ccache + - /workspace/infra/experimental/chronos/build_cache_local.sh - ${_PROJECT} -# Build the ccached instance + - ${_FUZZING_LANGUAGE} + - address + env: + - RUN_ALL=1 - name: 'gcr.io/cloud-builders/docker' + entrypoint: /bin/bash args: - - build - - -t - - us-central1-docker.pkg.dev/oss-fuzz/oss-fuzz-gen/${_PROJECT}-ofg-cached-address - - . - dir: projects/${_PROJECT} -# Cleanup the ccache stored in the project folder -- name: ubuntu - args: - - rm - - -rf - - projects/${_PROJECT}/ccache-cache - - /workspace/ccache -# Build coverage image -- name: 'gcr.io/cloud-builders/docker' - args: - - run - - --entrypoint=/bin/sh - - --env=SANITIZER=coverage - - --env=CCACHE_DIR=/workspace/ccache - - --env=FUZZING_LANGUAGE=${_FUZZING_LANGUAGE} - - --name=${_PROJECT}-origin-cov - - --volume=/workspace/ccache:/workspace/ccache - - gcr.io/oss-fuzz/${_PROJECT} - - -c - - "export PATH=/ccache/bin:$$PATH && compile && rm -rf /out/*" -- name: ubuntu - args: - - cp - - -rf - - /workspace/ccache - - projects/${_PROJECT}/ccache-cache -# Prepare for ccache environment -- name: ubuntu - args: - - /workspace/infra/experimental/chronos/prepare-ccache + - /workspace/infra/experimental/chronos/build_cache_local.sh - ${_PROJECT} -# Build the ccached instance -- name: 'gcr.io/cloud-builders/docker' - args: - - build - - -t - - us-central1-docker.pkg.dev/oss-fuzz/oss-fuzz-gen/${_PROJECT}-ofg-cached-coverage - - . - dir: projects/${_PROJECT} + - ${_FUZZING_LANGUAGE} + - coverage + env: + - RUN_ALL=1 images: - us-central1-docker.pkg.dev/oss-fuzz/oss-fuzz-gen/${_PROJECT}-ofg-cached-address - us-central1-docker.pkg.dev/oss-fuzz/oss-fuzz-gen/${_PROJECT}-ofg-cached-coverage +- us-central1-docker.pkg.dev/oss-fuzz/oss-fuzz-gen/${_PROJECT}-ofg-ccache-address +- us-central1-docker.pkg.dev/oss-fuzz/oss-fuzz-gen/${_PROJECT}-ofg-ccache-coverage timeout: 72000s # 20 hours, same as build_lib.py logsBucket: oss-fuzz-gcb-logs tags: