mirror of https://github.com/google/oss-fuzz.git
chronos: Run build_cache_local.sh on cloud build (#12962)
Also make some changes:
1. Support coverage builds by parameterising the sanitizer.
2. Build a total of 2 images:
-
us-central1-docker.pkg.dev/oss-fuzz/oss-fuzz-gen/${_PROJECT}-ofg-cached-${_SANITIZER}
-
us-central1-docker.pkg.dev/oss-fuzz/oss-fuzz-gen/${_PROJECT}-ofg-ccache-${_SANITIZER}
The "cached" image will be different depending on whether replay worked
or not. The "ccache" image will always use ccache.
3. Fix an incorrect exception path in bash_parser.py
This commit is contained in:
Oliver Chang
GitHub
parent
feacfb3200
commit
e3710bc84a
|
|
@ -196,7 +196,7 @@ def parse_script(bash_script, all_scripts) -> str:
|
|||
build_script = f.read()
|
||||
try:
|
||||
parts = bashlex.parse(build_script)
|
||||
except bashlex.error.ParsingError:
|
||||
except bashlex.errors.ParsingError:
|
||||
return ''
|
||||
for part in parts:
|
||||
new_script += handle_node(part, all_scripts, build_script)
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@
|
|||
From the OSS-Fuzz root
|
||||
|
||||
```sh
|
||||
$ RUN_ALL=1 ./infra/experimental/chronos/build_cache_local.sh htslib c
|
||||
$ RUN_ALL=1 ./infra/experimental/chronos/build_cache_local.sh htslib c address
|
||||
...
|
||||
...
|
||||
Vanilla compile time:
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
#!/bin/bash -eux
|
||||
#!/bin/bash -eu
|
||||
# Copyright 2024 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
|
|
@ -17,9 +17,16 @@
|
|||
|
||||
_PROJECT=$1
|
||||
_FUZZING_LANGUAGE=$2
|
||||
_SANITIZER=${3:-address}
|
||||
|
||||
BASE=$PWD
|
||||
|
||||
# Final image is either ccache or replay script, depending on which worked.
|
||||
FINAL_IMAGE_NAME=us-central1-docker.pkg.dev/oss-fuzz/oss-fuzz-gen/${_PROJECT}-ofg-cached-${_SANITIZER}
|
||||
|
||||
# Always build an image with ccache.
|
||||
CCACHE_IMAGE_NAME=us-central1-docker.pkg.dev/oss-fuzz/oss-fuzz-gen/${_PROJECT}-ofg-ccache-${_SANITIZER}
|
||||
|
||||
# Step 1: build the base image
|
||||
cd projects/${_PROJECT}
|
||||
docker build -t gcr.io/oss-fuzz/${_PROJECT} .
|
||||
|
|
@ -31,13 +38,16 @@ cd ${BASE}
|
|||
mkdir -p ccaches/${_PROJECT}
|
||||
mkdir -p build/out/${_PROJECT}
|
||||
B_START=$SECONDS
|
||||
|
||||
docker container rm -f ${_PROJECT}-origin-${_SANITIZER}
|
||||
|
||||
docker run \
|
||||
--entrypoint=/bin/bash \
|
||||
--env=SANITIZER=address \
|
||||
--env=SANITIZER=${_SANITIZER} \
|
||||
--env=CCACHE_DIR=/workspace/ccache \
|
||||
--env=FUZZING_LANGUAGE=${_FUZZING_LANGUAGE} \
|
||||
--env=CAPTURE_REPLAY_SCRIPT=1 \
|
||||
--name=${_PROJECT}-origin-asan \
|
||||
--name=${_PROJECT}-origin-${_SANITIZER} \
|
||||
-v=$PWD/ccaches/${_PROJECT}/ccache:/workspace/ccache \
|
||||
-v=$PWD/build/out/${_PROJECT}/:/out/ \
|
||||
gcr.io/oss-fuzz/${_PROJECT} \
|
||||
|
|
@ -46,13 +56,12 @@ docker run \
|
|||
B_TIME=$(($SECONDS - $B_START))
|
||||
|
||||
# Step 3: save (commit, locally) the cached container as an image
|
||||
docker container commit ${_PROJECT}-origin-asan local/ossfuzz/${_PROJECT}-origin-asan
|
||||
|
||||
docker container commit -c "ENV ENTRYPOINT=/bin/sh" -c "ENV REPLAY_ENABLED=1" ${_PROJECT}-origin-${_SANITIZER} $FINAL_IMAGE_NAME
|
||||
|
||||
# Step 4: save the list of executables created from a vanilla build. This is
|
||||
# needed for validating if replay and ccaching works.
|
||||
# notes: run a shell the container with e.g.
|
||||
# `docker run --entrypoint /bin/bash -it local/ossfuzz/htslib-origin-asan`
|
||||
# `docker run --entrypoint /bin/bash -it local/ossfuzz/htslib-origin-address`
|
||||
executables_vanilla="$(find ./build/out/${_PROJECT} -executable -type f | sort)"
|
||||
|
||||
|
||||
|
|
@ -62,13 +71,12 @@ executables_vanilla="$(find ./build/out/${_PROJECT} -executable -type f | sort)"
|
|||
# in the docker command.
|
||||
R_START=$SECONDS
|
||||
docker run \
|
||||
--entrypoint=/bin/bash \
|
||||
--env=SANITIZER=address \
|
||||
--env=REPLAY_ENABLED=1 \
|
||||
--rm \
|
||||
--env=SANITIZER=${_SANITIZER} \
|
||||
--env=FUZZING_LANGUAGE=${_FUZZING_LANGUAGE} \
|
||||
-v=$PWD/build/out/${_PROJECT}/:/out/ \
|
||||
--name=${_PROJECT}-origin-asan-replay-recached \
|
||||
local/ossfuzz/${_PROJECT}-origin-asan \
|
||||
--name=${_PROJECT}-origin-${_SANITIZER}-replay-recached \
|
||||
$FINAL_IMAGE_NAME \
|
||||
-c \
|
||||
"export PATH=/ccache/bin:\$PATH && rm -rf /out/* && compile"
|
||||
R_TIME=$(($SECONDS - $R_START))
|
||||
|
|
@ -83,17 +91,19 @@ echo "------------------------------------------------------"
|
|||
echo "Executables replay: "
|
||||
echo ${executables_replay}
|
||||
|
||||
REPLAY_WORKED=
|
||||
|
||||
# Step 7: match executables from vanilla builds and replay builds.
|
||||
# If this step is successful, then the process can exit as it's ready.
|
||||
if [[ "$executables_replay" == "$executables_vanilla" ]]
|
||||
then
|
||||
echo "Replay worked"
|
||||
echo "Vanilla compile time:"
|
||||
echo ${B_TIME}
|
||||
echo "Replay compile time:"
|
||||
echo ${R_TIME}
|
||||
echo "Vanilla compile time: ${B_TIME}"
|
||||
echo "Replay compile time: ${R_TIME}"
|
||||
|
||||
if [ -n "${RUN_ALL+1}" ]; then
|
||||
REPLAY_WORKED=1
|
||||
|
||||
if [ -z "${RUN_ALL+1}" ]; then
|
||||
exit 0
|
||||
fi
|
||||
else
|
||||
|
|
@ -109,7 +119,7 @@ cd projects/${_PROJECT}
|
|||
|
||||
# Step 9: Build an image with CCache's new items (modifications are done on the
|
||||
# dockerfile)
|
||||
docker build -t us-central1-docker.pkg.dev/oss-fuzz/oss-fuzz-gen/${_PROJECT}-ofg-cached-address .
|
||||
docker build -t $CCACHE_IMAGE_NAME .
|
||||
|
||||
cd ${BASE}
|
||||
|
||||
|
|
@ -117,12 +127,13 @@ cd ${BASE}
|
|||
# Run the ccache build
|
||||
A_START=$SECONDS
|
||||
docker run \
|
||||
--rm \
|
||||
--entrypoint=/bin/bash \
|
||||
--env=SANITIZER=address \
|
||||
--env=SANITIZER=${_SANITIZER} \
|
||||
--env=FUZZING_LANGUAGE=${_FUZZING_LANGUAGE} \
|
||||
--name=${_PROJECT}-origin-asan-recached \
|
||||
--name=${_PROJECT}-origin-${_SANITIZER}-recached \
|
||||
-v=$PWD/build/out/${_PROJECT}/:/out/ \
|
||||
us-central1-docker.pkg.dev/oss-fuzz/oss-fuzz-gen/${_PROJECT}-ofg-cached-address \
|
||||
$CCACHE_IMAGE_NAME \
|
||||
-c \
|
||||
"export PATH=/ccache/bin:\$PATH && rm -rf /out/* && compile"
|
||||
A_TIME=$(($SECONDS - $A_START))
|
||||
|
|
@ -134,21 +145,25 @@ executables_ccache="$(find ./build/out/${_PROJECT}/ -executable -type f | sort)"
|
|||
# Step 12: validate the ccache builds are successful
|
||||
if [[ "$executables_ccache" == "$executables_vanilla" ]]
|
||||
then
|
||||
echo "Vanilla compile time:"
|
||||
echo ${B_TIME}
|
||||
echo "Vanilla compile time: ${B_TIME}"
|
||||
if [[ "$executables_replay" == "$executables_vanilla" ]]
|
||||
then
|
||||
echo "Replay worked"
|
||||
echo "Replay compile time:"
|
||||
echo ${R_TIME}
|
||||
echo "Replay compile time: ${R_TIME}"
|
||||
fi
|
||||
|
||||
echo "Ccache compile time: "
|
||||
echo ${A_TIME}
|
||||
echo "Ccache compile time: ${A_TIME}"
|
||||
|
||||
if [ -z "${REPLAY_WORKED}" ]; then
|
||||
# Replay didn't work, so make the default "cached" image use the ccache one.
|
||||
docker image tag \
|
||||
$CCACHE_IMAGE_NAME \
|
||||
$FINAL_IMAGE_NAME
|
||||
fi
|
||||
|
||||
exit 0
|
||||
else
|
||||
echo "Replay and ccaching did not work."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -26,89 +26,28 @@
|
|||
# TODO (David): add support for use of dedicated replay_build.sh
|
||||
steps:
|
||||
- name: 'gcr.io/cloud-builders/docker'
|
||||
entrypoint: /bin/bash
|
||||
args:
|
||||
- build
|
||||
- -t
|
||||
- gcr.io/oss-fuzz/${_PROJECT}
|
||||
- .
|
||||
dir: projects/${_PROJECT}
|
||||
- name: ubuntu
|
||||
args:
|
||||
- mkdir
|
||||
- /workspace/ccache
|
||||
- name: 'gcr.io/cloud-builders/docker'
|
||||
args:
|
||||
- run
|
||||
- --entrypoint=/bin/sh
|
||||
- --env=SANITIZER=address
|
||||
- --env=CCACHE_DIR=/workspace/ccache
|
||||
- --env=FUZZING_LANGUAGE=${_FUZZING_LANGUAGE}
|
||||
- --name=${_PROJECT}-origin-asan
|
||||
- --volume=/workspace/ccache:/workspace/ccache
|
||||
- gcr.io/oss-fuzz/${_PROJECT}
|
||||
- -c
|
||||
- "export PATH=/ccache/bin:$$PATH && compile && rm -rf /out/*"
|
||||
- name: ubuntu
|
||||
args:
|
||||
- cp
|
||||
- -rf
|
||||
- /workspace/ccache
|
||||
- projects/${_PROJECT}/ccache-cache
|
||||
# Prepare for ccache environment
|
||||
- name: ubuntu
|
||||
args:
|
||||
- /workspace/infra/experimental/chronos/prepare-ccache
|
||||
- /workspace/infra/experimental/chronos/build_cache_local.sh
|
||||
- ${_PROJECT}
|
||||
# Build the ccached instance
|
||||
- ${_FUZZING_LANGUAGE}
|
||||
- address
|
||||
env:
|
||||
- RUN_ALL=1
|
||||
- name: 'gcr.io/cloud-builders/docker'
|
||||
entrypoint: /bin/bash
|
||||
args:
|
||||
- build
|
||||
- -t
|
||||
- us-central1-docker.pkg.dev/oss-fuzz/oss-fuzz-gen/${_PROJECT}-ofg-cached-address
|
||||
- .
|
||||
dir: projects/${_PROJECT}
|
||||
# Cleanup the ccache stored in the project folder
|
||||
- name: ubuntu
|
||||
args:
|
||||
- rm
|
||||
- -rf
|
||||
- projects/${_PROJECT}/ccache-cache
|
||||
- /workspace/ccache
|
||||
# Build coverage image
|
||||
- name: 'gcr.io/cloud-builders/docker'
|
||||
args:
|
||||
- run
|
||||
- --entrypoint=/bin/sh
|
||||
- --env=SANITIZER=coverage
|
||||
- --env=CCACHE_DIR=/workspace/ccache
|
||||
- --env=FUZZING_LANGUAGE=${_FUZZING_LANGUAGE}
|
||||
- --name=${_PROJECT}-origin-cov
|
||||
- --volume=/workspace/ccache:/workspace/ccache
|
||||
- gcr.io/oss-fuzz/${_PROJECT}
|
||||
- -c
|
||||
- "export PATH=/ccache/bin:$$PATH && compile && rm -rf /out/*"
|
||||
- name: ubuntu
|
||||
args:
|
||||
- cp
|
||||
- -rf
|
||||
- /workspace/ccache
|
||||
- projects/${_PROJECT}/ccache-cache
|
||||
# Prepare for ccache environment
|
||||
- name: ubuntu
|
||||
args:
|
||||
- /workspace/infra/experimental/chronos/prepare-ccache
|
||||
- /workspace/infra/experimental/chronos/build_cache_local.sh
|
||||
- ${_PROJECT}
|
||||
# Build the ccached instance
|
||||
- name: 'gcr.io/cloud-builders/docker'
|
||||
args:
|
||||
- build
|
||||
- -t
|
||||
- us-central1-docker.pkg.dev/oss-fuzz/oss-fuzz-gen/${_PROJECT}-ofg-cached-coverage
|
||||
- .
|
||||
dir: projects/${_PROJECT}
|
||||
- ${_FUZZING_LANGUAGE}
|
||||
- coverage
|
||||
env:
|
||||
- RUN_ALL=1
|
||||
images:
|
||||
- us-central1-docker.pkg.dev/oss-fuzz/oss-fuzz-gen/${_PROJECT}-ofg-cached-address
|
||||
- us-central1-docker.pkg.dev/oss-fuzz/oss-fuzz-gen/${_PROJECT}-ofg-cached-coverage
|
||||
- us-central1-docker.pkg.dev/oss-fuzz/oss-fuzz-gen/${_PROJECT}-ofg-ccache-address
|
||||
- us-central1-docker.pkg.dev/oss-fuzz/oss-fuzz-gen/${_PROJECT}-ofg-ccache-coverage
|
||||
timeout: 72000s # 20 hours, same as build_lib.py
|
||||
logsBucket: oss-fuzz-gcb-logs
|
||||
tags:
|
||||
|
|
|
|||
Loading…
Reference in New Issue