[infra] Enable bad build checks once again. (#838)

* [infra] Enable bad build checks once again.

* Minor typo.

* [bad_example] Update build flags for reproducing bad instrumentation scenario.

* [bad_example] split bad/no instrumentation case into two different ones.

* Use new approach for partial instrumentation detection + do that only for libFuzzer.

* Rename bad_example_bad_instrumentation into bad_example_partial_instrumentation.

* Calculate number of broken targets and fail if 10+% are broken.

* Multiprocess madness.

* Always run all checks and store all errors + clean up the code and add comments

* Add special handling for the projects with very small fuzz targets.

* Remove unnecessary semicolon.

* Address review comments.

* Address more review comments, small refactoring.
This commit is contained in:
Max Moroz 2018-04-17 09:31:53 -07:00 committed by GitHub
parent ad80480393
commit 4df2262466
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 191 additions and 102 deletions

View File

@ -1,4 +1,4 @@
#!/bin/bash -ux
#!/bin/bash -u
# Copyright 2017 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
@ -15,56 +15,65 @@
#
################################################################################
# Verify that the given fuzzer has proper coverage instrumentation.
# A minimal number of runs to test fuzz target with a non-empty input.
MIN_NUMBER_OF_RUNS=4
# The "example" target has 73 with ASan, 65 with UBSan, and 6648 with MSan.
# Real world targets have greater values (arduinojson: 407, zlib: 664).
THRESHOLD_FOR_NUMBER_OF_EDGES=256
# Threshold values for different sanitizers used by instrumentation checks.
ASAN_CALLS_THRESHOLD_FOR_ASAN_BUILD=1000
ASAN_CALLS_THRESHOLD_FOR_NON_ASAN_BUILD=0
MSAN_CALLS_THRESHOLD_FOR_MSAN_BUILD=1000
MSAN_CALLS_THRESHOLD_FOR_NON_MSAN_BUILD=0
UBSAN_CALLS_THRESHOLD_FOR_UBSAN_BUILD=350
UBSAN_CALLS_THRESHOLD_FOR_NON_UBSAN_BUILD=200
# Verify that the given fuzz target has proper coverage instrumentation.
function check_instrumentation {
local FUZZER=$1
local CHECK_FAILED=0
local FUZZER_OUTPUT="/tmp/$(basename $FUZZER).output"
if [[ "$FUZZING_ENGINE" = libfuzzer ]]; then
CHECK_FAILED=$($FUZZER -max_total_time=4 2>&1 | egrep "ERROR: no interesting inputs were found. Is the code instrumented" -c)
if (( $CHECK_FAILED > 0 )); then
echo "BAD BUILD: the code does not seem to have coverage instrumentation."
fi
fi
# honggfuzz is not fully integrated, avoid performing the following check.
if [[ "$FUZZING_ENGINE" != honggfuzz ]]; then
if (( $CHECK_FAILED == 0 )); then
# The "example" target has 93 when built with ASan and 67 with UBSan. Real
# targets have greater values (arduinojson: 413, libteken: 519, zlib: 586).
local THRESHOLD_FOR_NUMBER_OF_EDGES=90
if [[ $SANITIZER = undefined ]]; then
THRESHOLD_FOR_NUMBER_OF_EDGES=65
fi
local NUMBER_OF_EDGES=$(sancov -print-coverage-pcs $FUZZER | wc -l)
if (( $NUMBER_OF_EDGES < $THRESHOLD_FOR_NUMBER_OF_EDGES )); then
echo "BAD BUILD: the code does not seem to have coverage instrumentation."
CHECK_FAILED=1
fi
fi
if [[ "$FUZZING_ENGINE" != libfuzzer ]]; then
return
fi
# Store fuzz target's output into a temp file to be used for further checks.
$FUZZER -runs=$MIN_NUMBER_OF_RUNS &>$FUZZER_OUTPUT
CHECK_FAILED=$(egrep "ERROR: no interesting inputs were found. Is the code instrumented" -c $FUZZER_OUTPUT)
if (( $CHECK_FAILED > 0 )); then
exit 1
echo "BAD BUILD: the target does not seem to have coverage instrumentation."
# Bail out as the further check does not make any sense, there are 0 PCs.
return 1
fi
local NUMBER_OF_EDGES=$(grep -Po "INFO: Loaded [[:digit:]]+ module.*\(.*counters\):[[:space:]]+\K[[:digit:]]+" $FUZZER_OUTPUT)
if (( $NUMBER_OF_EDGES < $THRESHOLD_FOR_NUMBER_OF_EDGES )); then
echo "BAD BUILD: the target seems to have only partial coverage instrumentation."
fi
}
# Verify that the given fuzzer has been built properly and works as expected.
# Verify that the given fuzz target has been built properly and works.
function check_startup_crash {
local FUZZER=$1
local CHECK_PASSED=0
if [[ "$FUZZING_ENGINE" = libfuzzer ]]; then
CHECK_PASSED=$($FUZZER -runs=4 2>&1 | egrep "Done 4 runs" -c)
CHECK_PASSED=$($FUZZER -runs=$MIN_NUMBER_OF_RUNS 2>&1 | egrep "Done $MIN_NUMBER_OF_RUNS runs" -c)
else
# TODO: add checks for another fuzzing engines if possible.
CHECK_PASSED=1
fi
if (( $CHECK_PASSED == 0 )); then
if [ "$CHECK_PASSED" -eq "0" ]; then
echo "BAD BUILD: the fuzzer seems to have either startup crash or exit."
exit 1
fi
}
@ -75,26 +84,17 @@ function check_asan_build {
local MSAN_CALLS=$3
local UBSAN_CALLS=$4
# Perform all the checks for more verbose error message.
local CHECK_FAILED=0
if (( $ASAN_CALLS < 1000 )); then
# Perform all the checks for more detailed error message.
if (( $ASAN_CALLS < $ASAN_CALLS_THRESHOLD_FOR_ASAN_BUILD )); then
echo "BAD BUILD: $FUZZER does not seem to be compiled with ASan."
CHECK_FAILED=1
fi
if (( $MSAN_CALLS > 0 )); then
if (( $MSAN_CALLS > $MSAN_CALLS_THRESHOLD_FOR_NON_MSAN_BUILD )); then
echo "BAD BUILD: ASan build of $FUZZER seems to be compiled with MSan."
CHECK_FAILED=1
fi
if (( $UBSAN_CALLS > 250 )); then
if (( $UBSAN_CALLS > $UBSAN_CALLS_THRESHOLD_FOR_NON_UBSAN_BUILD )); then
echo "BAD BUILD: ASan build of $FUZZER seems to be compiled with UBSan."
CHECK_FAILED=1
fi
if (( $CHECK_FAILED > 0 )); then
exit 1
fi
}
@ -105,26 +105,17 @@ function check_msan_build {
local MSAN_CALLS=$3
local UBSAN_CALLS=$4
# Perform all the checks for more verbose error message.
local CHECK_FAILED=0
if (( $ASAN_CALLS > 0 )); then
# Perform all the checks for more detailed error message.
if (( $ASAN_CALLS > $ASAN_CALLS_THRESHOLD_FOR_NON_ASAN_BUILD )); then
echo "BAD BUILD: MSan build of $FUZZER seems to be compiled with ASan."
CHECK_FAILED=1
fi
if (( $MSAN_CALLS < 1000 )); then
if (( $MSAN_CALLS < $MSAN_CALLS_THRESHOLD_FOR_MSAN_BUILD )); then
echo "BAD BUILD: $FUZZER does not seem to be compiled with UBSan."
CHECK_FAILED=1
fi
if (( $UBSAN_CALLS > 250 )); then
if (( $UBSAN_CALLS > $UBSAN_CALLS_THRESHOLD_FOR_NON_UBSAN_BUILD )); then
echo "BAD BUILD: MSan build of $FUZZER seems to be compiled with UBSan."
CHECK_FAILED=1
fi
if (( $CHECK_FAILED > 0 )); then
exit 1
fi
}
@ -142,38 +133,28 @@ function check_ubsan_build {
return 0
fi
# Perform all the checks for more verbose error message.
local CHECK_FAILED=0
if (( $ASAN_CALLS > 0 )); then
# Perform all the checks for more detailed error message.
if (( $ASAN_CALLS > $ASAN_CALLS_THRESHOLD_FOR_NON_ASAN_BUILD )); then
echo "BAD BUILD: UBSan build of $FUZZER seems to be compiled with ASan."
CHECK_FAILED=1
fi
if (( $MSAN_CALLS > 0 )); then
if (( $MSAN_CALLS > $MSAN_CALLS_THRESHOLD_FOR_NON_MSAN_BUILD )); then
echo "BAD BUILD: UBSan build of $FUZZER seems to be compiled with MSan."
CHECK_FAILED=1
fi
if (( $UBSAN_CALLS < 250 )); then
if (( $UBSAN_CALLS < $UBSAN_CALLS_THRESHOLD_FOR_UBSAN_BUILD )); then
echo "BAD BUILD: $FUZZER does not seem to be compiled with UBSan."
CHECK_FAILED=1
fi
if (( $CHECK_FAILED > 0 )); then
exit 1
fi
}
# Verify that the given fuzz target is compiled with correct sanitizer.
function check_mixed_sanitizers {
local FUZZER=$1
local ASAN_CALLS=$(objdump -dC $FUZZER | egrep "callq\s+[0-9a-f]+\s+<__asan" -c)
local MSAN_CALLS=$(objdump -dC $FUZZER | egrep "callq\s+[0-9a-f]+\s+<__msan" -c)
local UBSAN_CALLS=$(objdump -dC $FUZZER | egrep "callq\s+[0-9a-f]+\s+<__ubsan" -c)
local CHECK_FAILED=0
if [[ "$SANITIZER" = address ]]; then
check_asan_build $FUZZER $ASAN_CALLS $MSAN_CALLS $UBSAN_CALLS
elif [[ "$SANITIZER" = memory ]]; then
@ -186,7 +167,6 @@ function check_mixed_sanitizers {
function main {
local FUZZER=$1
echo "INFO: performing bad build checks for $FUZZER"
check_instrumentation $FUZZER
check_mixed_sanitizers $FUZZER
@ -199,6 +179,7 @@ if [ $# -ne 1 ]; then
exit 1
fi
# Fuzz target path.
FUZZER=$1
main $FUZZER
echo "INFO: $FUZZER has passed bad_build_check tests."

View File

@ -1,4 +1,4 @@
#!/bin/bash -eu
#!/bin/bash -u
# Copyright 2016 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
@ -15,9 +15,24 @@
#
################################################################################
# Test every fuzzer. Fails on first fuzzer failure.
# Percentage threshold that needs to be reached for marking a build as failed.
ALLOWED_BROKEN_TARGETS_PERCENTAGE=10
N=0
# Test all fuzz targets in the $OUT/ dir.
TOTAL_TARGETS_COUNT=0
# Number of CPUs available, this is needed for running tests in parallel.
NPROC=$(nproc)
# Directories where bad build check results will be written to.
VALID_TARGETS_DIR="/tmp/valid_fuzz_targets"
BROKEN_TARGETS_DIR="/tmp/broken_fuzz_targets"
rm -rf $VALID_TARGETS_DIR
rm -rf $BROKEN_TARGETS_DIR
mkdir $VALID_TARGETS_DIR
mkdir $BROKEN_TARGETS_DIR
# Main loop that iterates through all fuzz targets and runs the check.
for FUZZER_BINARY in $(find $OUT/ -executable -type f); do
if file "$FUZZER_BINARY" | grep -v ELF > /dev/null 2>&1; then
continue
@ -31,8 +46,6 @@ for FUZZER_BINARY in $(find $OUT/ -executable -type f); do
continue
fi
echo "testing $FUZZER"
if [ "$SKIP_TEST_TARGET_RUN" != "1" ]; then
if [[ "$FUZZING_ENGINE" = honggfuzz ]]; then
timeout --preserve-status -s INT 20s run_fuzzer $FUZZER
@ -44,14 +57,80 @@ for FUZZER_BINARY in $(find $OUT/ -executable -type f); do
fi
fi
bad_build_check $FUZZER_BINARY
N=$[$N+1]
echo "INFO: performing bad build checks for $FUZZER_BINARY."
LOG_PATH_FOR_BROKEN_TARGET="${BROKEN_TARGETS_DIR}/${FUZZER}"
# Launch bad build check process in the background.
bad_build_check $FUZZER_BINARY > $LOG_PATH_FOR_BROKEN_TARGET &
# Count total number of fuzz targets being tested.
TOTAL_TARGETS_COUNT=$[$TOTAL_TARGETS_COUNT+1]
# Do not spawn more processes than the number of CPUs available.
n_child_proc=$(jobs -p | wc -l)
while [ "$n_child_proc" -eq "$NPROC" ]; do
sleep 4
n_child_proc=$(jobs -p | wc -l)
done
done
if [ "$N" -eq "0" ]; then
# Wait for background processes to finish.
wait
# Sanity check in case there are no fuzz targets in the $OUT/ dir.
if [ "$TOTAL_TARGETS_COUNT" -eq "0" ]; then
echo "ERROR: no fuzzers found in $OUT/"
ls -al $OUT
exit 1
fi
echo "$N fuzzers total"
# An empty log file indicated that corresponding fuzz target is not broken.
find $BROKEN_TARGETS_DIR -empty -exec mv {} $VALID_TARGETS_DIR \;
# Calculate number of valid and broken fuzz targets.
VALID_TARGETS_COUNT=$(ls $VALID_TARGETS_DIR | wc -l)
BROKEN_TARGETS_COUNT=$(ls $BROKEN_TARGETS_DIR | wc -l)
# Sanity check to make sure that bad build check doesn't skip any fuzz target.
if [ "$TOTAL_TARGETS_COUNT" -ne "$[$VALID_TARGETS_COUNT+$BROKEN_TARGETS_COUNT]" ]; then
echo "ERROR: bad_build_check seems to have a bug, total number of fuzz" \
"does not match number of fuzz targets tested."
echo "Total fuzz targets ($TOTAL_TARGETS_COUNT):"
ls -al $OUT
echo "Valid fuzz targets ($VALID_TARGETS_COUNT):"
ls -al $VALID_TARGETS_DIR
echo "Total fuzz targets ($BROKEN_TARGETS_COUNT):"
ls -al $BROKEN_TARGETS_DIR
exit 1
fi
# Build info about all broken fuzz targets (if any).
if [ "$BROKEN_TARGETS_COUNT" -gt "0" ]; then
echo "Broken fuzz targets ($BROKEN_TARGETS_COUNT):"
for target in $(ls $BROKEN_TARGETS_DIR); do
echo "${target}:"
cat ${BROKEN_TARGETS_DIR}/${target}
done
fi
# Calculate the percentage of broken fuzz targets and make the finel decision.
BROKEN_TARGETS_PERCENTAGE=$[$BROKEN_TARGETS_COUNT*100/$TOTAL_TARGETS_COUNT]
if [ "$BROKEN_TARGETS_PERCENTAGE" -gt "$ALLOWED_BROKEN_TARGETS_PERCENTAGE" ]; then
echo "ERROR: $BROKEN_TARGETS_PERCENTAGE% of fuzz targets seem to be broken." \
"See the list above for a detailed information."
# TODO: figure out how to not fail the "special" cases handled below. Those
# are from "example" and "c-ares" projects and are too small targets to pass.
if [ "$(ls $OUT/do_stuff_fuzzer $OUT/ares_*_fuzzer 2>/dev/null | wc -l)" -gt "0" ]; then
exit 0
fi
exit 1
else
echo "$TOTAL_TARGETS_COUNT fuzzers total, $BROKEN_TARGETS_COUNT seem to be" \
"broken ($BROKEN_TARGETS_PERCENTAGE%)."
exit 0
fi

View File

@ -195,27 +195,43 @@ def get_build_steps(project_yaml, dockerfile_path):
env.append('OUT=' + out)
env.append('MSAN_LIBS_PATH=/workspace/msan')
# To disable running of all fuzz targets while doing |test_all| step, as
# that step is currently being used for performing bad build checks only.
env.append('SKIP_TEST_TARGET_RUN=1')
workdir = workdir_from_dockerfile(dockerfile_path)
if not workdir:
workdir = '/src'
build_steps.append({
build_steps.extend([
# compile
'name': image,
'env': env,
'args': [
'bash',
'-c',
# Remove /out to break loudly when a build script incorrectly uses
# /out instead of $OUT.
# `cd /src && cd {workdir}` (where {workdir} is parsed from the
# Dockerfile). Container Builder overrides our workdir so we need to add
# this step to set it back.
# We also remove /work and /src to save disk space after a step.
# Container Builder doesn't pass --rm to docker run yet.
'rm -r /out && cd /src && cd {1} && mkdir -p {0} && compile && rm -rf /work && rm -rf /src'.format(out, workdir),
],
})
{'name': image,
'env': env,
'args': [
'bash',
'-c',
# Remove /out to break loudly when a build script incorrectly uses
# /out instead of $OUT.
# `cd /src && cd {workdir}` (where {workdir} is parsed from the
# Dockerfile). Container Builder overrides our workdir so we need to add
# this step to set it back.
# We also remove /work and /src to save disk space after a step.
# Container Builder doesn't pass --rm to docker run yet.
'rm -r /out && cd /src && cd {1} && mkdir -p {0} && compile && rm -rf /work && rm -rf /src'.format(out, workdir),
],
},
# test binaries
{'name': 'gcr.io/oss-fuzz-base/base-runner',
'env': env,
'args': [
'bash',
'-c',
# Verify that fuzzers have been built properly and are not broken.
# TODO(mmoroz): raise a notification if not passing the tests.
'test_all'
],
},
])
if sanitizer == 'memory':
# Patch dynamic libraries to use instrumented ones.

View File

@ -28,21 +28,34 @@ if [[ $SANITIZER = *coverage* ]] || [[ $SANITIZER = *profile* ]]; then
fi
# Testcase 3. Ignore the flags provided by OSS-Fuzz.
# Testcase 3. Partially ignore the flags provided by OSS-Fuzz.
################################################################################
export CFLAGS_ORIG="$CFLAGS"
export CFLAGS="-O1"
export CXXFLAGS_ORIG="$CXXFLAGS"
export CXXFLAGS="-O1 -stdlib=libc++"
./configure
make -j$(nproc) clean
make -j$(nproc) all
$CXX -fsanitize=$SANITIZER $CXXFLAGS_ORIG -std=c++11 -I. \
$SRC/bad_example_fuzzer.cc -o $OUT/bad_example_partial_instrumentation \
-lFuzzingEngine ./libz.a
# Testcase 4. Completely ignore the flags provided by OSS-Fuzz.
################################################################################
./configure
make -j$(nproc) clean
make -j$(nproc) all
$CXX -fsanitize=$SANITIZER $CXXFLAGS -std=c++11 -I. \
$SRC/bad_example_fuzzer.cc -o $OUT/bad_example_no_instrumentation \
-lFuzzingEngine ./libz.a
# Testcase 4. Enable multiple sanitizers.
# Testcase 5. Enable multiple sanitizers.
################################################################################
# Add UBSan to ASan or MSan build. Add ASan to UBSan build.
EXTRA_SANITIZER="undefined"
@ -50,8 +63,8 @@ if [[ $SANITIZER = *undefined* ]]; then
EXTRA_SANITIZER="address"
fi
export CFLAGS="-O1 -fsanitize=$SANITIZER,$EXTRA_SANITIZER -fsanitize-coverage=trace-pc-guard,trace-cmp"
export CXXFLAGS="-O1 -fsanitize=$SANITIZER,$EXTRA_SANITIZER -fsanitize-coverage=trace-pc-guard,trace-cmp -stdlib=libc++"
export CFLAGS="$CFLAGS_ORIG -fsanitize=$EXTRA_SANITIZER"
export CXXFLAGS="$CXXFLAGS_ORIG -fsanitize=$EXTRA_SANITIZER"
./configure
make -j$(nproc) clean