mirror of https://github.com/google/oss-fuzz.git
[infra] Enable bad build checks once again. (#838)
* [infra] Enable bad build checks once again. * Minor typo. * [bad_example] Update build flags for reproducing bad instrumentation scenario. * [bad_example] split bad/no instrumentation case into two different ones. * Use new approach for partial instrumentation detection + do that only for libFuzzer. * Rename bad_example_bad_instrumentation into bad_example_partial_instrumentation. * Calculate number of broken targets and fail if 10+% are broken. * Multiprocess madness. * Always run all checks and store all errors + clean up the code and add comments * Add special handling for the projects with very small fuzz targets. * Remove unnecessary semicolon. * Address review comments. * Address more review comments, small refactoring.
This commit is contained in:
Max Moroz
GitHub
parent
ad80480393
commit
4df2262466
|
|
@ -1,4 +1,4 @@
|
|||
#!/bin/bash -ux
|
||||
#!/bin/bash -u
|
||||
# Copyright 2017 Google Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
|
|
@ -15,56 +15,65 @@
|
|||
#
|
||||
################################################################################
|
||||
|
||||
# Verify that the given fuzzer has proper coverage instrumentation.
|
||||
# A minimal number of runs to test fuzz target with a non-empty input.
|
||||
MIN_NUMBER_OF_RUNS=4
|
||||
|
||||
# The "example" target has 73 with ASan, 65 with UBSan, and 6648 with MSan.
|
||||
# Real world targets have greater values (arduinojson: 407, zlib: 664).
|
||||
THRESHOLD_FOR_NUMBER_OF_EDGES=256
|
||||
|
||||
# Threshold values for different sanitizers used by instrumentation checks.
|
||||
ASAN_CALLS_THRESHOLD_FOR_ASAN_BUILD=1000
|
||||
ASAN_CALLS_THRESHOLD_FOR_NON_ASAN_BUILD=0
|
||||
|
||||
MSAN_CALLS_THRESHOLD_FOR_MSAN_BUILD=1000
|
||||
MSAN_CALLS_THRESHOLD_FOR_NON_MSAN_BUILD=0
|
||||
|
||||
UBSAN_CALLS_THRESHOLD_FOR_UBSAN_BUILD=350
|
||||
UBSAN_CALLS_THRESHOLD_FOR_NON_UBSAN_BUILD=200
|
||||
|
||||
|
||||
# Verify that the given fuzz target has proper coverage instrumentation.
|
||||
function check_instrumentation {
|
||||
local FUZZER=$1
|
||||
local CHECK_FAILED=0
|
||||
local FUZZER_OUTPUT="/tmp/$(basename $FUZZER).output"
|
||||
|
||||
if [[ "$FUZZING_ENGINE" = libfuzzer ]]; then
|
||||
CHECK_FAILED=$($FUZZER -max_total_time=4 2>&1 | egrep "ERROR: no interesting inputs were found. Is the code instrumented" -c)
|
||||
if [[ "$FUZZING_ENGINE" != libfuzzer ]]; then
|
||||
return
|
||||
fi
|
||||
|
||||
# Store fuzz target's output into a temp file to be used for further checks.
|
||||
$FUZZER -runs=$MIN_NUMBER_OF_RUNS &>$FUZZER_OUTPUT
|
||||
CHECK_FAILED=$(egrep "ERROR: no interesting inputs were found. Is the code instrumented" -c $FUZZER_OUTPUT)
|
||||
if (( $CHECK_FAILED > 0 )); then
|
||||
echo "BAD BUILD: the code does not seem to have coverage instrumentation."
|
||||
fi
|
||||
echo "BAD BUILD: the target does not seem to have coverage instrumentation."
|
||||
|
||||
# Bail out as the further check does not make any sense, there are 0 PCs.
|
||||
return 1
|
||||
fi
|
||||
|
||||
# honggfuzz is not fully integrated, avoid performing the following check.
|
||||
if [[ "$FUZZING_ENGINE" != honggfuzz ]]; then
|
||||
if (( $CHECK_FAILED == 0 )); then
|
||||
# The "example" target has 93 when built with ASan and 67 with UBSan. Real
|
||||
# targets have greater values (arduinojson: 413, libteken: 519, zlib: 586).
|
||||
local THRESHOLD_FOR_NUMBER_OF_EDGES=90
|
||||
if [[ $SANITIZER = undefined ]]; then
|
||||
THRESHOLD_FOR_NUMBER_OF_EDGES=65
|
||||
fi
|
||||
local NUMBER_OF_EDGES=$(grep -Po "INFO: Loaded [[:digit:]]+ module.*\(.*counters\):[[:space:]]+\K[[:digit:]]+" $FUZZER_OUTPUT)
|
||||
|
||||
local NUMBER_OF_EDGES=$(sancov -print-coverage-pcs $FUZZER | wc -l)
|
||||
if (( $NUMBER_OF_EDGES < $THRESHOLD_FOR_NUMBER_OF_EDGES )); then
|
||||
echo "BAD BUILD: the code does not seem to have coverage instrumentation."
|
||||
CHECK_FAILED=1
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
if (( $CHECK_FAILED > 0 )); then
|
||||
exit 1
|
||||
echo "BAD BUILD: the target seems to have only partial coverage instrumentation."
|
||||
fi
|
||||
}
|
||||
|
||||
# Verify that the given fuzzer has been built properly and works as expected.
|
||||
# Verify that the given fuzz target has been built properly and works.
|
||||
function check_startup_crash {
|
||||
local FUZZER=$1
|
||||
local CHECK_PASSED=0
|
||||
|
||||
if [[ "$FUZZING_ENGINE" = libfuzzer ]]; then
|
||||
CHECK_PASSED=$($FUZZER -runs=4 2>&1 | egrep "Done 4 runs" -c)
|
||||
CHECK_PASSED=$($FUZZER -runs=$MIN_NUMBER_OF_RUNS 2>&1 | egrep "Done $MIN_NUMBER_OF_RUNS runs" -c)
|
||||
else
|
||||
# TODO: add checks for another fuzzing engines if possible.
|
||||
CHECK_PASSED=1
|
||||
fi
|
||||
|
||||
if (( $CHECK_PASSED == 0 )); then
|
||||
if [ "$CHECK_PASSED" -eq "0" ]; then
|
||||
echo "BAD BUILD: the fuzzer seems to have either startup crash or exit."
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
|
|
@ -75,26 +84,17 @@ function check_asan_build {
|
|||
local MSAN_CALLS=$3
|
||||
local UBSAN_CALLS=$4
|
||||
|
||||
# Perform all the checks for more verbose error message.
|
||||
local CHECK_FAILED=0
|
||||
|
||||
if (( $ASAN_CALLS < 1000 )); then
|
||||
# Perform all the checks for more detailed error message.
|
||||
if (( $ASAN_CALLS < $ASAN_CALLS_THRESHOLD_FOR_ASAN_BUILD )); then
|
||||
echo "BAD BUILD: $FUZZER does not seem to be compiled with ASan."
|
||||
CHECK_FAILED=1
|
||||
fi
|
||||
|
||||
if (( $MSAN_CALLS > 0 )); then
|
||||
if (( $MSAN_CALLS > $MSAN_CALLS_THRESHOLD_FOR_NON_MSAN_BUILD )); then
|
||||
echo "BAD BUILD: ASan build of $FUZZER seems to be compiled with MSan."
|
||||
CHECK_FAILED=1
|
||||
fi
|
||||
|
||||
if (( $UBSAN_CALLS > 250 )); then
|
||||
if (( $UBSAN_CALLS > $UBSAN_CALLS_THRESHOLD_FOR_NON_UBSAN_BUILD )); then
|
||||
echo "BAD BUILD: ASan build of $FUZZER seems to be compiled with UBSan."
|
||||
CHECK_FAILED=1
|
||||
fi
|
||||
|
||||
if (( $CHECK_FAILED > 0 )); then
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
|
|
@ -105,26 +105,17 @@ function check_msan_build {
|
|||
local MSAN_CALLS=$3
|
||||
local UBSAN_CALLS=$4
|
||||
|
||||
# Perform all the checks for more verbose error message.
|
||||
local CHECK_FAILED=0
|
||||
|
||||
if (( $ASAN_CALLS > 0 )); then
|
||||
# Perform all the checks for more detailed error message.
|
||||
if (( $ASAN_CALLS > $ASAN_CALLS_THRESHOLD_FOR_NON_ASAN_BUILD )); then
|
||||
echo "BAD BUILD: MSan build of $FUZZER seems to be compiled with ASan."
|
||||
CHECK_FAILED=1
|
||||
fi
|
||||
|
||||
if (( $MSAN_CALLS < 1000 )); then
|
||||
if (( $MSAN_CALLS < $MSAN_CALLS_THRESHOLD_FOR_MSAN_BUILD )); then
|
||||
echo "BAD BUILD: $FUZZER does not seem to be compiled with UBSan."
|
||||
CHECK_FAILED=1
|
||||
fi
|
||||
|
||||
if (( $UBSAN_CALLS > 250 )); then
|
||||
if (( $UBSAN_CALLS > $UBSAN_CALLS_THRESHOLD_FOR_NON_UBSAN_BUILD )); then
|
||||
echo "BAD BUILD: MSan build of $FUZZER seems to be compiled with UBSan."
|
||||
CHECK_FAILED=1
|
||||
fi
|
||||
|
||||
if (( $CHECK_FAILED > 0 )); then
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
|
|
@ -142,38 +133,28 @@ function check_ubsan_build {
|
|||
return 0
|
||||
fi
|
||||
|
||||
# Perform all the checks for more verbose error message.
|
||||
local CHECK_FAILED=0
|
||||
|
||||
if (( $ASAN_CALLS > 0 )); then
|
||||
# Perform all the checks for more detailed error message.
|
||||
if (( $ASAN_CALLS > $ASAN_CALLS_THRESHOLD_FOR_NON_ASAN_BUILD )); then
|
||||
echo "BAD BUILD: UBSan build of $FUZZER seems to be compiled with ASan."
|
||||
CHECK_FAILED=1
|
||||
fi
|
||||
|
||||
if (( $MSAN_CALLS > 0 )); then
|
||||
if (( $MSAN_CALLS > $MSAN_CALLS_THRESHOLD_FOR_NON_MSAN_BUILD )); then
|
||||
echo "BAD BUILD: UBSan build of $FUZZER seems to be compiled with MSan."
|
||||
CHECK_FAILED=1
|
||||
fi
|
||||
|
||||
if (( $UBSAN_CALLS < 250 )); then
|
||||
if (( $UBSAN_CALLS < $UBSAN_CALLS_THRESHOLD_FOR_UBSAN_BUILD )); then
|
||||
echo "BAD BUILD: $FUZZER does not seem to be compiled with UBSan."
|
||||
CHECK_FAILED=1
|
||||
fi
|
||||
|
||||
if (( $CHECK_FAILED > 0 )); then
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
# Verify that the given fuzz target is compiled with correct sanitizer.
|
||||
function check_mixed_sanitizers {
|
||||
local FUZZER=$1
|
||||
|
||||
local ASAN_CALLS=$(objdump -dC $FUZZER | egrep "callq\s+[0-9a-f]+\s+<__asan" -c)
|
||||
local MSAN_CALLS=$(objdump -dC $FUZZER | egrep "callq\s+[0-9a-f]+\s+<__msan" -c)
|
||||
local UBSAN_CALLS=$(objdump -dC $FUZZER | egrep "callq\s+[0-9a-f]+\s+<__ubsan" -c)
|
||||
|
||||
local CHECK_FAILED=0
|
||||
|
||||
if [[ "$SANITIZER" = address ]]; then
|
||||
check_asan_build $FUZZER $ASAN_CALLS $MSAN_CALLS $UBSAN_CALLS
|
||||
elif [[ "$SANITIZER" = memory ]]; then
|
||||
|
|
@ -186,7 +167,6 @@ function check_mixed_sanitizers {
|
|||
|
||||
function main {
|
||||
local FUZZER=$1
|
||||
echo "INFO: performing bad build checks for $FUZZER"
|
||||
|
||||
check_instrumentation $FUZZER
|
||||
check_mixed_sanitizers $FUZZER
|
||||
|
|
@ -199,6 +179,7 @@ if [ $# -ne 1 ]; then
|
|||
exit 1
|
||||
fi
|
||||
|
||||
# Fuzz target path.
|
||||
FUZZER=$1
|
||||
|
||||
main $FUZZER
|
||||
echo "INFO: $FUZZER has passed bad_build_check tests."
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
#!/bin/bash -eu
|
||||
#!/bin/bash -u
|
||||
# Copyright 2016 Google Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
|
|
@ -15,9 +15,24 @@
|
|||
#
|
||||
################################################################################
|
||||
|
||||
# Test every fuzzer. Fails on first fuzzer failure.
|
||||
# Percentage threshold that needs to be reached for marking a build as failed.
|
||||
ALLOWED_BROKEN_TARGETS_PERCENTAGE=10
|
||||
|
||||
N=0
|
||||
# Test all fuzz targets in the $OUT/ dir.
|
||||
TOTAL_TARGETS_COUNT=0
|
||||
|
||||
# Number of CPUs available, this is needed for running tests in parallel.
|
||||
NPROC=$(nproc)
|
||||
|
||||
# Directories where bad build check results will be written to.
|
||||
VALID_TARGETS_DIR="/tmp/valid_fuzz_targets"
|
||||
BROKEN_TARGETS_DIR="/tmp/broken_fuzz_targets"
|
||||
rm -rf $VALID_TARGETS_DIR
|
||||
rm -rf $BROKEN_TARGETS_DIR
|
||||
mkdir $VALID_TARGETS_DIR
|
||||
mkdir $BROKEN_TARGETS_DIR
|
||||
|
||||
# Main loop that iterates through all fuzz targets and runs the check.
|
||||
for FUZZER_BINARY in $(find $OUT/ -executable -type f); do
|
||||
if file "$FUZZER_BINARY" | grep -v ELF > /dev/null 2>&1; then
|
||||
continue
|
||||
|
|
@ -31,8 +46,6 @@ for FUZZER_BINARY in $(find $OUT/ -executable -type f); do
|
|||
continue
|
||||
fi
|
||||
|
||||
echo "testing $FUZZER"
|
||||
|
||||
if [ "$SKIP_TEST_TARGET_RUN" != "1" ]; then
|
||||
if [[ "$FUZZING_ENGINE" = honggfuzz ]]; then
|
||||
timeout --preserve-status -s INT 20s run_fuzzer $FUZZER
|
||||
|
|
@ -44,14 +57,80 @@ for FUZZER_BINARY in $(find $OUT/ -executable -type f); do
|
|||
fi
|
||||
fi
|
||||
|
||||
bad_build_check $FUZZER_BINARY
|
||||
N=$[$N+1]
|
||||
echo "INFO: performing bad build checks for $FUZZER_BINARY."
|
||||
|
||||
LOG_PATH_FOR_BROKEN_TARGET="${BROKEN_TARGETS_DIR}/${FUZZER}"
|
||||
|
||||
# Launch bad build check process in the background.
|
||||
bad_build_check $FUZZER_BINARY > $LOG_PATH_FOR_BROKEN_TARGET &
|
||||
|
||||
# Count total number of fuzz targets being tested.
|
||||
TOTAL_TARGETS_COUNT=$[$TOTAL_TARGETS_COUNT+1]
|
||||
|
||||
# Do not spawn more processes than the number of CPUs available.
|
||||
n_child_proc=$(jobs -p | wc -l)
|
||||
while [ "$n_child_proc" -eq "$NPROC" ]; do
|
||||
sleep 4
|
||||
n_child_proc=$(jobs -p | wc -l)
|
||||
done
|
||||
done
|
||||
|
||||
if [ "$N" -eq "0" ]; then
|
||||
# Wait for background processes to finish.
|
||||
wait
|
||||
|
||||
# Sanity check in case there are no fuzz targets in the $OUT/ dir.
|
||||
if [ "$TOTAL_TARGETS_COUNT" -eq "0" ]; then
|
||||
echo "ERROR: no fuzzers found in $OUT/"
|
||||
ls -al $OUT
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "$N fuzzers total"
|
||||
# An empty log file indicated that corresponding fuzz target is not broken.
|
||||
find $BROKEN_TARGETS_DIR -empty -exec mv {} $VALID_TARGETS_DIR \;
|
||||
|
||||
# Calculate number of valid and broken fuzz targets.
|
||||
VALID_TARGETS_COUNT=$(ls $VALID_TARGETS_DIR | wc -l)
|
||||
BROKEN_TARGETS_COUNT=$(ls $BROKEN_TARGETS_DIR | wc -l)
|
||||
|
||||
# Sanity check to make sure that bad build check doesn't skip any fuzz target.
|
||||
if [ "$TOTAL_TARGETS_COUNT" -ne "$[$VALID_TARGETS_COUNT+$BROKEN_TARGETS_COUNT]" ]; then
|
||||
echo "ERROR: bad_build_check seems to have a bug, total number of fuzz" \
|
||||
"does not match number of fuzz targets tested."
|
||||
echo "Total fuzz targets ($TOTAL_TARGETS_COUNT):"
|
||||
ls -al $OUT
|
||||
echo "Valid fuzz targets ($VALID_TARGETS_COUNT):"
|
||||
ls -al $VALID_TARGETS_DIR
|
||||
echo "Total fuzz targets ($BROKEN_TARGETS_COUNT):"
|
||||
ls -al $BROKEN_TARGETS_DIR
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Build info about all broken fuzz targets (if any).
|
||||
if [ "$BROKEN_TARGETS_COUNT" -gt "0" ]; then
|
||||
echo "Broken fuzz targets ($BROKEN_TARGETS_COUNT):"
|
||||
for target in $(ls $BROKEN_TARGETS_DIR); do
|
||||
echo "${target}:"
|
||||
cat ${BROKEN_TARGETS_DIR}/${target}
|
||||
done
|
||||
fi
|
||||
|
||||
# Calculate the percentage of broken fuzz targets and make the finel decision.
|
||||
BROKEN_TARGETS_PERCENTAGE=$[$BROKEN_TARGETS_COUNT*100/$TOTAL_TARGETS_COUNT]
|
||||
|
||||
|
||||
if [ "$BROKEN_TARGETS_PERCENTAGE" -gt "$ALLOWED_BROKEN_TARGETS_PERCENTAGE" ]; then
|
||||
echo "ERROR: $BROKEN_TARGETS_PERCENTAGE% of fuzz targets seem to be broken." \
|
||||
"See the list above for a detailed information."
|
||||
|
||||
# TODO: figure out how to not fail the "special" cases handled below. Those
|
||||
# are from "example" and "c-ares" projects and are too small targets to pass.
|
||||
if [ "$(ls $OUT/do_stuff_fuzzer $OUT/ares_*_fuzzer 2>/dev/null | wc -l)" -gt "0" ]; then
|
||||
exit 0
|
||||
fi
|
||||
|
||||
exit 1
|
||||
else
|
||||
echo "$TOTAL_TARGETS_COUNT fuzzers total, $BROKEN_TARGETS_COUNT seem to be" \
|
||||
"broken ($BROKEN_TARGETS_PERCENTAGE%)."
|
||||
exit 0
|
||||
fi
|
||||
|
|
|
|||
|
|
@ -195,13 +195,17 @@ def get_build_steps(project_yaml, dockerfile_path):
|
|||
env.append('OUT=' + out)
|
||||
env.append('MSAN_LIBS_PATH=/workspace/msan')
|
||||
|
||||
# To disable running of all fuzz targets while doing |test_all| step, as
|
||||
# that step is currently being used for performing bad build checks only.
|
||||
env.append('SKIP_TEST_TARGET_RUN=1')
|
||||
|
||||
workdir = workdir_from_dockerfile(dockerfile_path)
|
||||
if not workdir:
|
||||
workdir = '/src'
|
||||
|
||||
build_steps.append({
|
||||
build_steps.extend([
|
||||
# compile
|
||||
'name': image,
|
||||
{'name': image,
|
||||
'env': env,
|
||||
'args': [
|
||||
'bash',
|
||||
|
|
@ -215,7 +219,19 @@ def get_build_steps(project_yaml, dockerfile_path):
|
|||
# Container Builder doesn't pass --rm to docker run yet.
|
||||
'rm -r /out && cd /src && cd {1} && mkdir -p {0} && compile && rm -rf /work && rm -rf /src'.format(out, workdir),
|
||||
],
|
||||
})
|
||||
},
|
||||
# test binaries
|
||||
{'name': 'gcr.io/oss-fuzz-base/base-runner',
|
||||
'env': env,
|
||||
'args': [
|
||||
'bash',
|
||||
'-c',
|
||||
# Verify that fuzzers have been built properly and are not broken.
|
||||
# TODO(mmoroz): raise a notification if not passing the tests.
|
||||
'test_all'
|
||||
],
|
||||
},
|
||||
])
|
||||
|
||||
if sanitizer == 'memory':
|
||||
# Patch dynamic libraries to use instrumented ones.
|
||||
|
|
|
|||
|
|
@ -28,21 +28,34 @@ if [[ $SANITIZER = *coverage* ]] || [[ $SANITIZER = *profile* ]]; then
|
|||
fi
|
||||
|
||||
|
||||
# Testcase 3. Ignore the flags provided by OSS-Fuzz.
|
||||
# Testcase 3. Partially ignore the flags provided by OSS-Fuzz.
|
||||
################################################################################
|
||||
export CFLAGS_ORIG="$CFLAGS"
|
||||
export CFLAGS="-O1"
|
||||
export CXXFLAGS_ORIG="$CXXFLAGS"
|
||||
export CXXFLAGS="-O1 -stdlib=libc++"
|
||||
|
||||
./configure
|
||||
make -j$(nproc) clean
|
||||
make -j$(nproc) all
|
||||
|
||||
$CXX -fsanitize=$SANITIZER $CXXFLAGS_ORIG -std=c++11 -I. \
|
||||
$SRC/bad_example_fuzzer.cc -o $OUT/bad_example_partial_instrumentation \
|
||||
-lFuzzingEngine ./libz.a
|
||||
|
||||
|
||||
# Testcase 4. Completely ignore the flags provided by OSS-Fuzz.
|
||||
################################################################################
|
||||
./configure
|
||||
make -j$(nproc) clean
|
||||
make -j$(nproc) all
|
||||
|
||||
$CXX -fsanitize=$SANITIZER $CXXFLAGS -std=c++11 -I. \
|
||||
$SRC/bad_example_fuzzer.cc -o $OUT/bad_example_no_instrumentation \
|
||||
-lFuzzingEngine ./libz.a
|
||||
|
||||
|
||||
# Testcase 4. Enable multiple sanitizers.
|
||||
# Testcase 5. Enable multiple sanitizers.
|
||||
################################################################################
|
||||
# Add UBSan to ASan or MSan build. Add ASan to UBSan build.
|
||||
EXTRA_SANITIZER="undefined"
|
||||
|
|
@ -50,8 +63,8 @@ if [[ $SANITIZER = *undefined* ]]; then
|
|||
EXTRA_SANITIZER="address"
|
||||
fi
|
||||
|
||||
export CFLAGS="-O1 -fsanitize=$SANITIZER,$EXTRA_SANITIZER -fsanitize-coverage=trace-pc-guard,trace-cmp"
|
||||
export CXXFLAGS="-O1 -fsanitize=$SANITIZER,$EXTRA_SANITIZER -fsanitize-coverage=trace-pc-guard,trace-cmp -stdlib=libc++"
|
||||
export CFLAGS="$CFLAGS_ORIG -fsanitize=$EXTRA_SANITIZER"
|
||||
export CXXFLAGS="$CXXFLAGS_ORIG -fsanitize=$EXTRA_SANITIZER"
|
||||
|
||||
./configure
|
||||
make -j$(nproc) clean
|
||||
|
|
|
|||
Loading…
Reference in New Issue