diff --git a/infra/base-images/base-runner/bad_build_check b/infra/base-images/base-runner/bad_build_check index e133ea9fc..b2c25214f 100755 --- a/infra/base-images/base-runner/bad_build_check +++ b/infra/base-images/base-runner/bad_build_check @@ -1,4 +1,4 @@ -#!/bin/bash -ux +#!/bin/bash -u # Copyright 2017 Google Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -15,56 +15,65 @@ # ################################################################################ -# Verify that the given fuzzer has proper coverage instrumentation. +# A minimal number of runs to test fuzz target with a non-empty input. +MIN_NUMBER_OF_RUNS=4 + +# The "example" target has 73 with ASan, 65 with UBSan, and 6648 with MSan. +# Real world targets have greater values (arduinojson: 407, zlib: 664). +THRESHOLD_FOR_NUMBER_OF_EDGES=256 + +# Threshold values for different sanitizers used by instrumentation checks. +ASAN_CALLS_THRESHOLD_FOR_ASAN_BUILD=1000 +ASAN_CALLS_THRESHOLD_FOR_NON_ASAN_BUILD=0 + +MSAN_CALLS_THRESHOLD_FOR_MSAN_BUILD=1000 +MSAN_CALLS_THRESHOLD_FOR_NON_MSAN_BUILD=0 + +UBSAN_CALLS_THRESHOLD_FOR_UBSAN_BUILD=350 +UBSAN_CALLS_THRESHOLD_FOR_NON_UBSAN_BUILD=200 + + +# Verify that the given fuzz target has proper coverage instrumentation. function check_instrumentation { local FUZZER=$1 local CHECK_FAILED=0 + local FUZZER_OUTPUT="/tmp/$(basename $FUZZER).output" - if [[ "$FUZZING_ENGINE" = libfuzzer ]]; then - CHECK_FAILED=$($FUZZER -max_total_time=4 2>&1 | egrep "ERROR: no interesting inputs were found. Is the code instrumented" -c) - if (( $CHECK_FAILED > 0 )); then - echo "BAD BUILD: the code does not seem to have coverage instrumentation." - fi - fi - - # honggfuzz is not fully integrated, avoid performing the following check. - if [[ "$FUZZING_ENGINE" != honggfuzz ]]; then - if (( $CHECK_FAILED == 0 )); then - # The "example" target has 93 when built with ASan and 67 with UBSan. Real - # targets have greater values (arduinojson: 413, libteken: 519, zlib: 586). - local THRESHOLD_FOR_NUMBER_OF_EDGES=90 - if [[ $SANITIZER = undefined ]]; then - THRESHOLD_FOR_NUMBER_OF_EDGES=65 - fi - - local NUMBER_OF_EDGES=$(sancov -print-coverage-pcs $FUZZER | wc -l) - if (( $NUMBER_OF_EDGES < $THRESHOLD_FOR_NUMBER_OF_EDGES )); then - echo "BAD BUILD: the code does not seem to have coverage instrumentation." - CHECK_FAILED=1 - fi - fi + if [[ "$FUZZING_ENGINE" != libfuzzer ]]; then + return fi + # Store fuzz target's output into a temp file to be used for further checks. + $FUZZER -runs=$MIN_NUMBER_OF_RUNS &>$FUZZER_OUTPUT + CHECK_FAILED=$(egrep "ERROR: no interesting inputs were found. Is the code instrumented" -c $FUZZER_OUTPUT) if (( $CHECK_FAILED > 0 )); then - exit 1 + echo "BAD BUILD: the target does not seem to have coverage instrumentation." + + # Bail out as the further check does not make any sense, there are 0 PCs. + return 1 + fi + + local NUMBER_OF_EDGES=$(grep -Po "INFO: Loaded [[:digit:]]+ module.*\(.*counters\):[[:space:]]+\K[[:digit:]]+" $FUZZER_OUTPUT) + + if (( $NUMBER_OF_EDGES < $THRESHOLD_FOR_NUMBER_OF_EDGES )); then + echo "BAD BUILD: the target seems to have only partial coverage instrumentation." fi } -# Verify that the given fuzzer has been built properly and works as expected. +# Verify that the given fuzz target has been built properly and works. function check_startup_crash { local FUZZER=$1 local CHECK_PASSED=0 if [[ "$FUZZING_ENGINE" = libfuzzer ]]; then - CHECK_PASSED=$($FUZZER -runs=4 2>&1 | egrep "Done 4 runs" -c) + CHECK_PASSED=$($FUZZER -runs=$MIN_NUMBER_OF_RUNS 2>&1 | egrep "Done $MIN_NUMBER_OF_RUNS runs" -c) else # TODO: add checks for another fuzzing engines if possible. CHECK_PASSED=1 fi - if (( $CHECK_PASSED == 0 )); then + if [ "$CHECK_PASSED" -eq "0" ]; then echo "BAD BUILD: the fuzzer seems to have either startup crash or exit." - exit 1 fi } @@ -75,26 +84,17 @@ function check_asan_build { local MSAN_CALLS=$3 local UBSAN_CALLS=$4 - # Perform all the checks for more verbose error message. - local CHECK_FAILED=0 - - if (( $ASAN_CALLS < 1000 )); then + # Perform all the checks for more detailed error message. + if (( $ASAN_CALLS < $ASAN_CALLS_THRESHOLD_FOR_ASAN_BUILD )); then echo "BAD BUILD: $FUZZER does not seem to be compiled with ASan." - CHECK_FAILED=1 fi - if (( $MSAN_CALLS > 0 )); then + if (( $MSAN_CALLS > $MSAN_CALLS_THRESHOLD_FOR_NON_MSAN_BUILD )); then echo "BAD BUILD: ASan build of $FUZZER seems to be compiled with MSan." - CHECK_FAILED=1 fi - if (( $UBSAN_CALLS > 250 )); then + if (( $UBSAN_CALLS > $UBSAN_CALLS_THRESHOLD_FOR_NON_UBSAN_BUILD )); then echo "BAD BUILD: ASan build of $FUZZER seems to be compiled with UBSan." - CHECK_FAILED=1 - fi - - if (( $CHECK_FAILED > 0 )); then - exit 1 fi } @@ -105,26 +105,17 @@ function check_msan_build { local MSAN_CALLS=$3 local UBSAN_CALLS=$4 - # Perform all the checks for more verbose error message. - local CHECK_FAILED=0 - - if (( $ASAN_CALLS > 0 )); then + # Perform all the checks for more detailed error message. + if (( $ASAN_CALLS > $ASAN_CALLS_THRESHOLD_FOR_NON_ASAN_BUILD )); then echo "BAD BUILD: MSan build of $FUZZER seems to be compiled with ASan." - CHECK_FAILED=1 fi - if (( $MSAN_CALLS < 1000 )); then + if (( $MSAN_CALLS < $MSAN_CALLS_THRESHOLD_FOR_MSAN_BUILD )); then echo "BAD BUILD: $FUZZER does not seem to be compiled with UBSan." - CHECK_FAILED=1 fi - if (( $UBSAN_CALLS > 250 )); then + if (( $UBSAN_CALLS > $UBSAN_CALLS_THRESHOLD_FOR_NON_UBSAN_BUILD )); then echo "BAD BUILD: MSan build of $FUZZER seems to be compiled with UBSan." - CHECK_FAILED=1 - fi - - if (( $CHECK_FAILED > 0 )); then - exit 1 fi } @@ -142,38 +133,28 @@ function check_ubsan_build { return 0 fi - # Perform all the checks for more verbose error message. - local CHECK_FAILED=0 - - if (( $ASAN_CALLS > 0 )); then + # Perform all the checks for more detailed error message. + if (( $ASAN_CALLS > $ASAN_CALLS_THRESHOLD_FOR_NON_ASAN_BUILD )); then echo "BAD BUILD: UBSan build of $FUZZER seems to be compiled with ASan." - CHECK_FAILED=1 fi - if (( $MSAN_CALLS > 0 )); then + if (( $MSAN_CALLS > $MSAN_CALLS_THRESHOLD_FOR_NON_MSAN_BUILD )); then echo "BAD BUILD: UBSan build of $FUZZER seems to be compiled with MSan." - CHECK_FAILED=1 fi - if (( $UBSAN_CALLS < 250 )); then + if (( $UBSAN_CALLS < $UBSAN_CALLS_THRESHOLD_FOR_UBSAN_BUILD )); then echo "BAD BUILD: $FUZZER does not seem to be compiled with UBSan." - CHECK_FAILED=1 - fi - - if (( $CHECK_FAILED > 0 )); then - exit 1 fi } # Verify that the given fuzz target is compiled with correct sanitizer. function check_mixed_sanitizers { local FUZZER=$1 + local ASAN_CALLS=$(objdump -dC $FUZZER | egrep "callq\s+[0-9a-f]+\s+<__asan" -c) local MSAN_CALLS=$(objdump -dC $FUZZER | egrep "callq\s+[0-9a-f]+\s+<__msan" -c) local UBSAN_CALLS=$(objdump -dC $FUZZER | egrep "callq\s+[0-9a-f]+\s+<__ubsan" -c) - local CHECK_FAILED=0 - if [[ "$SANITIZER" = address ]]; then check_asan_build $FUZZER $ASAN_CALLS $MSAN_CALLS $UBSAN_CALLS elif [[ "$SANITIZER" = memory ]]; then @@ -186,7 +167,6 @@ function check_mixed_sanitizers { function main { local FUZZER=$1 - echo "INFO: performing bad build checks for $FUZZER" check_instrumentation $FUZZER check_mixed_sanitizers $FUZZER @@ -199,6 +179,7 @@ if [ $# -ne 1 ]; then exit 1 fi +# Fuzz target path. FUZZER=$1 + main $FUZZER -echo "INFO: $FUZZER has passed bad_build_check tests." diff --git a/infra/base-images/base-runner/test_all b/infra/base-images/base-runner/test_all index 06e781713..4b0ec79a9 100755 --- a/infra/base-images/base-runner/test_all +++ b/infra/base-images/base-runner/test_all @@ -1,4 +1,4 @@ -#!/bin/bash -eu +#!/bin/bash -u # Copyright 2016 Google Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -15,9 +15,24 @@ # ################################################################################ -# Test every fuzzer. Fails on first fuzzer failure. +# Percentage threshold that needs to be reached for marking a build as failed. +ALLOWED_BROKEN_TARGETS_PERCENTAGE=10 -N=0 +# Test all fuzz targets in the $OUT/ dir. +TOTAL_TARGETS_COUNT=0 + +# Number of CPUs available, this is needed for running tests in parallel. +NPROC=$(nproc) + +# Directories where bad build check results will be written to. +VALID_TARGETS_DIR="/tmp/valid_fuzz_targets" +BROKEN_TARGETS_DIR="/tmp/broken_fuzz_targets" +rm -rf $VALID_TARGETS_DIR +rm -rf $BROKEN_TARGETS_DIR +mkdir $VALID_TARGETS_DIR +mkdir $BROKEN_TARGETS_DIR + +# Main loop that iterates through all fuzz targets and runs the check. for FUZZER_BINARY in $(find $OUT/ -executable -type f); do if file "$FUZZER_BINARY" | grep -v ELF > /dev/null 2>&1; then continue @@ -31,8 +46,6 @@ for FUZZER_BINARY in $(find $OUT/ -executable -type f); do continue fi - echo "testing $FUZZER" - if [ "$SKIP_TEST_TARGET_RUN" != "1" ]; then if [[ "$FUZZING_ENGINE" = honggfuzz ]]; then timeout --preserve-status -s INT 20s run_fuzzer $FUZZER @@ -44,14 +57,80 @@ for FUZZER_BINARY in $(find $OUT/ -executable -type f); do fi fi - bad_build_check $FUZZER_BINARY - N=$[$N+1] + echo "INFO: performing bad build checks for $FUZZER_BINARY." + + LOG_PATH_FOR_BROKEN_TARGET="${BROKEN_TARGETS_DIR}/${FUZZER}" + + # Launch bad build check process in the background. + bad_build_check $FUZZER_BINARY > $LOG_PATH_FOR_BROKEN_TARGET & + + # Count total number of fuzz targets being tested. + TOTAL_TARGETS_COUNT=$[$TOTAL_TARGETS_COUNT+1] + + # Do not spawn more processes than the number of CPUs available. + n_child_proc=$(jobs -p | wc -l) + while [ "$n_child_proc" -eq "$NPROC" ]; do + sleep 4 + n_child_proc=$(jobs -p | wc -l) + done done -if [ "$N" -eq "0" ]; then +# Wait for background processes to finish. +wait + +# Sanity check in case there are no fuzz targets in the $OUT/ dir. +if [ "$TOTAL_TARGETS_COUNT" -eq "0" ]; then echo "ERROR: no fuzzers found in $OUT/" ls -al $OUT exit 1 fi -echo "$N fuzzers total" +# An empty log file indicated that corresponding fuzz target is not broken. +find $BROKEN_TARGETS_DIR -empty -exec mv {} $VALID_TARGETS_DIR \; + +# Calculate number of valid and broken fuzz targets. +VALID_TARGETS_COUNT=$(ls $VALID_TARGETS_DIR | wc -l) +BROKEN_TARGETS_COUNT=$(ls $BROKEN_TARGETS_DIR | wc -l) + +# Sanity check to make sure that bad build check doesn't skip any fuzz target. +if [ "$TOTAL_TARGETS_COUNT" -ne "$[$VALID_TARGETS_COUNT+$BROKEN_TARGETS_COUNT]" ]; then + echo "ERROR: bad_build_check seems to have a bug, total number of fuzz" \ + "does not match number of fuzz targets tested." + echo "Total fuzz targets ($TOTAL_TARGETS_COUNT):" + ls -al $OUT + echo "Valid fuzz targets ($VALID_TARGETS_COUNT):" + ls -al $VALID_TARGETS_DIR + echo "Total fuzz targets ($BROKEN_TARGETS_COUNT):" + ls -al $BROKEN_TARGETS_DIR + exit 1 +fi + +# Build info about all broken fuzz targets (if any). +if [ "$BROKEN_TARGETS_COUNT" -gt "0" ]; then + echo "Broken fuzz targets ($BROKEN_TARGETS_COUNT):" + for target in $(ls $BROKEN_TARGETS_DIR); do + echo "${target}:" + cat ${BROKEN_TARGETS_DIR}/${target} + done +fi + +# Calculate the percentage of broken fuzz targets and make the finel decision. +BROKEN_TARGETS_PERCENTAGE=$[$BROKEN_TARGETS_COUNT*100/$TOTAL_TARGETS_COUNT] + + +if [ "$BROKEN_TARGETS_PERCENTAGE" -gt "$ALLOWED_BROKEN_TARGETS_PERCENTAGE" ]; then + echo "ERROR: $BROKEN_TARGETS_PERCENTAGE% of fuzz targets seem to be broken." \ + "See the list above for a detailed information." + + # TODO: figure out how to not fail the "special" cases handled below. Those + # are from "example" and "c-ares" projects and are too small targets to pass. + if [ "$(ls $OUT/do_stuff_fuzzer $OUT/ares_*_fuzzer 2>/dev/null | wc -l)" -gt "0" ]; then + exit 0 + fi + + exit 1 +else + echo "$TOTAL_TARGETS_COUNT fuzzers total, $BROKEN_TARGETS_COUNT seem to be" \ + "broken ($BROKEN_TARGETS_PERCENTAGE%)." + exit 0 +fi diff --git a/infra/gcb/build.py b/infra/gcb/build.py index 48bb11799..5fec82f00 100755 --- a/infra/gcb/build.py +++ b/infra/gcb/build.py @@ -195,27 +195,43 @@ def get_build_steps(project_yaml, dockerfile_path): env.append('OUT=' + out) env.append('MSAN_LIBS_PATH=/workspace/msan') + # To disable running of all fuzz targets while doing |test_all| step, as + # that step is currently being used for performing bad build checks only. + env.append('SKIP_TEST_TARGET_RUN=1') + workdir = workdir_from_dockerfile(dockerfile_path) if not workdir: workdir = '/src' - build_steps.append({ + build_steps.extend([ # compile - 'name': image, - 'env': env, - 'args': [ - 'bash', - '-c', - # Remove /out to break loudly when a build script incorrectly uses - # /out instead of $OUT. - # `cd /src && cd {workdir}` (where {workdir} is parsed from the - # Dockerfile). Container Builder overrides our workdir so we need to add - # this step to set it back. - # We also remove /work and /src to save disk space after a step. - # Container Builder doesn't pass --rm to docker run yet. - 'rm -r /out && cd /src && cd {1} && mkdir -p {0} && compile && rm -rf /work && rm -rf /src'.format(out, workdir), - ], - }) + {'name': image, + 'env': env, + 'args': [ + 'bash', + '-c', + # Remove /out to break loudly when a build script incorrectly uses + # /out instead of $OUT. + # `cd /src && cd {workdir}` (where {workdir} is parsed from the + # Dockerfile). Container Builder overrides our workdir so we need to add + # this step to set it back. + # We also remove /work and /src to save disk space after a step. + # Container Builder doesn't pass --rm to docker run yet. + 'rm -r /out && cd /src && cd {1} && mkdir -p {0} && compile && rm -rf /work && rm -rf /src'.format(out, workdir), + ], + }, + # test binaries + {'name': 'gcr.io/oss-fuzz-base/base-runner', + 'env': env, + 'args': [ + 'bash', + '-c', + # Verify that fuzzers have been built properly and are not broken. + # TODO(mmoroz): raise a notification if not passing the tests. + 'test_all' + ], + }, + ]) if sanitizer == 'memory': # Patch dynamic libraries to use instrumented ones. diff --git a/projects/bad_example/build.sh b/projects/bad_example/build.sh index 473859162..dbfb8b434 100755 --- a/projects/bad_example/build.sh +++ b/projects/bad_example/build.sh @@ -28,21 +28,34 @@ if [[ $SANITIZER = *coverage* ]] || [[ $SANITIZER = *profile* ]]; then fi -# Testcase 3. Ignore the flags provided by OSS-Fuzz. +# Testcase 3. Partially ignore the flags provided by OSS-Fuzz. ################################################################################ +export CFLAGS_ORIG="$CFLAGS" export CFLAGS="-O1" +export CXXFLAGS_ORIG="$CXXFLAGS" export CXXFLAGS="-O1 -stdlib=libc++" ./configure make -j$(nproc) clean make -j$(nproc) all +$CXX -fsanitize=$SANITIZER $CXXFLAGS_ORIG -std=c++11 -I. \ + $SRC/bad_example_fuzzer.cc -o $OUT/bad_example_partial_instrumentation \ + -lFuzzingEngine ./libz.a + + +# Testcase 4. Completely ignore the flags provided by OSS-Fuzz. +################################################################################ +./configure +make -j$(nproc) clean +make -j$(nproc) all + $CXX -fsanitize=$SANITIZER $CXXFLAGS -std=c++11 -I. \ $SRC/bad_example_fuzzer.cc -o $OUT/bad_example_no_instrumentation \ -lFuzzingEngine ./libz.a -# Testcase 4. Enable multiple sanitizers. +# Testcase 5. Enable multiple sanitizers. ################################################################################ # Add UBSan to ASan or MSan build. Add ASan to UBSan build. EXTRA_SANITIZER="undefined" @@ -50,8 +63,8 @@ if [[ $SANITIZER = *undefined* ]]; then EXTRA_SANITIZER="address" fi -export CFLAGS="-O1 -fsanitize=$SANITIZER,$EXTRA_SANITIZER -fsanitize-coverage=trace-pc-guard,trace-cmp" -export CXXFLAGS="-O1 -fsanitize=$SANITIZER,$EXTRA_SANITIZER -fsanitize-coverage=trace-pc-guard,trace-cmp -stdlib=libc++" +export CFLAGS="$CFLAGS_ORIG -fsanitize=$EXTRA_SANITIZER" +export CXXFLAGS="$CXXFLAGS_ORIG -fsanitize=$EXTRA_SANITIZER" ./configure make -j$(nproc) clean