[infra] Add post build checks for catching and reporting bad builds + add test projects. (#754)

* [WIP] Add post build checks for catching and reporting bad builds + test projects.

* Move all type of issues into a single project. Tested all sanitizers.

* First version of the script for performing bad build checks. Still not ready to commit.\

* Added a valid example that should pass all the checks + fixed instrumentation check.

* Use SIGTERM and the handler calling _exit(0) for the startup crash case.

* Small fixes.

* Ignore startup_crash check_for fuzzing engines other than libFuzzer.

* Use "local" for local variables in the script, fix container name.

* Add COPY step for bad_build_check into base-runner Dockerfile.

* Make bad_build_check temporary noop unless EXPERIMENTAL_BAD_BUILD_CHECK is set to 1.

* Temporary remove test binaries step.

* Remove empty line.

* Remove unnecessary env variable.

infra/base-images/base-runner/Dockerfile

@@ -17,7 +17,7 @@
 FROM gcr.io/oss-fuzz-base/base-image
 MAINTAINER mike.aizatsky@gmail.com
 RUN apt-get install -y zip file libunwind8 binutils libblocksruntime0
-COPY llvm-symbolizer reproduce run_fuzzer test_all test_report \
+COPY bad_build_check llvm-symbolizer reproduce run_fuzzer test_all test_report \
     /usr/local/bin/
 
 # Default environment options for various sanitizers.

infra/base-images/base-runner/bad_build_check

@@ -0,0 +1,188 @@
+#!/bin/bash -u
+# Copyright 2017 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+################################################################################
+
+# Verify that the given fuzzer has proper coverage instrumentation.
+function check_instrumentation {
+  local FUZZER=$1
+  local CHECK_FAILED=0
+
+  if [[ "$FUZZING_ENGINE" = libfuzzer ]]; then
+    CHECK_FAILED=$($FUZZER -max_total_time=4 2>&1 | egrep "ERROR: no interesting inputs were found. Is the code instrumented" -c)
+    if (( $CHECK_FAILED > 0 )); then
+      echo "BAD BUILD: the code does not seem to have coverage instrumentation."
+    fi
+  fi
+
+  if (( $CHECK_FAILED == 0 )); then
+    # Should be 90 at least. The "example" target has 93. Real targets have the
+    # following values: arduinojson: 413, libteken: 519, zlib: 586.
+    local NUMBER_OF_EDGES=$(sancov -print-coverage-pcs $FUZZER | wc -l)
+    if (( $NUMBER_OF_EDGES < 90 )); then
+      echo "BAD BUILD: the code does not seem to have coverage instrumentation."
+      CHECK_FAILED=1
+    fi
+  fi
+
+  if (( $CHECK_FAILED > 0 )); then
+    exit 1
+  fi
+}
+
+# Verify that the given fuzzer has been built properly and works as expected.
+function check_startup_crash {
+  local FUZZER=$1
+  local CHECK_PASSED=0
+
+  if [[ "$FUZZING_ENGINE" = libfuzzer ]]; then
+    CHECK_PASSED=$($FUZZER -runs=4 2>&1 | egrep "Done 4 runs" -c)
+  else
+    # TODO: add checks for another fuzzing engines if possible.
+    CHECK_PASSED=1
+  fi
+
+  if (( $CHECK_PASSED == 0 )); then
+    echo "BAD BUILD: the fuzzer seems to have either startup crash or exit."
+    exit 1
+  fi
+}
+
+# Mixed sanitizers check for ASan build.
+function check_asan_build {
+  local FUZZER=$1
+  local ASAN_CALLS=$2
+  local MSAN_CALLS=$3
+  local UBSAN_CALLS=$4
+
+  # Perform all the checks for more verbose error message.
+  local CHECK_FAILED=0
+
+  if (( $ASAN_CALLS < 1000 )); then
+    echo "BAD BUILD: $FUZZER does not seem to be compiled with ASan."
+    CHECK_FAILED=1
+  fi
+
+  if (( $MSAN_CALLS > 0 )); then
+    echo "BAD BUILD: ASan build of $FUZZER seems to be compiled with MSan."
+    CHECK_FAILED=1
+  fi
+
+  if (( $UBSAN_CALLS > 250 )); then
+    echo "BAD BUILD: ASan build of $FUZZER seems to be compiled with UBSan."
+    CHECK_FAILED=1
+  fi
+
+  if (( $CHECK_FAILED > 0 )); then
+    exit 1
+  fi
+}
+
+# Mixed sanitizers check for MSan build.
+function check_msan_build {
+  local FUZZER=$1
+  local ASAN_CALLS=$2
+  local MSAN_CALLS=$3
+  local UBSAN_CALLS=$4
+
+  # Perform all the checks for more verbose error message.
+  local CHECK_FAILED=0
+
+  if (( $ASAN_CALLS > 0 )); then
+    echo "BAD BUILD: MSan build of $FUZZER seems to be compiled with ASan."
+    CHECK_FAILED=1
+  fi
+
+  if (( $MSAN_CALLS < 1000 )); then
+    echo "BAD BUILD: $FUZZER does not seem to be compiled with UBSan."
+    CHECK_FAILED=1
+  fi
+
+  if (( $UBSAN_CALLS > 250 )); then
+    echo "BAD BUILD: MSan build of $FUZZER seems to be compiled with UBSan."
+    CHECK_FAILED=1
+  fi
+
+  if (( $CHECK_FAILED > 0 )); then
+    exit 1
+  fi
+}
+
+# Mixed sanitizers check for UBSan build.
+function check_ubsan_build {
+  local FUZZER=$1
+  local ASAN_CALLS=$2
+  local MSAN_CALLS=$3
+  local UBSAN_CALLS=$4
+
+  # Perform all the checks for more verbose error message.
+  local CHECK_FAILED=0
+
+  if (( $ASAN_CALLS > 0 )); then
+    echo "BAD BUILD: UBSan build of $FUZZER seems to be compiled with ASan."
+    CHECK_FAILED=1
+  fi
+
+  if (( $MSAN_CALLS > 0 )); then
+    echo "BAD BUILD: UBSan build of $FUZZER seems to be compiled with MSan."
+    CHECK_FAILED=1
+  fi
+
+  if (( $UBSAN_CALLS < 250 )); then
+    echo "BAD BUILD: $FUZZER does not seem to be compiled with UBSan."
+    CHECK_FAILED=1
+  fi
+
+  if (( $CHECK_FAILED > 0 )); then
+    exit 1
+  fi
+}
+
+# Verify that the given fuzz target is compiled with correct sanitizer.
+function check_mixed_sanitizers {
+  local FUZZER=$1
+  local ASAN_CALLS=$(objdump -dC $FUZZER | egrep "callq\s+[0-9a-f]+\s+<__asan" -c)
+  local MSAN_CALLS=$(objdump -dC $FUZZER | egrep "callq\s+[0-9a-f]+\s+<__msan" -c)
+  local UBSAN_CALLS=$(objdump -dC $FUZZER | egrep "callq\s+[0-9a-f]+\s+<__ubsan" -c)
+
+  local CHECK_FAILED=0
+
+  if [[ "$SANITIZER" = address ]]; then
+    check_asan_build $FUZZER $ASAN_CALLS $MSAN_CALLS $UBSAN_CALLS
+  elif [[ "$SANITIZER" = memory ]]; then
+    check_msan_build $FUZZER $ASAN_CALLS $MSAN_CALLS $UBSAN_CALLS
+  elif [[ "$SANITIZER" = undefined ]]; then
+    check_ubsan_build $FUZZER $ASAN_CALLS $MSAN_CALLS $UBSAN_CALLS
+  fi
+}
+
+
+function main {
+  local FUZZER=$1
+
+  check_instrumentation $FUZZER
+  check_mixed_sanitizers $FUZZER
+  check_startup_crash $FUZZER
+}
+
+
+if [ $# -ne 1 ]; then
+    echo "Usage: $0 <fuzz_target_binary>"
+    exit 1
+fi
+
+FUZZER=$1
+main $FUZZER
+echo "INFO: $FUZZER has passed bad_build_check tests."

infra/base-images/base-runner/test_all

@@ -41,6 +41,7 @@ for FUZZER_BINARY in $(find $OUT/ -executable -type f); do
     export AFL_NO_UI=1
     timeout --preserve-status -s INT 20s run_fuzzer $FUZZER
   fi
+  bad_build_check $FUZZER
   N=$[$N+1]
 done
 

infra/gcb/build.py

@@ -187,7 +187,7 @@ def get_build_steps(project_yaml, dockerfile_path):
       build_steps.extend([
           # compile
           {'name': image,
-            'env' : env,
+            'env': env,
             'args': [
               'bash',
               '-c',
@@ -199,8 +199,8 @@ def get_build_steps(project_yaml, dockerfile_path):
               # We also remove /work and /src to save disk space after a step.
               # Container Builder doesn't pass --rm to docker run yet.
               'rm -r /out && cd /src && cd {1} && mkdir -p {0} && compile && rm -rf /work && rm -rf /src'.format(out, workdir),
-              ],
-            },
+            ],
+          },
           # zip binaries
           {'name': image,
             'args': [
@@ -214,14 +214,14 @@ def get_build_steps(project_yaml, dockerfile_path):
            'args': [
                os.path.join(out, zip_file),
                upload_url,
-           ],
+            ],
           },
           # upload srcmap
           {'name': 'gcr.io/oss-fuzz-base/uploader',
            'args': [
                '/workspace/srcmap.json',
                srcmap_url,
-           ],
+            ],
           },
           # cleanup
           {'name': image,

projects/bad_example/Dockerfile

@@ -0,0 +1,25 @@
+# Copyright 2017 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+################################################################################
+
+FROM gcr.io/oss-fuzz-base/base-builder
+MAINTAINER mmoroz@chromium.org
+RUN apt-get update && apt-get install -y make autoconf automake libtool
+
+# Using a real zlib project, but with broken build script and/or fuzz target.
+
+RUN git clone --depth 1 https://github.com/madler/zlib.git
+WORKDIR zlib
+COPY build.sh bad_example_fuzzer.cc $SRC/

projects/bad_example/bad_example_fuzzer.cc

@@ -0,0 +1,42 @@
+// Copyright 2017 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include <stddef.h>
+#include <stdint.h>
+#include <string.h>
+
+#include <signal.h>
+#include <unistd.h>
+
+#include "zlib.h"
+
+static Bytef buffer[256 * 1024] = { 0 };
+
+
+#ifdef INTENTIONAL_STARTUP_CRASH
+void bad_term_handler(int signum) {
+  _exit(0);
+}
+#endif
+
+// Entry point for LibFuzzer.
+extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) {
+#ifdef INTENTIONAL_STARTUP_CRASH
+  // Simulates the worst case, fuzz target silently dies without any error.
+  struct sigaction action = { 0 };
+  action.sa_handler = bad_term_handler;
+  sigaction(SIGTERM, &action, NULL);
+
+  // Cannot call _exit(0) directly, as it's even worse -- sancov does not print
+  // any coverage information in that case.
+  kill(getpid(), SIGTERM);
+#endif
+
+  uLongf buffer_length = static_cast<uLongf>(sizeof(buffer));
+  if (Z_OK != uncompress(buffer, &buffer_length, data,
+                         static_cast<uLong>(size))) {
+    return 0;
+  }
+  return 0;
+}

projects/bad_example/build.sh

@@ -0,0 +1,57 @@
+#!/bin/bash -eu
+
+# Testcase 1. Valid fuzzer build.
+################################################################################
+./configure
+make -j$(nproc) clean
+make -j$(nproc) all
+
+$CXX $CXXFLAGS -std=c++11 -I. \
+    $SRC/bad_example_fuzzer.cc -o $OUT/bad_example_valid_build \
+    -lFuzzingEngine ./libz.a
+
+
+# Testcase 2. Silent startup crash.
+################################################################################
+./configure
+make -j$(nproc) clean
+make -j$(nproc) all
+
+$CXX $CXXFLAGS -std=c++11 -I. -DINTENTIONAL_STARTUP_CRASH \
+    $SRC/bad_example_fuzzer.cc -o $OUT/bad_example_startup_crash \
+    -lFuzzingEngine ./libz.a
+
+
+# Testcase 3. Ignore the flags provided by OSS-Fuzz.
+################################################################################
+export CFLAGS="-O1"
+export CXXFLAGS="-O1 -stdlib=libc++"
+
+./configure
+make -j$(nproc) clean
+make -j$(nproc) all
+
+$CXX -fsanitize=$SANITIZER $CXXFLAGS -std=c++11 -I. \
+    $SRC/bad_example_fuzzer.cc -o $OUT/bad_example_no_instrumentation \
+    -lFuzzingEngine ./libz.a
+
+
+# Testcase 4. Enable multiple sanitizers.
+################################################################################
+# Add UBSan to ASan or MSan build. Add ASan to UBSan build.
+EXTRA_SANITIZER="undefined"
+if [[ $SANITIZER = *undefined* ]]
+then
+  EXTRA_SANITIZER="address"
+fi
+
+export CFLAGS="-O1 -fsanitize=$SANITIZER,$EXTRA_SANITIZER -fsanitize-coverage=trace-pc-guard,trace-cmp"
+export CXXFLAGS="-O1 -fsanitize=$SANITIZER,$EXTRA_SANITIZER -fsanitize-coverage=trace-pc-guard,trace-cmp -stdlib=libc++"
+
+./configure
+make -j$(nproc) clean
+make -j$(nproc) all
+
+$CXX $CXXFLAGS -std=c++11 -I. \
+    $SRC/bad_example_fuzzer.cc -o $OUT/bad_example_mixed_sanitizers \
+    -lFuzzingEngine ./libz.a

projects/bad_example/project.yaml

@@ -0,0 +1,6 @@
+disabled: True
+homepage: "http://www.zlib.net/"
+sanitizers:
+  - address
+  - memory
+  - undefined