From 9e39d350e81bc19eabdf99c64ebbe329493a8c93 Mon Sep 17 00:00:00 2001 From: Navidem Date: Wed, 19 Jan 2022 19:22:27 -0600 Subject: [PATCH] apply patches needed for fuzz introspector integration (#7122) --- infra/base-images/base-builder/Dockerfile | 6 ++++ infra/base-images/base-builder/compile | 32 +++++++++++++++++++ infra/base-images/base-clang/Dockerfile | 9 +++++- .../base-clang/checkout_build_install_llvm.sh | 26 +++++++++++++++ infra/base-images/base-runner/coverage | 3 ++ infra/constants.py | 3 +- infra/helper.py | 1 + 7 files changed, 78 insertions(+), 2 deletions(-) diff --git a/infra/base-images/base-builder/Dockerfile b/infra/base-images/base-builder/Dockerfile index 53522aaed..af020d891 100644 --- a/infra/base-images/base-builder/Dockerfile +++ b/infra/base-images/base-builder/Dockerfile @@ -83,6 +83,8 @@ ENV SANITIZER_FLAGS_dataflow "-fsanitize=dataflow" ENV SANITIZER_FLAGS_thread "-fsanitize=thread" +ENV SANITIZER_FLAGS_introspector "-flegacy-pass-manager -flto" + # Do not use any sanitizers in the coverage build. ENV SANITIZER_FLAGS_coverage "" @@ -140,6 +142,10 @@ RUN cd $SRC && \ COPY precompile_afl /usr/local/bin/ RUN precompile_afl +RUN git clone https://github.com/ossf/fuzz-introspector.git fuzz-introspector && \ + cd fuzz-introspector && \ + git checkout 9e3393cd551b65a6bf2de6495013b415f315f74e + COPY precompile_honggfuzz /usr/local/bin/ RUN precompile_honggfuzz diff --git a/infra/base-images/base-builder/compile b/infra/base-images/base-builder/compile index c934d3b5b..a2837b8c8 100755 --- a/infra/base-images/base-builder/compile +++ b/infra/base-images/base-builder/compile @@ -146,6 +146,19 @@ if [ "$FUZZING_LANGUAGE" = "jvm" ]; then export CXXFLAGS="$CXXFLAGS -fno-sanitize=leak" fi +if [ "$SANITIZER" = "introspector" ]; then + export LDFLAGS="-fuse-ld=gold" + export AR=llvm-ar + export RANLIB=llvm-ranlib + + # Move ar and ranlib + mv /usr/bin/ar /usr/bin/old-ar + mv /usr/bin/ranlib /usr/bin/old-ranlib + + ln -sf /usr/local/bin/llvm-ar /usr/bin/ar + ln -sf /usr/local/bin/llvm-ranlib /usr/bin/ranlib +fi + echo "---------------------------------------------------------------" echo "CC=$CC" echo "CXX=$CXX" @@ -186,6 +199,25 @@ else fi fi +if [ "$SANITIZER" = "introspector" ]; then + unset CXXFLAGS + unset CFLAGS + apt-get install -y libjpeg-dev zlib1g-dev + pip3 install --upgrade setuptools + pip3 install cxxfilt pyyaml beautifulsoup4 lxml soupsieve matplotlib + mkdir -p $SRC/inspector + + find $SRC/ -name "*.data" -exec cp {} $SRC/inspector/ \; + find $SRC/ -name "*.data.yaml" -exec cp {} $SRC/inspector/ \; + # Move coverage report. + find /workspace/ -name "*.covreport" -exec cp {} $SRC/inspector/ \; + + cd $SRC/inspector + python3 $SRC/fuzz-introspector/post-processing/main.py --target_dir=$SRC/inspector --git_repo_url=$GIT_REPO --coverage_url=$COVERAGE_URL + + cp -rf $SRC/inspector $OUT/inspector +fi + if [[ "$FUZZING_ENGINE" = "dataflow" ]]; then # Remove seed corpus as it can be huge but is not needed for a dataflow build. rm -f $OUT/*.zip diff --git a/infra/base-images/base-clang/Dockerfile b/infra/base-images/base-clang/Dockerfile index 45260941c..a403b20d0 100644 --- a/infra/base-images/base-clang/Dockerfile +++ b/infra/base-images/base-clang/Dockerfile @@ -18,9 +18,12 @@ FROM gcr.io/oss-fuzz-base/base-image +ARG introspector +ENV INTROSPECTOR_PATCHES=$introspector + # Install newer cmake. ENV CMAKE_VERSION 3.21.1 -RUN apt-get update && apt-get install -y wget sudo && \ +RUN apt-get update && apt-get install -y wget sudo git && \ wget https://github.com/Kitware/CMake/releases/download/v$CMAKE_VERSION/cmake-$CMAKE_VERSION-Linux-x86_64.sh && \ chmod +x cmake-$CMAKE_VERSION-Linux-x86_64.sh && \ ./cmake-$CMAKE_VERSION-Linux-x86_64.sh --skip-license --prefix="/usr/local" && \ @@ -28,6 +31,10 @@ RUN apt-get update && apt-get install -y wget sudo && \ SUDO_FORCE_REMOVE=yes apt-get remove --purge -y wget sudo && \ rm -rf /usr/local/doc/cmake /usr/local/bin/cmake-gui +RUN git clone https://github.com/ossf/fuzz-introspector.git fuzz-introspector && \ + cd fuzz-introspector && \ + git checkout 9e3393cd551b65a6bf2de6495013b415f315f74e + COPY checkout_build_install_llvm.sh /root/ # Keep all steps in the same script to decrease the number of intermediate # layes in docker file. diff --git a/infra/base-images/base-clang/checkout_build_install_llvm.sh b/infra/base-images/base-clang/checkout_build_install_llvm.sh index 4c3ea6a88..08a816249 100755 --- a/infra/base-images/base-clang/checkout_build_install_llvm.sh +++ b/infra/base-images/base-clang/checkout_build_install_llvm.sh @@ -24,6 +24,11 @@ NPROC=$(expr $(nproc) / 2) LLVM_DEP_PACKAGES="build-essential make cmake ninja-build git python3 python3-distutils g++-multilib binutils-dev zlib1g-dev" apt-get update && apt-get install -y $LLVM_DEP_PACKAGES --no-install-recommends +INTROSPECTOR_DEP_PACKAGES="texinfo bison flex" +if [ -n "$INTROSPECTOR_PATCHES" ]; then + apt-get install -y $INTROSPECTOR_DEP_PACKAGES +fi + # Checkout CHECKOUT_RETRIES=10 function clone_with_retries { @@ -93,6 +98,24 @@ fi git -C $LLVM_SRC checkout $LLVM_REVISION echo "Using LLVM revision: $LLVM_REVISION" +if [ -n "$INTROSPECTOR_PATCHES" ]; then + # For fuzz introspector. + echo "Applying introspector changes" + BBBASE=$PWD + cd $LLVM_SRC + cp -rf /fuzz-introspector/llvm/include/llvm/Transforms/Inspector/ ./llvm/include/llvm/Transforms//Inspector + cp -rf /fuzz-introspector/llvm/lib/Transforms/Inspector ./llvm/lib/Transforms/Inspector + + # LLVM currently does not support dynamically loading LTO passes. Thus, + # we hardcode it into Clang instead. + # Ref: https://reviews.llvm.org/D77704 + sed -i 's/whole-program devirtualization and bitset lowering./whole-program devirtualization and bitset lowering.\nPM.add(createInspectorPass());/g' ./llvm/lib/Transforms/IPO/PassManagerBuilder.cpp + sed -i 's/using namespace/#include "llvm\/Transforms\/Inspector\/Inspector.h"\nusing namespace/g' ./llvm/lib/Transforms/IPO/PassManagerBuilder.cpp + echo "add_subdirectory(Inspector)" >> ./llvm/lib/Transforms/CMakeLists.txt + sed -i 's/Instrumentation/Instrumentation\n Inspector/g' ./llvm/lib/Transforms/IPO/CMakeLists.txt + cd $BBBASE +fi + # Build & install. mkdir -p $WORK/llvm-stage2 $WORK/llvm-stage1 python3 $SRC/chromium_tools/clang/scripts/update.py --output-dir $WORK/llvm-stage1 @@ -187,6 +210,9 @@ cp -r $LLVM_SRC/compiler-rt/lib/fuzzer $SRC/libfuzzer rm -rf $LLVM_SRC rm -rf $SRC/chromium_tools apt-get remove --purge -y $LLVM_DEP_PACKAGES +if [ -n "$INTROSPECTOR_PATCHES" ]; then + apt-get remove --purge -y $INTROSPECTOR_DEP_PACKAGES +fi apt-get autoremove -y # Delete unneeded parts of LLVM to reduce image size. diff --git a/infra/base-images/base-runner/coverage b/infra/base-images/base-runner/coverage index 8312f38ce..30b8f682e 100755 --- a/infra/base-images/base-runner/coverage +++ b/infra/base-images/base-runner/coverage @@ -112,6 +112,9 @@ function run_fuzz_target { llvm-cov export -summary-only -instr-profile=$profdata_file -object=$target \ $shared_libraries $LLVM_COV_COMMON_ARGS > $FUZZER_STATS_DIR/$target.json + # For introspector. + llvm-cov show -instr-profile=$profdata_file -object=$target -line-coverage-gt=0 $shared_libraries $LLVM_COV_COMMON_ARGS > ${FUZZER_STATS_DIR}/$target.covreport + if [ -n "${FULL_SUMMARY_PER_TARGET-}" ]; then # This is needed for dataflow strategy analysis, can be removed later. See # - https://github.com/google/oss-fuzz/pull/3306 diff --git a/infra/constants.py b/infra/constants.py index a323a4368..c95c49c18 100644 --- a/infra/constants.py +++ b/infra/constants.py @@ -32,7 +32,8 @@ LANGUAGES = [ ] LANGUAGES_WITH_COVERAGE_SUPPORT = ['c', 'c++', 'go', 'jvm', 'rust', 'swift'] SANITIZERS = [ - 'address', 'none', 'memory', 'undefined', 'dataflow', 'thread', 'coverage' + 'address', 'none', 'memory', 'undefined', 'dataflow', 'thread', 'coverage', + 'introspector' ] ARCHITECTURES = ['i386', 'x86_64'] ENGINES = ['libfuzzer', 'afl', 'honggfuzz', 'dataflow', 'none'] diff --git a/infra/helper.py b/infra/helper.py index 2478048a0..b948c9f03 100755 --- a/infra/helper.py +++ b/infra/helper.py @@ -641,6 +641,7 @@ def build_fuzzers_impl( # pylint: disable=too-many-arguments,too-many-locals,to 'FUZZING_ENGINE=' + engine, 'SANITIZER=' + sanitizer, 'ARCHITECTURE=' + architecture, + 'GIT_REPO=', # TODO(navidem): load from main_repo in project.yaml. ] _add_oss_fuzz_ci_if_needed(env)