apply patches needed for fuzz introspector integration (#7122)

infra/base-images/base-builder/Dockerfile

@@ -83,6 +83,8 @@ ENV SANITIZER_FLAGS_dataflow "-fsanitize=dataflow"
 
 ENV SANITIZER_FLAGS_thread "-fsanitize=thread"
 
+ENV SANITIZER_FLAGS_introspector "-flegacy-pass-manager -flto"
+
 # Do not use any sanitizers in the coverage build.
 ENV SANITIZER_FLAGS_coverage ""
 
@@ -140,6 +142,10 @@ RUN cd $SRC && \
 COPY precompile_afl /usr/local/bin/
 RUN precompile_afl
 
+RUN git clone https://github.com/ossf/fuzz-introspector.git fuzz-introspector && \
+    cd fuzz-introspector && \
+    git checkout 9e3393cd551b65a6bf2de6495013b415f315f74e
+
 COPY precompile_honggfuzz /usr/local/bin/
 RUN precompile_honggfuzz
 

infra/base-images/base-builder/compile

@@ -146,6 +146,19 @@ if [ "$FUZZING_LANGUAGE" = "jvm" ]; then
   export CXXFLAGS="$CXXFLAGS -fno-sanitize=leak"
 fi
 
+if [ "$SANITIZER" = "introspector" ]; then
+  export LDFLAGS="-fuse-ld=gold"
+  export AR=llvm-ar
+  export RANLIB=llvm-ranlib
+
+  # Move ar and ranlib
+  mv /usr/bin/ar /usr/bin/old-ar
+  mv /usr/bin/ranlib /usr/bin/old-ranlib
+
+  ln -sf /usr/local/bin/llvm-ar /usr/bin/ar
+  ln -sf /usr/local/bin/llvm-ranlib /usr/bin/ranlib
+fi
+
 echo "---------------------------------------------------------------"
 echo "CC=$CC"
 echo "CXX=$CXX"
@@ -186,6 +199,25 @@ else
   fi
 fi
 
+if [ "$SANITIZER" = "introspector" ]; then
+  unset CXXFLAGS
+  unset CFLAGS
+  apt-get install -y libjpeg-dev zlib1g-dev
+  pip3 install --upgrade setuptools
+  pip3 install cxxfilt pyyaml beautifulsoup4 lxml soupsieve matplotlib
+  mkdir -p $SRC/inspector
+
+  find $SRC/ -name "*.data" -exec cp {} $SRC/inspector/ \;
+  find $SRC/ -name "*.data.yaml" -exec cp {} $SRC/inspector/ \;
+  # Move coverage report.
+  find /workspace/ -name "*.covreport" -exec cp {} $SRC/inspector/ \;
+  
+  cd $SRC/inspector
+  python3 $SRC/fuzz-introspector/post-processing/main.py --target_dir=$SRC/inspector --git_repo_url=$GIT_REPO --coverage_url=$COVERAGE_URL
+
+  cp -rf $SRC/inspector $OUT/inspector
+fi
+
 if [[ "$FUZZING_ENGINE" = "dataflow" ]]; then
   # Remove seed corpus as it can be huge but is not needed for a dataflow build.
   rm -f $OUT/*.zip

infra/base-images/base-clang/Dockerfile

@@ -18,9 +18,12 @@
 
 FROM gcr.io/oss-fuzz-base/base-image
 
+ARG introspector
+ENV INTROSPECTOR_PATCHES=$introspector
+
 # Install newer cmake.
 ENV CMAKE_VERSION 3.21.1
-RUN apt-get update && apt-get install -y wget sudo && \
+RUN apt-get update && apt-get install -y wget sudo git && \
     wget https://github.com/Kitware/CMake/releases/download/v$CMAKE_VERSION/cmake-$CMAKE_VERSION-Linux-x86_64.sh && \
     chmod +x cmake-$CMAKE_VERSION-Linux-x86_64.sh && \
     ./cmake-$CMAKE_VERSION-Linux-x86_64.sh --skip-license --prefix="/usr/local" && \
@@ -28,6 +31,10 @@ RUN apt-get update && apt-get install -y wget sudo && \
     SUDO_FORCE_REMOVE=yes apt-get remove --purge -y wget sudo && \
     rm -rf /usr/local/doc/cmake /usr/local/bin/cmake-gui
 
+RUN git clone https://github.com/ossf/fuzz-introspector.git fuzz-introspector && \
+    cd fuzz-introspector && \
+    git checkout 9e3393cd551b65a6bf2de6495013b415f315f74e
+
 COPY checkout_build_install_llvm.sh /root/
 # Keep all steps in the same script to decrease the number of intermediate
 # layes in docker file.

infra/base-images/base-clang/checkout_build_install_llvm.sh

@@ -24,6 +24,11 @@ NPROC=$(expr $(nproc) / 2)
 LLVM_DEP_PACKAGES="build-essential make cmake ninja-build git python3 python3-distutils g++-multilib binutils-dev zlib1g-dev"
 apt-get update && apt-get install -y $LLVM_DEP_PACKAGES --no-install-recommends
 
+INTROSPECTOR_DEP_PACKAGES="texinfo bison flex"
+if [ -n "$INTROSPECTOR_PATCHES" ]; then
+  apt-get install -y $INTROSPECTOR_DEP_PACKAGES
+fi
+
 # Checkout
 CHECKOUT_RETRIES=10
 function clone_with_retries {
@@ -93,6 +98,24 @@ fi
 git -C $LLVM_SRC checkout $LLVM_REVISION
 echo "Using LLVM revision: $LLVM_REVISION"
 
+if [ -n "$INTROSPECTOR_PATCHES" ]; then
+  # For fuzz introspector.
+  echo "Applying introspector changes"
+  BBBASE=$PWD
+  cd $LLVM_SRC 
+  cp -rf /fuzz-introspector/llvm/include/llvm/Transforms/Inspector/ ./llvm/include/llvm/Transforms//Inspector
+  cp -rf /fuzz-introspector/llvm/lib/Transforms/Inspector ./llvm/lib/Transforms/Inspector
+
+  # LLVM currently does not support dynamically loading LTO passes. Thus,
+  # we hardcode it into Clang instead.
+  # Ref: https://reviews.llvm.org/D77704
+  sed -i 's/whole-program devirtualization and bitset lowering./whole-program devirtualization and bitset lowering.\nPM.add(createInspectorPass());/g' ./llvm/lib/Transforms/IPO/PassManagerBuilder.cpp
+  sed -i 's/using namespace/#include "llvm\/Transforms\/Inspector\/Inspector.h"\nusing namespace/g' ./llvm/lib/Transforms/IPO/PassManagerBuilder.cpp
+  echo "add_subdirectory(Inspector)" >> ./llvm/lib/Transforms/CMakeLists.txt
+  sed -i 's/Instrumentation/Instrumentation\n  Inspector/g' ./llvm/lib/Transforms/IPO/CMakeLists.txt
+  cd $BBBASE  
+fi
+
 # Build & install.
 mkdir -p $WORK/llvm-stage2 $WORK/llvm-stage1
 python3 $SRC/chromium_tools/clang/scripts/update.py --output-dir $WORK/llvm-stage1
@@ -187,6 +210,9 @@ cp -r $LLVM_SRC/compiler-rt/lib/fuzzer $SRC/libfuzzer
 rm -rf $LLVM_SRC
 rm -rf $SRC/chromium_tools
 apt-get remove --purge -y $LLVM_DEP_PACKAGES
+if [ -n "$INTROSPECTOR_PATCHES" ]; then
+  apt-get remove --purge -y $INTROSPECTOR_DEP_PACKAGES
+fi
 apt-get autoremove -y
 
 # Delete unneeded parts of LLVM to reduce image size.

infra/base-images/base-runner/coverage

@@ -112,6 +112,9 @@ function run_fuzz_target {
   llvm-cov export -summary-only -instr-profile=$profdata_file -object=$target \
       $shared_libraries $LLVM_COV_COMMON_ARGS > $FUZZER_STATS_DIR/$target.json
 
+  # For introspector.
+  llvm-cov show -instr-profile=$profdata_file -object=$target -line-coverage-gt=0 $shared_libraries $LLVM_COV_COMMON_ARGS > ${FUZZER_STATS_DIR}/$target.covreport
+  
   if [ -n "${FULL_SUMMARY_PER_TARGET-}" ]; then
     # This is needed for dataflow strategy analysis, can be removed later. See
     # - https://github.com/google/oss-fuzz/pull/3306

infra/constants.py

@@ -32,7 +32,8 @@ LANGUAGES = [
 ]
 LANGUAGES_WITH_COVERAGE_SUPPORT = ['c', 'c++', 'go', 'jvm', 'rust', 'swift']
 SANITIZERS = [
-    'address', 'none', 'memory', 'undefined', 'dataflow', 'thread', 'coverage'
+    'address', 'none', 'memory', 'undefined', 'dataflow', 'thread', 'coverage',
+    'introspector'
 ]
 ARCHITECTURES = ['i386', 'x86_64']
 ENGINES = ['libfuzzer', 'afl', 'honggfuzz', 'dataflow', 'none']

infra/helper.py

@@ -641,6 +641,7 @@ def build_fuzzers_impl(  # pylint: disable=too-many-arguments,too-many-locals,to
       'FUZZING_ENGINE=' + engine,
       'SANITIZER=' + sanitizer,
       'ARCHITECTURE=' + architecture,
+      'GIT_REPO=',  # TODO(navidem): load from main_repo in project.yaml.
   ]
 
   _add_oss_fuzz_ci_if_needed(env)