oss-fuzz/infra/base-images/base-builder/compile

#!/bin/bash -eu
# Copyright 2016 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
################################################################################

echo "---------------------------------------------------------------"

sysctl -w vm.mmap_rnd_bits=28

OSS_FUZZ_ON_DEMAND="${OSS_FUZZ_ON_DEMAND:-0}"

if [ "$FUZZING_LANGUAGE" = "jvm" ]; then
  if [ "$FUZZING_ENGINE" != "libfuzzer" ] && [ "$FUZZING_ENGINE" != "wycheproof" ]; then
    echo "ERROR: JVM projects can be fuzzed with libFuzzer or tested with wycheproof engines only."
    exit 1
  fi
  if [ "$SANITIZER" != "address" ] && [ "$SANITIZER" != "coverage" ] && [ "$SANITIZER" != "undefined" ] && [ "$SANITIZER" != "none" ] && [ "$SANITIZER" != "introspector" ]; then
    echo "ERROR: JVM projects can be fuzzed with AddressSanitizer or UndefinedBehaviorSanitizer or Introspector only."
    exit 1
  fi
  if [ "$ARCHITECTURE" != "x86_64" ]; then
    echo "ERROR: JVM projects can be fuzzed on x86_64 architecture only."
    exit 1
  fi
fi

if [ "$FUZZING_LANGUAGE" = "javascript" ]; then
  if [ "$FUZZING_ENGINE" != "libfuzzer" ]; then
    echo "ERROR: JavaScript projects can be fuzzed with libFuzzer engine only."
    exit 1
  fi
  if [ "$SANITIZER" != "coverage" ] && [ "$SANITIZER" != "none" ]; then
    echo "ERROR: JavaScript projects cannot be fuzzed with sanitizers."
    exit 1
  fi
  if [ "$ARCHITECTURE" != "x86_64" ]; then
    echo "ERROR: JavaScript projects can be fuzzed on x86_64 architecture only."
    exit 1
  fi
fi

if [ "$FUZZING_LANGUAGE" = "python" ]; then
  if [ "$FUZZING_ENGINE" != "libfuzzer" ]; then
    echo "ERROR: Python projects can be fuzzed with libFuzzer engine only."
    exit 1
  fi
  if [ "$SANITIZER" != "address" ] && [ "$SANITIZER" != "undefined" ] && [ "$SANITIZER" != "coverage" ] && [ "$SANITIZER" != "introspector" ]; then
    echo "ERROR: Python projects can be fuzzed with AddressSanitizer or UndefinedBehaviorSanitizer or Coverage or Fuzz Introspector only."
    exit 1
  fi
  if [ "$ARCHITECTURE" != "x86_64" ]; then
    echo "ERROR: Python projects can be fuzzed on x86_64 architecture only."
    exit 1
  fi
fi

if [ -z "${SANITIZER_FLAGS-}" ]; then
  FLAGS_VAR="SANITIZER_FLAGS_${SANITIZER}"
  export SANITIZER_FLAGS=${!FLAGS_VAR-}
fi

if [[ $ARCHITECTURE == "i386" ]]; then
    export CFLAGS="-m32 $CFLAGS"
    cp -R /usr/i386/lib/* /usr/local/lib
    cp -R /usr/i386/include/* /usr/local/include
fi

# Don't use a fuzzing engine with Jazzer which has libFuzzer built-in or with
# FuzzBench which will provide the fuzzing engine.
if [[ $FUZZING_ENGINE != "none" ]] && [[ $FUZZING_LANGUAGE != "jvm" ]] && [[ "${OSS_FUZZ_ON_DEMAND}" == "0" ]] ; then
  # compile script might override environment, use . to call it.
  . compile_${FUZZING_ENGINE}
fi

if [[ $SANITIZER_FLAGS = *sanitize=memory* ]]
then
  # Take all libraries from lib/msan
  # export CXXFLAGS_EXTRA="-L/usr/msan/lib $CXXFLAGS_EXTRA"
  cp -R /usr/msan/lib/* /usr/local/lib/x86_64-unknown-linux-gnu/
  cp -R /usr/msan/include/* /usr/local/include

  echo 'Building without MSan instrumented libraries.'
fi

# Coverage flag overrides.
COVERAGE_FLAGS_VAR="COVERAGE_FLAGS_${SANITIZER}"
if [[ -n ${!COVERAGE_FLAGS_VAR+x} ]]
then
  export COVERAGE_FLAGS="${!COVERAGE_FLAGS_VAR}"
fi

# Only need the default coverage instrumentation for libFuzzer or honggfuzz.
# Other engines bring their own.
if [ $FUZZING_ENGINE = "none" ] || [ $FUZZING_ENGINE = "afl" ] || [ $FUZZING_ENGINE = "centipede" ] || [ "${OSS_FUZZ_ON_DEMAND}" != "0" ]; then
  export COVERAGE_FLAGS=
fi

# Rust does not support sanitizers and coverage flags via CFLAGS/CXXFLAGS, so
# use RUSTFLAGS.
# FIXME: Support code coverage once support is in.
# See https://github.com/rust-lang/rust/issues/34701.
if [ "$SANITIZER" != "undefined" ] && [ "$SANITIZER" != "coverage" ] && [ "$SANITIZER" != "none" ] && [ "$ARCHITECTURE" != 'i386' ]; then
  export RUSTFLAGS="--cfg fuzzing -Zsanitizer=${SANITIZER} -Cdebuginfo=1 -Cforce-frame-pointers"
else
  export RUSTFLAGS="--cfg fuzzing -Cdebuginfo=1 -Cforce-frame-pointers"
fi
if [ "$SANITIZER" = "coverage" ]
then
    # link to C++ from comment in f5098035eb1a14aa966c8651d88ea3d64323823d
    export RUSTFLAGS="$RUSTFLAGS -Cinstrument-coverage -C link-arg=-lc++"
fi

# Add Rust libfuzzer flags.
# See https://github.com/rust-fuzz/libfuzzer/blob/master/build.rs#L12.
export CUSTOM_LIBFUZZER_PATH="$LIB_FUZZING_ENGINE_DEPRECATED"
export CUSTOM_LIBFUZZER_STD_CXX=c++

export CFLAGS="$CFLAGS $SANITIZER_FLAGS $COVERAGE_FLAGS"
export CXXFLAGS="$CFLAGS $CXXFLAGS_EXTRA"

if [ "$SANITIZER" = "undefined" ]; then
  # Disable "function" sanitizer for C code for now, because many projects,
  # possibly via legacy C code are affected.
  # The projects should be fixed and this workaround be removed in the future.
  # TODO(#11778):
  # https://github.com/google/oss-fuzz/issues/11778
  export CFLAGS="$CFLAGS -fno-sanitize=function"
fi

if [ "$FUZZING_LANGUAGE" = "go" ]; then
  # required by Go 1.20
  export CXX="${CXX} -lresolv"
fi

if [ "$FUZZING_LANGUAGE" = "python" ]; then
  sanitizer_with_fuzzer_lib_dir=`python3 -c "import atheris; import os; print(atheris.path())"`
  sanitizer_with_fuzzer_output_lib=$OUT/sanitizer_with_fuzzer.so
  if [ "$SANITIZER" = "address" ]; then
    cp $sanitizer_with_fuzzer_lib_dir/asan_with_fuzzer.so $sanitizer_with_fuzzer_output_lib
  elif [ "$SANITIZER" = "undefined" ]; then
    cp $sanitizer_with_fuzzer_lib_dir/ubsan_with_fuzzer.so $sanitizer_with_fuzzer_output_lib
  fi

  # Disable leak checking as it is unsupported.
  export CFLAGS="$CFLAGS -fno-sanitize=function,leak,vptr,"
  export CXXFLAGS="$CXXFLAGS -fno-sanitize=function,leak,vptr"
fi

# Copy latest llvm-symbolizer in $OUT for stack symbolization.
cp $(which llvm-symbolizer) $OUT/

# Copy Jazzer to $OUT if needed.
if [ "$FUZZING_LANGUAGE" = "jvm" ]; then
  cp $(which jazzer_agent_deploy.jar) $(which jazzer_driver) $(which jazzer_junit.jar) $OUT/
  jazzer_driver_with_sanitizer=$OUT/jazzer_driver_with_sanitizer
  if [ "$SANITIZER" = "address" ]; then
    cat > $jazzer_driver_with_sanitizer << 'EOF'
#!/bin/bash
this_dir=$(dirname "$0")
"$this_dir/jazzer_driver" --asan "$@"
EOF
  elif [ "$SANITIZER" = "undefined" ]; then
    cat > $jazzer_driver_with_sanitizer << 'EOF'
#!/bin/bash
this_dir=$(dirname "$0")
"$this_dir/jazzer_driver" --ubsan "$@"
EOF
  elif [ "$SANITIZER" = "coverage" ] ||  [ "$SANITIZER" = "introspector" ]; then
    # Coverage & introspector builds require no instrumentation.
    cp $(which jazzer_driver) $jazzer_driver_with_sanitizer
  fi
  chmod +x $jazzer_driver_with_sanitizer

  # Disable leak checking since the JVM triggers too many false positives.
  export CFLAGS="$CFLAGS -fno-sanitize=leak"
  export CXXFLAGS="$CXXFLAGS -fno-sanitize=leak"
fi

if [ "$SANITIZER" = "introspector" ]; then
  export AR=llvm-ar
  export NM=llvm-nm
  export RANLIB=llvm-ranlib

  export CFLAGS="$CFLAGS -g"
  export CXXFLAGS="$CXXFLAGS -g"
  export FI_BRANCH_PROFILE=1
  export FUZZ_INTROSPECTOR=1
  export FUZZ_INTROSPECTOR_AUTO_FUZZ=1

  # Move ar and ranlib
  mv /usr/bin/ar /usr/bin/old-ar
  mv /usr/bin/nm /usr/bin/old-nm
  mv /usr/bin/ranlib /usr/bin/old-ranlib

  ln -sf /usr/local/bin/llvm-ar /usr/bin/ar
  ln -sf /usr/local/bin/llvm-nm /usr/bin/nm
  ln -sf /usr/local/bin/llvm-ranlib /usr/bin/ranlib

  apt-get install -y libjpeg-dev zlib1g-dev libyaml-dev
  python3 -m pip install --upgrade pip setuptools
  python3 -m pip install cxxfilt pyyaml beautifulsoup4 lxml soupsieve
  python3 -m pip install --prefer-binary matplotlib

  python3 /fuzz-introspector/src/main.py light
  rsync -avu --delete "$SRC/inspector/" "$OUT/inspector"
fi

echo "---------------------------------------------------------------"
echo "CC=$CC"
echo "CXX=$CXX"
echo "CFLAGS=$CFLAGS"
echo "CXXFLAGS=$CXXFLAGS"
echo "RUSTFLAGS=$RUSTFLAGS"
echo "---------------------------------------------------------------"

if [ "${OSS_FUZZ_ON_DEMAND}" != "0" ]; then
  fuzzbench_build
  cp $(which llvm-symbolizer) $OUT/
  exit 0
fi


if [[ ! -z "${CAPTURE_REPLAY_SCRIPT-}" ]]; then
  # Capture a replaying build script which can be used for replaying the build
  # after a vanilla build. This script is meant to be used in a cached
  # container.
  python3 -m pip install bashlex
  python3 /usr/local/bin/bash_parser.py $SRC/build.sh
fi

# Prepare the build command to run the project's build script.
if [[ ! -z "${REPLAY_ENABLED-}" ]]; then
  # If this is a replay, then use replay_build.sh. This is expected to be
  # running in a cached container where a build has already happened prior.
  BUILD_CMD="bash -eux $SRC/replay_build.sh"
else
  BUILD_CMD="bash -eux $SRC/build.sh"
fi

# Set +u temporarily to continue even if GOPATH and OSSFUZZ_RUSTPATH are undefined.
set +u
# We need to preserve source code files for generating a code coverage report.
# We need exact files that were compiled, so copy both $SRC and $WORK dirs.
COPY_SOURCES_CMD="cp -rL --parents $SRC $WORK /usr/include /usr/local/include $GOPATH $OSSFUZZ_RUSTPATH /rustc $OUT"
set -u

if [ "$FUZZING_LANGUAGE" = "rust" ]; then
  # Copy rust std lib to its path with a hash.
  export rustch=`rustc --version --verbose | grep commit-hash | cut -d' ' -f2`
  mkdir -p /rustc/$rustch/
  export rustdef=`rustup toolchain list | grep default | cut -d' ' -f1`
  cp -r /rust/rustup/toolchains/$rustdef/lib/rustlib/src/rust/library/ /rustc/$rustch/
fi

if [ "${BUILD_UID-0}" -ne "0" ]; then
  adduser -u $BUILD_UID --disabled-password --gecos '' builder
  chown -R builder $SRC $OUT $WORK
  su -c "$BUILD_CMD" builder
  if [ "$SANITIZER" = "coverage" ]; then
    # Some directories have broken symlinks (e.g. honggfuzz), ignore the errors.
    su -c "$COPY_SOURCES_CMD" builder 2>/dev/null || true
  fi
else
  $BUILD_CMD
  if [ "$SANITIZER" = "coverage" ]; then
    # Some directories have broken symlinks (e.g. honggfuzz), ignore the errors.
    $COPY_SOURCES_CMD 2>/dev/null || true
  fi
fi

if [ "$SANITIZER" = "introspector" ]; then
  unset CXXFLAGS
  unset CFLAGS
  export G_ANALYTICS_TAG="G-8WTFM1Y62J"

  if [ "$FUZZING_LANGUAGE" = "jvm" ]; then
    echo "GOING jvm route"

    set -x
    # Output will be put in /out/
    python3 /fuzz-introspector/frontends/java/oss-fuzz-main.py
    # Move files temporarily to fit workflow of other languages.
    mkdir -p $SRC/my-fi-data
    find $OUT/ -name *.data -exec mv {} $SRC/my-fi-data/ \;
    find $OUT/ -name *.data.yaml -exec mv {} $SRC/my-fi-data/ \;
  fi

  mkdir -p $SRC/inspector
  find $SRC/ -name "fuzzerLogFile-*.data" -exec cp {} $SRC/inspector/ \;
  find $SRC/ -name "fuzzerLogFile-*.data.yaml" -exec cp {} $SRC/inspector/ \;
  find $SRC/ -name "fuzzerLogFile-*.data.debug_*" -exec cp {} $SRC/inspector/ \;
  find $SRC/ -name "allFunctionsWithMain-*.yaml" -exec cp {} $SRC/inspector/ \;

  # Move coverage report.
  if [ -d "$OUT/textcov_reports" ]
  then
    find $OUT/textcov_reports/ -name "*.covreport" -exec cp {} $SRC/inspector/ \;
    find $OUT/textcov_reports/ -name "*.json" -exec cp {} $SRC/inspector/ \;
  fi

  cd $SRC/inspector

  # Make fuzz-introspector HTML report.
  REPORT_ARGS="--name=$PROJECT_NAME"
  # Only pass coverage_url when COVERAGE_URL is set (in cloud builds)
  if [[ ! -z "${COVERAGE_URL+x}" ]]; then
    REPORT_ARGS="$REPORT_ARGS --coverage_url=${COVERAGE_URL}"
  fi

  # Do different things depending on languages
  if [ "$FUZZING_LANGUAGE" = "python" ]; then
    echo "GOING python route"
    set -x
    REPORT_ARGS="$REPORT_ARGS --target_dir=$SRC/inspector"
    REPORT_ARGS="$REPORT_ARGS --language=python"
    python3 /fuzz-introspector/src/main.py report $REPORT_ARGS
    cp -rf $SRC/inspector $OUT/inspector
  elif [ "$FUZZING_LANGUAGE" = "jvm" ]; then
    echo "GOING jvm route"
    set -x
    find $OUT/ -name "jacoco.xml" -exec cp {} $SRC/inspector/ \;
    REPORT_ARGS="$REPORT_ARGS --target_dir=$SRC/inspector"
    REPORT_ARGS="$REPORT_ARGS --language=jvm"
    python3 /fuzz-introspector/src/main.py report $REPORT_ARGS
    cp -rf $SRC/inspector $OUT/inspector
  else
    # C/C++

    # Correlate fuzzer binaries to fuzz-introspector's raw data
    python3 /fuzz-introspector/src/main.py correlate --binaries_dir=$OUT/

    # Generate fuzz-introspector HTML report, this generates
    # the file exe_to_fuzz_introspector_logs.yaml
    REPORT_ARGS="$REPORT_ARGS --target_dir=$SRC/inspector"
    # Use the just-generated correlation file
    REPORT_ARGS="$REPORT_ARGS --correlation_file=exe_to_fuzz_introspector_logs.yaml"
    python3 /fuzz-introspector/src/main.py report $REPORT_ARGS

    rsync -avu --delete "$SRC/inspector/" "$OUT/inspector"
  fi
fi