From 71a9130e3ead1f43e043dce3f8f056d33f57a334 Mon Sep 17 00:00:00 2001 From: jonathanmetzman <31354670+jonathanmetzman@users.noreply.github.com> Date: Thu, 26 Aug 2021 14:17:55 -0700 Subject: [PATCH] [Ubuntu upgrade] Copy files from base-builder/ to base-builder-new/ (#6322) Two of these changes that are important for the Ubuntu upgrade are the changes to "compile" and write_labels.py" --- .../base-builder-new/bazel_build_fuzz_tests | 10 ++++++++- infra/base-images/base-builder-new/compile | 21 +++++++++++-------- .../base-images/base-builder-new/compile_afl | 10 ++++++--- .../base-builder-new/write_labels.py | 2 +- 4 files changed, 29 insertions(+), 14 deletions(-) diff --git a/infra/base-images/base-builder-new/bazel_build_fuzz_tests b/infra/base-images/base-builder-new/bazel_build_fuzz_tests index d843d75f6..dca79f3f2 100755 --- a/infra/base-images/base-builder-new/bazel_build_fuzz_tests +++ b/infra/base-images/base-builder-new/bazel_build_fuzz_tests @@ -22,10 +22,17 @@ : "${BAZEL_TOOL:=bazel}" : "${BAZEL_EXTRA_BUILD_FLAGS:=}" +if [ "$FUZZING_LANGUAGE" = "jvm" ]; then + BAZEL_LANGUAGE=java +else + BAZEL_LANGUAGE=cc +fi + if [[ -z "${BAZEL_FUZZ_TEST_QUERY:-}" ]]; then BAZEL_FUZZ_TEST_QUERY=" let all_fuzz_tests = attr(tags, \"${BAZEL_FUZZ_TEST_TAG}\", \"//...\") in - \$all_fuzz_tests - attr(tags, \"${BAZEL_FUZZ_TEST_EXCLUDE_TAG}\", \$all_fuzz_tests) + let lang_fuzz_tests = attr(generator_function, \"^${BAZEL_LANGUAGE}_fuzz_test\$\", \$all_fuzz_tests) in + \$lang_fuzz_tests - attr(tags, \"${BAZEL_FUZZ_TEST_EXCLUDE_TAG}\", \$lang_fuzz_tests) " fi @@ -45,6 +52,7 @@ declare -r BAZEL_BUILD_FLAGS=( "--@rules_fuzzing//fuzzing:cc_engine=@rules_fuzzing_oss_fuzz//:oss_fuzz_engine" \ "--@rules_fuzzing//fuzzing:cc_engine_instrumentation=oss-fuzz" \ "--@rules_fuzzing//fuzzing:cc_engine_sanitizer=none" \ + "--cxxopt=-stdlib=libc++" \ "--linkopt=-lc++" \ "--action_env=CC=${CC}" "--action_env=CXX=${CXX}" \ ${BAZEL_EXTRA_BUILD_FLAGS[*]} diff --git a/infra/base-images/base-builder-new/compile b/infra/base-images/base-builder-new/compile index 6e51df8a9..85ccd693f 100755 --- a/infra/base-images/base-builder-new/compile +++ b/infra/base-images/base-builder-new/compile @@ -73,13 +73,7 @@ then # export CXXFLAGS_EXTRA="-L/usr/msan/lib $CXXFLAGS_EXTRA" cp -R /usr/msan/lib/* /usr/lib/ - if [[ -z "${MSAN_LIBS_PATH-}" ]]; then - echo 'WARNING: Building without MSan instrumented libraries.' - else - # Copy all static libraries only. Don't include .so files because they can - # break non MSan compiled programs. - (cd "$MSAN_LIBS_PATH" && find . -name '*.a' -exec cp --parents '{}' / ';') - fi + echo 'Building without MSan instrumented libraries.' fi # Coverage flag overrides. @@ -118,7 +112,7 @@ export CFLAGS="$CFLAGS $SANITIZER_FLAGS $COVERAGE_FLAGS" export CXXFLAGS="$CFLAGS $CXXFLAGS_EXTRA" if [ "$FUZZING_LANGUAGE" = "python" ]; then - sanitizer_with_fuzzer_lib_dir=`python3 -c "import atheris; import os; print(os.path.dirname(atheris.path()))"` + sanitizer_with_fuzzer_lib_dir=`python3 -c "import atheris; import os; print(atheris.path())"` sanitizer_with_fuzzer_output_lib=$OUT/sanitizer_with_fuzzer.so if [ "$SANITIZER" = "address" ]; then cp $sanitizer_with_fuzzer_lib_dir/asan_with_fuzzer.so $sanitizer_with_fuzzer_output_lib @@ -146,6 +140,10 @@ if [ "$FUZZING_LANGUAGE" = "jvm" ]; then # Coverage builds require no instrumentation. cp $(which jazzer_driver) $jazzer_driver_with_sanitizer fi + + # Disable leak checking since the JVM triggers too many false positives. + export CFLAGS="$CFLAGS -fno-sanitize=leak" + export CXXFLAGS="$CXXFLAGS -fno-sanitize=leak" fi echo "---------------------------------------------------------------" @@ -160,7 +158,12 @@ BUILD_CMD="bash -eux $SRC/build.sh" # We need to preserve source code files for generating a code coverage report. # We need exact files that were compiled, so copy both $SRC and $WORK dirs. -COPY_SOURCES_CMD="cp -rL --parents $SRC $WORK /usr/include /usr/local/include $OUT" +COPY_SOURCES_CMD="cp -rL --parents $SRC $WORK /usr/include /usr/local/include $GOPATH $OSSFUZZ_RUSTPATH /rustc $OUT" + +# Copy rust std lib to its path with a hash +export rustch=`rustc --version --verbose | grep commit-hash | cut -d' ' -f2` +mkdir -p /rustc/$rustch/ +cp -r /rust/rustup/toolchains/nightly-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/ /rustc/$rustch/ if [ "${BUILD_UID-0}" -ne "0" ]; then adduser -u $BUILD_UID --disabled-password --gecos '' builder diff --git a/infra/base-images/base-builder-new/compile_afl b/infra/base-images/base-builder-new/compile_afl index 2764d8d9c..d6509c74c 100644 --- a/infra/base-images/base-builder-new/compile_afl +++ b/infra/base-images/base-builder-new/compile_afl @@ -22,6 +22,8 @@ # AFL++ settings. export AFL_LLVM_MODE_WORKAROUND=0 export AFL_ENABLE_DICTIONARY=0 +export AFL_ENABLE_CMPLOG=1 +export AFL_LAF_CHANCE=3 # Start compiling afl++. echo "Copying precompiled afl++" @@ -49,6 +51,8 @@ export ASAN_OPTIONS="detect_leaks=0:symbolize=0:detect_odr_violation=0:abort_on_ # edge coverage). test $(($RANDOM % 100)) -lt 20 && { export AFL_LLVM_INSTRUMENT=CLASSIC,CTX-2 + export AFL_ENABLE_CMPLOG=0 + export AFL_LAF_CHANCE=30 } # 40% chance to create a dictionary. @@ -56,15 +60,15 @@ test $(($RANDOM % 100)) -lt 40 && { export AFL_ENABLE_DICTIONARY=1 } -# 40% chance to perform CMPLOG/REDQUEEN. +# 60% chance to perform CMPLOG/REDQUEEN. rm -f "$OUT/afl_cmplog.txt" -test $(($RANDOM % 100)) -lt 20 && { +test "$AFL_ENABLE_CMPLOG" = "1" -a $(($RANDOM % 100)) -lt 60 && { export AFL_LLVM_CMPLOG=1 touch "$OUT/afl_cmplog.txt" } # 3% chance to perform COMPCOV/LAF_INTEL. -test $(($RANDOM % 100)) -lt 3 && { +test $(($RANDOM % 100)) -lt $AFL_LAF_CHANCE && { export AFL_LLVM_LAF_ALL=1 } diff --git a/infra/base-images/base-builder-new/write_labels.py b/infra/base-images/base-builder-new/write_labels.py index 6766e37fe..92a820a43 100755 --- a/infra/base-images/base-builder-new/write_labels.py +++ b/infra/base-images/base-builder-new/write_labels.py @@ -1,4 +1,4 @@ -#!/usr/bin/python3 +#!/usr/bin/env python3 # Copyright 2021 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License");