[Ubuntu upgrade] Copy files from base-builder/ to base-builder-new/ (#6322)

Two of these changes that are important for the Ubuntu upgrade are
the changes to "compile" and write_labels.py"

infra/base-images/base-builder-new/bazel_build_fuzz_tests

@@ -22,10 +22,17 @@
 : "${BAZEL_TOOL:=bazel}"
 : "${BAZEL_EXTRA_BUILD_FLAGS:=}"
 
+if [ "$FUZZING_LANGUAGE" = "jvm" ]; then
+    BAZEL_LANGUAGE=java
+else
+    BAZEL_LANGUAGE=cc
+fi
+
 if [[ -z "${BAZEL_FUZZ_TEST_QUERY:-}" ]]; then
     BAZEL_FUZZ_TEST_QUERY="
         let all_fuzz_tests = attr(tags, \"${BAZEL_FUZZ_TEST_TAG}\", \"//...\") in
-        \$all_fuzz_tests - attr(tags, \"${BAZEL_FUZZ_TEST_EXCLUDE_TAG}\", \$all_fuzz_tests)
+        let lang_fuzz_tests = attr(generator_function, \"^${BAZEL_LANGUAGE}_fuzz_test\$\", \$all_fuzz_tests) in
+        \$lang_fuzz_tests - attr(tags, \"${BAZEL_FUZZ_TEST_EXCLUDE_TAG}\", \$lang_fuzz_tests)
     "
 fi
 
@@ -45,6 +52,7 @@ declare -r BAZEL_BUILD_FLAGS=(
     "--@rules_fuzzing//fuzzing:cc_engine=@rules_fuzzing_oss_fuzz//:oss_fuzz_engine" \
     "--@rules_fuzzing//fuzzing:cc_engine_instrumentation=oss-fuzz" \
     "--@rules_fuzzing//fuzzing:cc_engine_sanitizer=none" \
+    "--cxxopt=-stdlib=libc++" \
     "--linkopt=-lc++" \
     "--action_env=CC=${CC}" "--action_env=CXX=${CXX}" \
     ${BAZEL_EXTRA_BUILD_FLAGS[*]}

infra/base-images/base-builder-new/compile

@@ -73,13 +73,7 @@ then
   # export CXXFLAGS_EXTRA="-L/usr/msan/lib $CXXFLAGS_EXTRA"
   cp -R /usr/msan/lib/* /usr/lib/
 
-  if [[ -z "${MSAN_LIBS_PATH-}" ]]; then
-    echo 'WARNING: Building without MSan instrumented libraries.'
-  else
-    # Copy all static libraries only. Don't include .so files because they can
-    # break non MSan compiled programs.
-    (cd "$MSAN_LIBS_PATH" && find . -name '*.a' -exec cp --parents '{}' / ';')
-  fi
+  echo 'Building without MSan instrumented libraries.'
 fi
 
 # Coverage flag overrides.
@@ -118,7 +112,7 @@ export CFLAGS="$CFLAGS $SANITIZER_FLAGS $COVERAGE_FLAGS"
 export CXXFLAGS="$CFLAGS $CXXFLAGS_EXTRA"
 
 if [ "$FUZZING_LANGUAGE" = "python" ]; then
-  sanitizer_with_fuzzer_lib_dir=`python3 -c "import atheris; import os; print(os.path.dirname(atheris.path()))"`
+  sanitizer_with_fuzzer_lib_dir=`python3 -c "import atheris; import os; print(atheris.path())"`
   sanitizer_with_fuzzer_output_lib=$OUT/sanitizer_with_fuzzer.so
   if [ "$SANITIZER" = "address" ]; then
     cp $sanitizer_with_fuzzer_lib_dir/asan_with_fuzzer.so $sanitizer_with_fuzzer_output_lib
@@ -146,6 +140,10 @@ if [ "$FUZZING_LANGUAGE" = "jvm" ]; then
     # Coverage builds require no instrumentation.
     cp $(which jazzer_driver) $jazzer_driver_with_sanitizer
   fi
+
+  # Disable leak checking since the JVM triggers too many false positives.
+  export CFLAGS="$CFLAGS -fno-sanitize=leak"
+  export CXXFLAGS="$CXXFLAGS -fno-sanitize=leak"
 fi
 
 echo "---------------------------------------------------------------"
@@ -160,7 +158,12 @@ BUILD_CMD="bash -eux $SRC/build.sh"
 
 # We need to preserve source code files for generating a code coverage report.
 # We need exact files that were compiled, so copy both $SRC and $WORK dirs.
-COPY_SOURCES_CMD="cp -rL --parents $SRC $WORK /usr/include /usr/local/include $OUT"
+COPY_SOURCES_CMD="cp -rL --parents $SRC $WORK /usr/include /usr/local/include $GOPATH $OSSFUZZ_RUSTPATH /rustc $OUT"
+
+# Copy rust std lib to its path with a hash
+export rustch=`rustc --version --verbose | grep commit-hash | cut -d' ' -f2`
+mkdir -p /rustc/$rustch/
+cp -r /rust/rustup/toolchains/nightly-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/ /rustc/$rustch/
 
 if [ "${BUILD_UID-0}" -ne "0" ]; then
   adduser -u $BUILD_UID --disabled-password --gecos '' builder

infra/base-images/base-builder-new/compile_afl

@@ -22,6 +22,8 @@
 # AFL++ settings.
 export AFL_LLVM_MODE_WORKAROUND=0
 export AFL_ENABLE_DICTIONARY=0
+export AFL_ENABLE_CMPLOG=1
+export AFL_LAF_CHANCE=3
 
 # Start compiling afl++.
 echo "Copying precompiled afl++"
@@ -49,6 +51,8 @@ export ASAN_OPTIONS="detect_leaks=0:symbolize=0:detect_odr_violation=0:abort_on_
 # edge coverage).
 test $(($RANDOM % 100)) -lt 20 && {
   export AFL_LLVM_INSTRUMENT=CLASSIC,CTX-2
+  export AFL_ENABLE_CMPLOG=0
+  export AFL_LAF_CHANCE=30
 }
 
 # 40% chance to create a dictionary.
@@ -56,15 +60,15 @@ test $(($RANDOM % 100)) -lt 40 && {
   export AFL_ENABLE_DICTIONARY=1
 }
 
-# 40% chance to perform CMPLOG/REDQUEEN.
+# 60% chance to perform CMPLOG/REDQUEEN.
 rm -f "$OUT/afl_cmplog.txt"
-test $(($RANDOM % 100)) -lt 20 && {
+test "$AFL_ENABLE_CMPLOG" = "1" -a $(($RANDOM % 100)) -lt 60 && {
   export AFL_LLVM_CMPLOG=1
   touch "$OUT/afl_cmplog.txt"
 }
 
 # 3% chance to perform COMPCOV/LAF_INTEL.
-test $(($RANDOM % 100)) -lt 3 && {
+test $(($RANDOM % 100)) -lt $AFL_LAF_CHANCE && {
   export AFL_LLVM_LAF_ALL=1
 }
 

infra/base-images/base-builder-new/write_labels.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
 # Copyright 2021 Google LLC
 #
 # Licensed under the Apache License, Version 2.0 (the "License");