fuzztest integration (#8784)

Signed-off-by: David Korczynski <david@adalogics.com>

Cross referencing https://github.com/google/fuzztest/pull/69

Signed-off-by: David Korczynski <david@adalogics.com>
Co-authored-by: Oliver Chang <oliverchang@users.noreply.github.com>

infra/base-images/base-builder/Dockerfile

@@ -144,6 +144,7 @@ COPY cargo compile compile_afl compile_libfuzzer compile_honggfuzz \
     compile_go_fuzzer \
     compile_native_go_fuzzer \
     compile_python_fuzzer \
+    compile_fuzztests.sh \
     python_coverage_helper.py \
     debug_afl srcmap \
     write_labels.py bazel_build_fuzz_tests \

infra/base-images/base-builder/compile_fuzztests.sh

@@ -0,0 +1,86 @@
+#!/bin/bash -eu
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+################################################################################
+
+# Trigger setup_configs rule of fuzztest as it generates the necessary
+# configuration file based on OSS-Fuzz environment variables.
+bazel run @com_google_fuzztest//bazel:setup_configs > fuzztest.bazelrc
+
+# Bazel target names of the fuzz binaries.
+FUZZ_TEST_BINARIES=$(bazel query 'kind("cc_test", rdeps(..., @com_google_fuzztest//fuzztest:fuzztest_gtest_main))')
+
+# Bazel output paths of the fuzz binaries.
+FUZZ_TEST_BINARIES_OUT_PATHS=$(bazel cquery 'kind("cc_test", rdeps(..., @com_google_fuzztest//fuzztest:fuzztest_gtest_main))' --output=files)
+
+#echo "Fuzz test binaries:"
+#echo ${FUZZ_TEST_BINARIES}
+#echo "-----------------------------"
+#echo ${FUZZ_TEST_BINARIES3}
+#echo "-----------------------------"
+
+# Build the project and fuzz binaries
+bazel build --subcommands --config=oss-fuzz ${FUZZ_TEST_BINARIES[*]}
+
+# Iterate the fuzz binaries and list each fuzz entrypoint in the binary. For
+# each entrypoint create a wrapper script that calls into the binaries the
+# given entrypoint as argument.
+# The scripts will be named:
+# {binary_name}@{fuzztest_entrypoint}
+for fuzz_main_file in $FUZZ_TEST_BINARIES_OUT_PATHS; do
+  FUZZ_TESTS=$($fuzz_main_file --list_fuzz_tests)
+  cp ${fuzz_main_file} $OUT/
+  fuzz_basename=$(basename $fuzz_main_file)
+  chmod -x $OUT/$fuzz_basename
+  for fuzz_entrypoint in $FUZZ_TESTS; do
+    TARGET_FUZZER="${fuzz_basename}@$fuzz_entrypoint"
+
+    # Write executer script
+    echo "#!/bin/sh
+# LLVMFuzzerTestOneInput for fuzzer detection.
+this_dir=\$(dirname \"\$0\")
+chmod +x \$this_dir/$fuzz_basename
+$fuzz_basename --fuzz=$fuzz_entrypoint -- \$@" > $OUT/$TARGET_FUZZER
+    chmod +x $OUT/$TARGET_FUZZER
+  done
+done
+
+# synchronise coverage directory to bazel generated code.
+if [ "$SANITIZER" = "coverage" ]
+then 
+  # Synchronize bazel source files to coverage collection.
+  declare -r REMAP_PATH="${OUT}/proc/self/cwd"
+  mkdir -p "${REMAP_PATH}"
+  
+  # Synchronize the folder bazel-BAZEL_OUT_PROJECT.
+  declare -r RSYNC_FILTER_ARGS=("--include" "*.h" "--include" "*.cc" "--include" \
+    "*.hpp" "--include" "*.cpp" "--include" "*.c" "--include" "*/" "--include" "*.inc" \
+    "--exclude" "*")
+  project_folders="$(find . -name 'bazel-*' -type l -printf '%P\n' | \
+                    grep -v -x -F \
+                    -e 'bazel-bin' \
+                    -e 'bazel-out' \
+                    -e 'bazel-testlogs')"
+  for link in $project_folders; do
+   if [[ -d "${PWD}"/$link/external  ]]
+   then
+     rsync -avLk "${RSYNC_FILTER_ARGS[@]}" "${PWD}"/$link/external "${REMAP_PATH}"
+   fi
+  done
+
+  # Delete symlinks and sync the current folder.
+  find . -type l -ls -delete
+  rsync -av ${PWD}/ "${REMAP_PATH}"
+fi

infra/base-images/base-runner/bad_build_check

@@ -320,6 +320,11 @@ function check_mixed_sanitizers {
     return 0
   fi
 
+  # For fuzztest fuzzers point to the binary instead of launcher script.
+  if [[ $FUZZER == *"@"* ]]; then
+    FUZZER=(${FUZZER//@/ }[0])
+  fi
+
   CALL_INSN=
   if [[ $ARCHITECTURE == "x86_64" ]]
   then
@@ -401,6 +406,11 @@ function check_architecture {
     FUZZER=${FUZZER}.pkg
   fi
 
+  # For fuzztest fuzzers point to the binary instead of launcher script.
+  if [[ $FUZZER == *"@"* ]]; then
+    FUZZER=(${FUZZER//@/ }[0])
+  fi
+
   FILE_OUTPUT=$(file $FUZZER)
   if [[ $ARCHITECTURE == "x86_64" ]]
   then

infra/base-images/base-runner/coverage

@@ -100,13 +100,16 @@ function run_fuzz_target {
   fi
 
   rm -rf $corpus_dummy
-
   if (( $(du -c $profraw_file_mask | tail -n 1 | cut -f 1) == 0 )); then
     # Skip fuzz targets that failed to produce profile dumps.
     return 0
   fi
 
   # If necessary translate to latest profraw version.
+  if [[ $target == *"@"* ]]; then
+    # Extract fuzztest binary name from fuzztest wrapper script.
+    target=(${target//@/ }[0])
+  fi
   profraw_update.py $OUT/$target $profraw_file_mask $profraw_file_mask
   llvm-profdata merge -j=1 -sparse $profraw_file_mask -o $profdata_file
 
@@ -259,6 +262,11 @@ for fuzz_target in $FUZZ_TARGETS; do
     echo "Running $fuzz_target"
     run_fuzz_target $fuzz_target &
 
+    # Rewrite object if its a FUZZTEST target
+    if [[ $fuzz_target == *"@"* ]]; then
+      # Extract fuzztest binary name from fuzztest wrapper script.
+      fuzz_target=(${fuzz_target//@/ }[0])
+    fi
     if [[ -z $objects ]]; then
       # The first object needs to be passed without -object= flag.
       objects="$fuzz_target"
@@ -267,6 +275,7 @@ for fuzz_target in $FUZZ_TARGETS; do
     fi
   fi
 
+
   # Do not spawn more processes than the number of CPUs available.
   n_child_proc=$(jobs -rp | wc -l)
   while [ "$n_child_proc" -eq "$NPROC" ]; do
@@ -381,13 +390,20 @@ else
 
   # Per target reports.
   for fuzz_target in $FUZZ_TARGETS; do
-    profdata_path=$DUMPS_DIR/$fuzz_target.profdata
+    if [[ $fuzz_target == *"@"* ]]; then
+      profdata_path=$DUMPS_DIR/$fuzz_target.profdata
+      report_dir=$REPORT_BY_TARGET_ROOT_DIR/$fuzz_target
+      # Extract fuzztest binary name from fuzztest wrapper script.
+      fuzz_target=(${fuzz_target//@/ }[0])
+    else
+      profdata_path=$DUMPS_DIR/$fuzz_target.profdata
+      report_dir=$REPORT_BY_TARGET_ROOT_DIR/$fuzz_target
+    fi
     if [[ ! -f "$profdata_path" ]]; then
       echo "WARNING: $fuzz_target has no profdata generated."
       continue
     fi
 
-    report_dir=$REPORT_BY_TARGET_ROOT_DIR/$fuzz_target
     generate_html $profdata_path "$shared_libraries" "$fuzz_target" "$report_dir"
   done
 fi

projects/fuzztest-example/Dockerfile

@@ -0,0 +1,20 @@
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+################################################################################
+
+FROM gcr.io/oss-fuzz-base/base-builder
+RUN git clone --depth=1 https://github.com/google/fuzztest
+COPY build.sh $SRC/
+COPY fuzztest-enable-fuzzers.diff $SRC/

projects/fuzztest-example/build.sh

@@ -0,0 +1,23 @@
+#!/bin/bash -eu
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+################################################################################
+
+cd fuzztest
+git apply  --ignore-space-change --ignore-whitespace $SRC/fuzztest-enable-fuzzers.diff
+
+# Compile gfuzztests
+cd codelab
+compile_fuzztests.sh

projects/fuzztest-example/fuzztest-enable-fuzzers.diff

@@ -0,0 +1,37 @@
+diff --git a/codelab/escaping_test.cc b/codelab/escaping_test.cc
+index 5e5310e..d83901e 100644
+--- a/codelab/escaping_test.cc
++++ b/codelab/escaping_test.cc
+@@ -48,20 +48,20 @@ TEST(EscapingTest, UnescapingReplacesEscapeSequences) {
+ 
+ // Comment out the following fuzz tests:
+ 
+-// void UnescapingEscapedStringGivesOriginal(std::string_view s) {
+-//   EXPECT_EQ(s, Unescape(Escape(s)));
+-// }
+-// FUZZ_TEST(EscapingTest, UnescapingEscapedStringGivesOriginal);
++void UnescapingEscapedStringGivesOriginal(std::string_view s) {
++  EXPECT_EQ(s, Unescape(Escape(s)));
++}
++FUZZ_TEST(EscapingTest, UnescapingEscapedStringGivesOriginal);
+ 
+-// void EscapingAStringNeverTriggersUndefinedBehavior(std::string_view s) {
+-//   Escape(s);
+-// }
+-// FUZZ_TEST(EscapingTest, EscapingAStringNeverTriggersUndefinedBehavior);
++void EscapingAStringNeverTriggersUndefinedBehavior(std::string_view s) {
++  Escape(s);
++}
++FUZZ_TEST(EscapingTest, EscapingAStringNeverTriggersUndefinedBehavior);
+ 
+-// void UnescapingAStringNeverTriggersUndefinedBehavior(std::string_view s) {
+-//   Unescape(s);
+-// }
+-// FUZZ_TEST(EscapingTest, UnescapingAStringNeverTriggersUndefinedBehavior);
++void UnescapingAStringNeverTriggersUndefinedBehavior(std::string_view s) {
++   Unescape(s);
++}
++FUZZ_TEST(EscapingTest, UnescapingAStringNeverTriggersUndefinedBehavior);
+ 
+ }  // namespace
+ }  // namespace codelab

projects/fuzztest-example/project.yaml

@@ -0,0 +1,6 @@
+homepage: "https://github.com/google/fuzztest"
+language: c++
+primary_contact: "david@adalogics.com"
+main_repo: "https://github.com/google/fuzztest"
+fuzzing_engines:
+ - libfuzzer

projects/fuzztest-raksha/Dockerfile

@@ -0,0 +1,39 @@
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+################################################################################
+
+FROM gcr.io/oss-fuzz-base/base-builder
+RUN apt-get update && apt-get install -y make autoconf automake libtool default-jdk bison m4 \
+    build-essential\
+    curl \
+    doxygen \
+    flex \
+    libffi-dev \
+    libncurses5-dev \
+    libtool \
+    libsqlite3-dev \
+    mcpp \
+    sqlite \
+    uuid-runtime \
+    zlib1g-dev 
+
+RUN wget https://github.com/bazelbuild/buildtools/releases/download/4.2.5/buildifier-linux-amd64 \
+    -O /usr/local/bin/buildifier && chmod a+x /usr/local/bin/buildifier
+
+RUN git clone https://github.com/google-research/raksha raksha
+RUN git clone --depth=1 https://github.com/google/fuzztest/
+
+COPY build.sh $SRC/
+COPY *.diff $SRC/

projects/fuzztest-raksha/build.sh

@@ -0,0 +1,22 @@
+#!/bin/bash -eu
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+################################################################################
+# Extend with oss-fuzz settings. To be upsteamed?
+cd $SRC/raksha
+git apply  --ignore-space-change --ignore-whitespace $SRC/raksha-fuzztest.diff
+
+# Compile gfuzztests
+compile_fuzztests.sh

projects/fuzztest-raksha/project.yaml

@@ -0,0 +1,6 @@
+homepage: "https://github.com/google/fuzztest"
+language: c++
+primary_contact: "david@adalogics.com"
+main_repo: "https://github.com/google/fuzztest"
+fuzzing_engines:
+ - libfuzzer

projects/fuzztest-raksha/raksha-fuzztest.diff

@@ -0,0 +1,139 @@
+diff --git a/.bazelrc b/.bazelrc
+index 6bc032b..f4d9da6 100644
+--- a/.bazelrc
++++ b/.bazelrc
+@@ -85,3 +85,5 @@ build:asan --copt -O1
+ build:asan --copt -g
+ build:asan --copt -fno-omit-frame-pointer
+ build:asan --linkopt -fsanitize=address
++
++try-import %workspace%/fuzztest.bazelrc
+diff --git a/WORKSPACE b/WORKSPACE
+index 4d48310..6ceee4e 100644
+--- a/WORKSPACE
++++ b/WORKSPACE
+@@ -3,13 +3,18 @@ load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
+ #--------------------
+ # googletest
+ #--------------------
++#http_archive(
++#    name = "com_google_googletest",
++#    sha256 = "5cf189eb6847b4f8fc603a3ffff3b0771c08eec7dd4bd961bfd45477dd13eb73",
++#    strip_prefix = "googletest-609281088cfefc76f9d0ce82e1ff6c30cc3591e5",
++#    urls = ["https://github.com/google/googletest/archive/609281088cfefc76f9d0ce82e1ff6c30cc3591e5.zip"],
++#)
+ http_archive(
+     name = "com_google_googletest",
+-    sha256 = "5cf189eb6847b4f8fc603a3ffff3b0771c08eec7dd4bd961bfd45477dd13eb73",
+-    strip_prefix = "googletest-609281088cfefc76f9d0ce82e1ff6c30cc3591e5",
+-    urls = ["https://github.com/google/googletest/archive/609281088cfefc76f9d0ce82e1ff6c30cc3591e5.zip"],
++    sha256 = "81964fe578e9bd7c94dfdb09c8e4d6e6759e19967e397dbea48d1c10e45d0df2",
++    strip_prefix = "googletest-release-1.12.1",
++    url = "https://github.com/google/googletest/archive/refs/tags/release-1.12.1.tar.gz",
+ )
+-
+ # Protobuf:
+ #
+ # (See https://github.com/rules-proto-grpc/rules_proto_grpc)
+@@ -55,17 +60,15 @@ rules_foreign_cc_dependencies()
+ #--------------------
+ # Rules_m4
+ #--------------------
+-load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
+-
+ http_archive(
+     name = "rules_m4",
+-    sha256 = "b0309baacfd1b736ed82dc2bb27b0ec38455a31a3d5d20f8d05e831ebeef1a8e",
+-    urls = ["https://github.com/jmillikin/rules_m4/releases/download/v0.2.2/rules_m4-v0.2.2.tar.xz"],
++    sha256 = "c67fa9891bb19e9e6c1050003ba648d35383b8cb3c9572f397ad24040fb7f0eb",
++    urls = ["https://github.com/jmillikin/rules_m4/releases/download/v0.2/rules_m4-v0.2.tar.xz"],
+ )
+ 
+ load("@rules_m4//m4:m4.bzl", "m4_register_toolchains")
+ 
+-m4_register_toolchains(version = "1.4.18")
++m4_register_toolchains()
+ 
+ #--------------------
+ # flex
+@@ -75,11 +78,8 @@ http_archive(
+     sha256 = "f1685512937c2e33a7ebc4d5c6cf38ed282c2ce3b7a9c7c0b542db7e5db59d52",
+     urls = ["https://github.com/jmillikin/rules_flex/releases/download/v0.2/rules_flex-v0.2.tar.xz"],
+ )
+-
+ load("@rules_flex//flex:flex.bzl", "flex_register_toolchains")
+-
+ flex_register_toolchains()
+-
+ #--------------------
+ # Bison
+ #--------------------
+@@ -282,11 +282,12 @@ http_archive(
+ fuzztest_commit_hash = "4a8253b7299cf1764550054fae55f1b58a65bbaa"
+ 
+ # Fuzztest
+-http_archive(
+-    name = "fuzztest",
+-    sha256 = "72ae2d28323748d2c466396fa47038351ae81bedc50b8a3c6f1de6a9f02cde33",
+-    strip_prefix = "fuzztest-" + fuzztest_commit_hash,
+-    urls = ["https://github.com/google/fuzztest/archive/" + fuzztest_commit_hash + ".zip"],
++local_repository(
++    name = "com_google_fuzztest",
++    path = "/src/fuzztest",
++    #sha256 = "72ae2d28323748d2c466396fa47038351ae81bedc50b8a3c6f1de6a9f02cde33",
++    #strip_prefix = "fuzztest-" + fuzztest_commit_hash,
++    #:urls = ["https://github.com/google/fuzztest/archive/" + fuzztest_commit_hash + ".zip"],
+ )
+ 
+ #zetasql
+diff --git a/src/ir/datalog/BUILD b/src/ir/datalog/BUILD
+index 6b3a0c7..62155ec 100644
+--- a/src/ir/datalog/BUILD
++++ b/src/ir/datalog/BUILD
+@@ -24,7 +24,6 @@ cc_library(
+     name = "program",
+     hdrs = ["program.h"],
+     deps = [
+-        "@com_google_absl//absl/container:flat_hash_set",
+         "@com_google_absl//absl/strings",
+         "@com_google_absl//absl/strings:str_format",
+     ],
+@@ -48,25 +47,6 @@ cc_test(
+     ],
+ )
+ 
+-cc_test(
+-    name = "is_numeric_operator_test",
+-    srcs = ["is_numeric_operator_test.cc"],
+-    deps = [
+-        ":program",
+-        "//src/common/testing:gtest",
+-    ],
+-)
+-
+-cc_test(
+-    name = "hash_test",
+-    srcs = ["hash_test.cc"],
+-    deps = [
+-        ":program",
+-        "//src/common/testing:gtest",
+-        "@com_google_absl//absl/hash:hash_testing",
+-    ],
+-)
+-
+ # This library contains our C++ model of the value types of Souffle.
+ cc_library(
+     name = "value",
+@@ -131,9 +111,10 @@ cc_test(
+     srcs = ["value_test.cc"],
+     deps = [
+         ":value",
+-        "//src/common/testing:gtest",
+         "@com_google_absl//absl/strings",
+-        "@fuzztest//fuzztest",
++        "@com_google_fuzztest//fuzztest",
++        "@com_google_fuzztest//fuzztest:fuzztest_gtest_main",
++        "//src/common/testing:gtest",
+     ],
+ )
+