From 9c5d905646e1456d75725e69f2e52f82c3b9317f Mon Sep 17 00:00:00 2001 From: jonvolfson <37542432+jonvolfson@users.noreply.github.com> Date: Wed, 8 Jul 2020 17:41:15 -0400 Subject: [PATCH] Dockerfile and build.sh modifications to get successful build (#4025) * Add spanner emulator project * Adding auto_css for Sneha and myself * Adding dockerfile and build.sh files * Further fixes for OSS-Fuzz integration * Update build.sh * Cleaning up commented code in build.sh * Fuzzing branch merged with main in the emulator repo, modified dockerfile to clone main now instead of the branch. * Updating build.sh to copy the binaries to out * Cleaning up build.sh by removing Envoy specific comments, removed dictionary code as no corpus exists yet * Updating yaml to include the memory sanitizer * Build.sh should now copy fuzzing binaries properly to , adding fuzzing_enginers parameter to yaml to bypass AFL timeout for now. Co-authored-by: Jonathan Volfson Co-authored-by: Oliver Chang --- projects/spanner_emulator/Dockerfile | 27 +++++++++ projects/spanner_emulator/build.sh | 78 ++++++++++++++++++++++++++ projects/spanner_emulator/project.yaml | 8 ++- 3 files changed, 112 insertions(+), 1 deletion(-) create mode 100644 projects/spanner_emulator/Dockerfile create mode 100755 projects/spanner_emulator/build.sh diff --git a/projects/spanner_emulator/Dockerfile b/projects/spanner_emulator/Dockerfile new file mode 100644 index 000000000..0f13a7f80 --- /dev/null +++ b/projects/spanner_emulator/Dockerfile @@ -0,0 +1,27 @@ +# Copyright 2020 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +################################################################################ + +FROM gcr.io/oss-fuzz-base/base-builder +MAINTAINER evmaus@google.com +RUN apt-get update && apt-get -y install make autoconf automake libtool wget openjdk-8-jdk python libunwind-dev tzdata + +# Install Bazelisk +RUN wget -O /usr/local/bin/bazel https://github.com/bazelbuild/bazelisk/releases/download/v0.0.8/bazelisk-linux-amd64 +RUN chmod +x /usr/local/bin/bazel + +RUN git clone https://github.com/googleinterns/cloud-spanner-emulator-fuzzing.git fuzz +WORKDIR fuzz +COPY build.sh $SRC/ diff --git a/projects/spanner_emulator/build.sh b/projects/spanner_emulator/build.sh new file mode 100755 index 000000000..79e7978ae --- /dev/null +++ b/projects/spanner_emulator/build.sh @@ -0,0 +1,78 @@ +#!/bin/bash -eu +# Copyright 2020 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +################################################################################ + +export CFLAGS="$CFLAGS" +export CXXFLAGS="$CXXFLAGS" + +declare -r FUZZER_TARGETS_CC=$(find . -name *_fuzz_test.cc) +declare -r FUZZER_TARGETS="$(for t in ${FUZZER_TARGETS_CC}; do echo "${t:2:-3}"; done)" + +# Copy $CFLAGS and $CXXFLAGS into Bazel command-line flags, for both +# compilation and linking. +# +# Some flags, such as `-stdlib=libc++`, generate warnings if used on a C source +# file. Since the build runs with `-Werror` this will cause it to break, so we +# use `--conlyopt` and `--cxxopt` instead of `--copt`. +declare -r EXTRA_BAZEL_FLAGS="$( +for f in ${CFLAGS}; do + echo "--conlyopt=${f}" "--linkopt=${f}" +done +for f in ${CXXFLAGS}; do + echo "--cxxopt=${f}" "--linkopt=${f}" +done +)" + +declare BAZEL_TARGET_PATH="k8-fastbuild/bin/src/fuzz" +declare BAZEL_BUILD_TARGETS="//src/fuzz:all" + +# Temporary hack, see https://github.com/google/oss-fuzz/issues/383 +readonly NO_VPTR='--copt=-fno-sanitize=vptr --linkopt=-fno-sanitize=vptr' + +# Build driverless libraries. +bazel build --verbose_failures --strip=never \ + --dynamic_mode=off \ + --copt=-fno-sanitize=vptr \ + --linkopt=-fno-sanitize=vptr \ + --copt -D__SANITIZE_ADDRESS__ \ + --copt -D__OSS_FUZZ__ \ + --copt -fno-sanitize-blacklist \ + --cxxopt="-stdlib=libc++" \ + --linkopt="--rtlib=compiler-rt" \ + --linkopt="--unwindlib=libunwind" \ + --linkopt="-stdlib=libc++" \ + --linkopt="-lc++" \ + --linkopt=-pthread ${EXTRA_BAZEL_FLAGS} \ + --define LIB_FUZZING_ENGINE=${LIB_FUZZING_ENGINE} \ + --linkopt="-rpath '\$ORIGIN\/lib'" \ + ${NO_VPTR} \ + ${EXTRA_BAZEL_FLAGS} \ + ${BAZEL_BUILD_TARGETS[*]} + +# Move out dynamically linked libraries +mkdir -p $OUT/lib +cp /usr/lib/x86_64-linux-gnu/libunwind.so.8 $OUT/lib/ + +# Move out tzdata +mkdir -p $OUT/data +cp -r /usr/share/zoneinfo $OUT/data/ + +# Move out fuzz target +cp "${SRC}"/fuzz/bazel-out/"${BAZEL_TARGET_PATH}"/*_fuzz_test "${OUT}"/ + +# Cleanup bazel- symlinks to avoid oss-fuzz trying to copy out of the build +# cache. +rm -f bazel-* diff --git a/projects/spanner_emulator/project.yaml b/projects/spanner_emulator/project.yaml index 53649751e..73cb68aa1 100644 --- a/projects/spanner_emulator/project.yaml +++ b/projects/spanner_emulator/project.yaml @@ -3,4 +3,10 @@ language: c++ primary_contact: "evmaus@google.com" auto_ccs: - "snehashah@google.com" - - "volfson@google.com" \ No newline at end of file + - "volfson@google.com" +fuzzing_engines: + - libfuzzer + - honggfuzz +sanitizers: + - address + - memory