From 87c0ba83132350cd2862bf9a81f555b54d18e795 Mon Sep 17 00:00:00 2001 From: Teju Nareddy Date: Fri, 28 Feb 2020 18:33:48 -0800 Subject: [PATCH] Add project: ESPv2 (#3445) * Add project: ESPv2 Copy most configs from envoy Signed-off-by: Teju Nareddy * Fix typo Signed-off-by: Teju Nareddy * Remove comments Signed-off-by: Teju Nareddy * Install all requirements Signed-off-by: Teju Nareddy * Fix format Signed-off-by: Teju Nareddy --- projects/esp-v2/Dockerfile | 47 ++++++++++++ projects/esp-v2/build.sh | 136 +++++++++++++++++++++++++++++++++++ projects/esp-v2/project.yaml | 10 +++ 3 files changed, 193 insertions(+) create mode 100644 projects/esp-v2/Dockerfile create mode 100755 projects/esp-v2/build.sh create mode 100644 projects/esp-v2/project.yaml diff --git a/projects/esp-v2/Dockerfile b/projects/esp-v2/Dockerfile new file mode 100644 index 000000000..59bb88d6c --- /dev/null +++ b/projects/esp-v2/Dockerfile @@ -0,0 +1,47 @@ +# Copyright 2020 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +################################################################################ + +## This file was copied from envoy (with minor changes). + +# TODO(https://github.com/google/oss-fuzz/issues/3093): Stop specifying the +# image SHA once the bug is fixed. +FROM gcr.io/oss-fuzz-base/base-builder@sha256:276813aef0ce5972db43c0230f96162003994fa742fb1b2f4e66c67498575c65 +MAINTAINER nareddyt@google.com + +RUN apt-get update && apt-get -y install \ + build-essential \ + openjdk-8-jdk \ + make \ + ninja-build \ + curl \ + autoconf \ + libtool \ + wget \ + golang \ + python + +# Install Bazelisk +RUN wget -O /usr/local/bin/bazel https://github.com/bazelbuild/bazelisk/releases/download/v0.0.8/bazelisk-linux-amd64; \ + chmod +x /usr/local/bin/bazel + +# Install cmake +RUN wget https://github.com/Kitware/CMake/releases/download/v3.14.5/cmake-3.14.5-Linux-x86_64.sh; \ + chmod +x cmake-3.14.5-Linux-x86_64.sh; \ + ./cmake-3.14.5-Linux-x86_64.sh --skip-license --prefix="/usr/local" + +RUN git clone --depth 1 https://github.com/GoogleCloudPlatform/esp-v2.git +WORKDIR $SRC/esp-v2/ +COPY build.sh $SRC/ diff --git a/projects/esp-v2/build.sh b/projects/esp-v2/build.sh new file mode 100755 index 000000000..6198e5f6d --- /dev/null +++ b/projects/esp-v2/build.sh @@ -0,0 +1,136 @@ +#!/bin/bash -eu +# Copyright 2020 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +################################################################################ + +## Copied from esp-v2 + +export CFLAGS="$CFLAGS" +export CXXFLAGS="$CXXFLAGS" + +declare -r FUZZER_TARGETS_CC=$(find . -name *_fuzz_test.cc) +declare -r FUZZER_TARGETS="$(for t in ${FUZZER_TARGETS_CC}; do echo "${t:2:-3}"; done)" + +FUZZER_DICTIONARIES="\ +" + +# Copy $CFLAGS and $CXXFLAGS into Bazel command-line flags, for both +# compilation and linking. +# +# Some flags, such as `-stdlib=libc++`, generate warnings if used on a C source +# file. Since the build runs with `-Werror` this will cause it to break, so we +# use `--conlyopt` and `--cxxopt` instead of `--copt`. +# +declare -r EXTRA_BAZEL_FLAGS="$( +for f in ${CFLAGS}; do + echo "--conlyopt=${f}" "--linkopt=${f}" +done +for f in ${CXXFLAGS}; do + echo "--cxxopt=${f}" "--linkopt=${f}" +done +if [ "$SANITIZER" = "undefined" ] +then + # Bazel uses clang to link binary, which does not link clang_rt ubsan library for C++ automatically. + # See issue: https://github.com/bazelbuild/bazel/issues/8777 + echo "--linkopt=\"$(find $(llvm-config --libdir) -name libclang_rt.ubsan_standalone_cxx-x86_64.a | head -1)\"" +fi +)" + +declare BAZEL_BUILD_TARGETS="" +declare BAZEL_CORPUS_TARGETS="" +declare FILTERED_FUZZER_TARGETS="" +for t in ${FUZZER_TARGETS} +do + declare BAZEL_PATH="//"$(dirname "$t")":"$(basename "$t") + declare TAGGED=$(bazel query "attr('tags', 'no_fuzz', ${BAZEL_PATH})") + if [ -z "${TAGGED}" ] + then + FILTERED_FUZZER_TARGETS+="$t " + BAZEL_BUILD_TARGETS+="${BAZEL_PATH}_driverless " + BAZEL_CORPUS_TARGETS+="${BAZEL_PATH}_corpus_tar " + fi +done + +# Build driverless libraries. +bazel build --verbose_failures --dynamic_mode=off --spawn_strategy=standalone \ + --genrule_strategy=standalone --strip=never \ + --copt=-fno-sanitize=vptr --linkopt=-fno-sanitize=vptr \ + --define tcmalloc=disabled --define signal_trace=disabled \ + --define ENVOY_CONFIG_ASAN=1 --copt -D__SANITIZE_ADDRESS__ \ + --define force_libcpp=enabled --build_tag_filters=-no_asan \ + --linkopt=-lc++ --linkopt=-pthread ${EXTRA_BAZEL_FLAGS} \ + ${BAZEL_BUILD_TARGETS[*]} ${BAZEL_CORPUS_TARGETS[*]} + +# Profiling with coverage requires that we resolve+copy all Bazel symlinks and +# also remap everything under proc/self/cwd to correspond to Bazel build paths. +if [ "$SANITIZER" = "coverage" ] +then + # The build invoker looks for sources in $SRC, but it turns out that we need + # to not be buried under src/, paths are expected at out/proc/self/cwd by + # the profiler. + declare -r REMAP_PATH="${OUT}/proc/self/cwd" + mkdir -p "${REMAP_PATH}" + + rsync -av "${SRC}"/esp-v2/src "${REMAP_PATH}" + # Remove filesystem loop manually. + rm -rf "${SRC}"/esp-v2/bazel-esp-v2/external/esp-v2 + # Clean up symlinks with a missing referrant. + find "${SRC}"/esp-v2/bazel-esp-v2/external -follow -type l -ls -delete || echo "Symlink cleanup soft fail" + rsync -avLk "${SRC}"/esp-v2/bazel-esp-v2/external "${REMAP_PATH}" + # For .h, and some generated artifacts, we need bazel-out/. Need to heavily + # filter out the build objects from bazel-out/. Also need to resolve symlinks, + # since they don't make sense outside the build container. + declare -r RSYNC_FILTER_ARGS=("--include" "*.h" "--include" "*.cc" "--include" \ + "*.hpp" "--include" "*.cpp" "--include" "*.c" "--include" "*/" "--exclude" "*") + rsync -avLk "${RSYNC_FILTER_ARGS[@]}" "${SRC}"/esp-v2/bazel-out "${REMAP_PATH}" + rsync -avLkR "${RSYNC_FILTER_ARGS[@]}" "${HOME}" "${OUT}" + rsync -avLkR "${RSYNC_FILTER_ARGS[@]}" /tmp "${OUT}" +fi + +# Copy out test driverless binaries from bazel-bin/. +for t in ${FILTERED_FUZZER_TARGETS} +do + TARGET_BASE="$(expr "$t" : '.*/\(.*\)_fuzz_test')" + TARGET_DRIVERLESS=bazel-bin/"${t}"_driverless + echo "Copying fuzzer $t" + cp "${TARGET_DRIVERLESS}" "${OUT}"/"${TARGET_BASE}"_fuzz_test +done + +# Zip up related test corpuses. +# TODO(nareddyt): just use the .tar directly when +# https://github.com/google/oss-fuzz/issues/1918 is fixed. +CORPUS_UNTAR_PATH="${PWD}"/_tmp_corpus +for t in ${FILTERED_FUZZER_TARGETS} +do + echo "Extracting and zipping fuzzer $t corpus" + rm -rf "${CORPUS_UNTAR_PATH}" + mkdir -p "${CORPUS_UNTAR_PATH}" + tar -C "${CORPUS_UNTAR_PATH}" -xvf bazel-bin/"${t}"_corpus_tar.tar + TARGET_BASE="$(expr "$t" : '.*/\(.*\)_fuzz_test')" + # There may be *.dict files in this folder that need to be moved into the OUT dir. + find "${CORPUS_UNTAR_PATH}" -type f -name *.dict -exec mv -n {} "${OUT}"/ \; + zip "${OUT}/${TARGET_BASE}"_fuzz_test_seed_corpus.zip \ + "${CORPUS_UNTAR_PATH}"/* +done +rm -rf "${CORPUS_UNTAR_PATH}" + +# Copy dictionaries and options files to $OUT/ +for d in $FUZZER_DICTIONARIES; do + cp "$d" "${OUT}"/ +done + +# Cleanup bazel- symlinks to avoid oss-fuzz trying to copy out of the build +# cache. +rm -f bazel-* diff --git a/projects/esp-v2/project.yaml b/projects/esp-v2/project.yaml new file mode 100644 index 000000000..23193cd51 --- /dev/null +++ b/projects/esp-v2/project.yaml @@ -0,0 +1,10 @@ +homepage: "https://github.com/GoogleCloudPlatform/esp-v2" +primary_contact: "jilinxia@google.com" +auto_ccs: +- "nareddyt@google.com" +- "taoxuy@google.com" +- "qiwzhang@google.com" +sanitizers: +- address +- undefined +coverage_extra_args: -ignore-filename-regex=.*\.cache.*esp-v2_deps_cache.* \ No newline at end of file