envoy: use Bazel to build corpora. (#1917)

* envoy: use Bazel to build corpora.

Rather than scraping this out of the source tree, build each corpus
under Bazel. The advantage is that we can now support synthesized
corpora, e.g. automated generation from unit tests.

Signed-off-by: Harvey Tuch <htuch@google.com>

* Review feedback.

Signed-off-by: Harvey Tuch <htuch@google.com>

projects/envoy/Dockerfile

@@ -37,4 +37,4 @@ RUN apt-get update && apt-get install -y bazel
 
 RUN git clone https://github.com/envoyproxy/envoy.git
 WORKDIR $SRC/envoy/
-COPY find_corpus.py build.sh $SRC/
+COPY build.sh $SRC/

projects/envoy/build.sh

@@ -45,6 +45,7 @@ done
 )"
 
 declare BAZEL_BUILD_TARGETS=""
+declare BAZEL_CORPUS_TARGETS=""
 declare FILTERED_FUZZER_TARGETS=""
 for t in ${FUZZER_TARGETS}
 do
@@ -54,6 +55,7 @@ do
   then
     FILTERED_FUZZER_TARGETS+="$t "
     BAZEL_BUILD_TARGETS+="${BAZEL_PATH}_driverless "
+    BAZEL_CORPUS_TARGETS+="${BAZEL_PATH}_corpus_tar "
   fi
 done
 
@@ -67,7 +69,7 @@ bazel build --verbose_failures --dynamic_mode=off --spawn_strategy=standalone \
   --build_tag_filters=-no_asan \
   ${EXTRA_BAZEL_FLAGS} \
   --linkopt="-lFuzzingEngine" \
-  ${BAZEL_BUILD_TARGETS[*]}
+  ${BAZEL_BUILD_TARGETS[*]} ${BAZEL_CORPUS_TARGETS[*]}
 
 # Profiling with coverage requires that we resolve+copy all Bazel symlinks and
 # also remap everything under proc/self/cwd to correspond to Bazel build paths.
@@ -100,19 +102,31 @@ then
   #   /root/.cache "${OUT}"
 fi
 
-# Copy out test driverless binaries from bazel-bin/ and zip up related test
-# corpuses.
+# Copy out test driverless binaries from bazel-bin/.
 for t in ${FILTERED_FUZZER_TARGETS}
 do
-  TARGET_CORPUS=$(python "${SRC}"/find_corpus.py "$t")
   TARGET_BASE="$(expr "$t" : '.*/\(.*\)_fuzz_test')"
   TARGET_DRIVERLESS=bazel-bin/"${t}"_driverless
-  echo "Copying fuzzer $t and corpus"
+  echo "Copying fuzzer $t"
   cp "${TARGET_DRIVERLESS}" "${OUT}"/"${TARGET_BASE}"_fuzz_test
-  zip "${OUT}/${TARGET_BASE}"_fuzz_test_seed_corpus.zip \
-    "$(dirname "${t}")"/"${TARGET_CORPUS}"/*
 done
 
+# Zip up related test corpuses.
+# TODO(htuch): just use the .tar directly when
+# https://github.com/google/oss-fuzz/issues/1918 is fixed.
+CORPUS_UNTAR_PATH="${PWD}"/_tmp_corpus
+for t in ${FILTERED_FUZZER_TARGETS}
+do
+  echo "Extracting and zipping fuzzer $t corpus"
+  rm -rf "${CORPUS_UNTAR_PATH}"
+  mkdir -p "${CORPUS_UNTAR_PATH}"
+  tar -C "${CORPUS_UNTAR_PATH}" -xvf bazel-bin/"${t}"_corpus_tar.tar
+  TARGET_BASE="$(expr "$t" : '.*/\(.*\)_fuzz_test')"
+  zip "${OUT}/${TARGET_BASE}"_fuzz_test_seed_corpus.zip \
+    "${CORPUS_UNTAR_PATH}"/*
+done
+rm -rf "${CORPUS_UNTAR_PATH}"
+
 # Copy dictionaries and options files to $OUT/
 for d in $FUZZER_DICTIONARIES; do
   cp "$d" "${OUT}"/

projects/envoy/find_corpus.py

@@ -1,23 +0,0 @@
-#!/usr/bin/python
-
-import os
-import sys
-import re
-
-fuzzer_target = sys.argv[1]
-directory, fuzzer_target_name = os.path.dirname(fuzzer_target), os.path.basename(fuzzer_target)
-path = os.path.join('..', 'envoy', directory, 'BUILD')
-
-with open(path, 'r') as f:
-  searchlines = f.readlines()
-  for i, line in enumerate(searchlines):
-      if fuzzer_target_name in line:
-        for l in searchlines[i:]:
-          if 'corpus =' in l:
-            corpus_path = l
-            break
-try:
-  corpus_path
-except NameError:  
-  raise Exception("No corpus path for the given fuzz target")
-print re.findall(r'"([^"]*)"', corpus_path)[0]