[infra] do not checkout oss-fuzz (#23)

Promising oss-fuzz in /src/oss-fuzz creates lots of confusion about where files come from. Let's make everything explicit. Fixes #20
2016-10-12 16:25:06 -07:00 · 2016-10-12 16:25:06 -07:00 · dae2012980
parent 1eddcd93ca
commit dae2012980
25 changed files with 60 additions and 52 deletions
--- a/docs/building_running_fuzzers_external.md
+++ b/docs/building_running_fuzzers_external.md
@ -29,7 +29,7 @@ docker build -t ossfuzz/$PROJECT_NAME oss-fuzz/$PROJECT_NAME
 ````
 2. Running a container:
 ````bash
-docker run -ti -v $PWD/$PROJECT_NAME:/src/$PROJECT_NAME -v $PWD/oss-fuzz:/src/oss-fuzz -v /tmp/out:/out ossfuzz/$PROJECT_NAME
+docker run -ti -v $PWD/$PROJECT_NAME:/src/$PROJECT_NAME -v /tmp/out:/out ossfuzz/$PROJECT_NAME
 ````

 `/tmp/out` will contain fuzzers.
--- a/docs/new_library.md
+++ b/docs/new_library.md
@ -43,14 +43,38 @@ Create a fuzzer and add it to the *library_name/* directory as well.

 This is the Docker image definition that build.sh will be executed in.
 It is very simple for most libraries:
-```bash
+```docker
 FROM ossfuzz/base-libfuzzer             # base image with clang toolchain
-MAINTAINER YOUR_EMAIL                   # each file should have a maintainer 
+MAINTAINER YOUR_EMAIL                   # each file should have a maintainer
 RUN apt-get install -y ...              # install required packages to build a project
 COPY build.sh /src/                     # install build script for the project.
 ```
 Expat example: [expat/Dockerfile](../expat/Dockerfile)

+## Create Fuzzer Source File
+
+Create a new .cc file, define a `LLVMFuzzerTestOneInput` function and call
+your library:
+
+```c++
+#include <stddef.h>
+#include <stdint.h>
+
+extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) {
+  // put your fuzzing code here and use data+size as input.
+  return 0;
+}
+```
+
+Make sure you add the file to your Docker image:
+```docker
+COPY build.sh my_fuzzer.cc /src/         # install build script & fuzzer.
+```
+
+There are [lots](../libxml2/libxml2_xml_read_memory_fuzzer.cc)
+[of](../expat/parse_fuzzer.cc) [examples](../zlib/zlib_uncompress_fuzzer.cc)
+in this project repository.
+
 ## build.sh

 This is where most of the work is done to build fuzzers for your library. The script will
@ -105,24 +129,6 @@ These flags are provided in following environment variables:
 Many well-crafted build scripts will automatically use these variables. If not,
 passing them manually to a build tool might be required.

-## Create Fuzzer Source File
-
-Create a new .cc file, define a `LLVMFuzzerTestOneInput` function and call
-your library:
-
-```c++
-#include <stddef.h>
-#include <stdint.h>
-
-extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) {
-  // put your fuzzing code here and use data+size as input.
-  return 0;
-}
-```
-
-There are [lots](../libxml2/libxml2_xml_read_memory_fuzzer.cc) 
-[of](../expat/parse_fuzzer.cc) [examples](../zlib/zlib_uncompress_fuzzer.cc)
-in this project repository.

 ### Dictionaries and custom libfuzzer options

--- a/expat/Dockerfile
+++ b/expat/Dockerfile
@ -21,4 +21,4 @@ RUN apt-get install -y make autoconf automake libtool docbook2x
 ENV GIT_CHECKOUT_DIR="expat"
 ENV GIT_URL="git://git.code.sf.net/p/expat/code_git"

-COPY build.sh /src/
+COPY build.sh parse_fuzzer.*  xml.dict /src/
--- a/expat/README.md
+++ b/expat/README.md
@ -10,7 +10,7 @@ git clone https://github.com/google/oss-fuzz.git
 git clone git://git.code.sf.net/p/expat/code_git expat
 # Build & run the image.
 docker build -t ossfuzz/expat oss-fizz/expat && \
-docker run -i -v $PWD/oss-fuzz:/src/oss-fuzz -v $PWD/expat:/src/expat -v $HOME/tmp/out:/out -t ossfuzz/expat
+docker run -i -v $PWD/expat:/src/expat -v $HOME/tmp/out:/out -t ossfuzz/expat
 ````
 Fuzzers will be in `$HOME/tmp/out`.

--- a/expat/build.sh
+++ b/expat/build.sh
@ -6,5 +6,7 @@ cd /src/expat/expat
 make clean all

 $CXX $CXXFLAGS -std=c++11 -Ilib/ \
-    /src/oss-fuzz/expat/parse_fuzzer.cc -o /out/expat_parse_fuzzer \
+    /src/parse_fuzzer.cc -o /out/expat_parse_fuzzer \
    /work/libfuzzer/*.o .libs/libexpat.a $LDFLAGS
+
+cp /src/*.dict /src/*.options /out/
--- a/infra/base-images/base-libfuzzer/Dockerfile
+++ b/infra/base-images/base-libfuzzer/Dockerfile
@ -18,8 +18,6 @@ FROM ossfuzz/base-clang
 MAINTAINER mike.aizatsky@gmail.com
 RUN apt-get install -y git libc6-dev

-RUN cd /src && git clone --depth 1 https://github.com/google/oss-fuzz.git
-
 RUN mkdir -p /work/libfuzzer

 ENV SANITIZER_FLAGS="-fsanitize=address"
--- a/infra/libfuzzer-pipeline.groovy
+++ b/infra/libfuzzer-pipeline.groovy
@ -34,7 +34,6 @@ def call(body) {

    def date = java.time.format.DateTimeFormatter.ofPattern("yyyyMMddHHmm")
        .format(java.time.LocalDateTime.now())
-    def ossFuzzUrl = 'https://github.com/google/oss-fuzz.git'

    node {
      def workspace = pwd()
@ -44,10 +43,6 @@ def call(body) {

      stage("docker image") {
          def revisions = [:]
-          dir('oss-fuzz') {
-              git url: ossFuzzUrl
-          }
-
          dir(checkoutDir) {
              git url: gitUrl
              revisions[gitUrl] = sh(returnStdout: true, script: 'git rev-parse HEAD').trim()
@ -74,11 +69,7 @@ def call(body) {
            // Run image to produce fuzzers
            sh "rm -rf $out"
            sh "mkdir -p $out"
-            sh "docker run -v $workspace/$checkoutDir:/src/$checkoutDir -v $workspace/oss-fuzz:/src/oss-fuzz -v $out:/out -e SANITIZER_FLAGS=\"-fsanitize=$sanitizer\" -t $dockerTag"
-
-            // Copy dict and options files
-            sh "cp $workspace/oss-fuzz/$projectName/*.dict $out/ || true"
-            sh "cp $workspace/oss-fuzz/$projectName/*.options $out/ || true"
+            sh "docker run -v $workspace/$checkoutDir:/src/$checkoutDir -v $out:/out -e SANITIZER_FLAGS=\"-fsanitize=$sanitizer\" -t $dockerTag"
          }
        }
      }
--- a/libchewing/Dockerfile
+++ b/libchewing/Dockerfile
@ -18,4 +18,4 @@ FROM ossfuzz/base-libfuzzer
 MAINTAINER kcwu@csie.org
 RUN apt-get install -y make autoconf automake libtool texinfo

-COPY build.sh /src/
+COPY build.sh chewing_fuzzer.c /src/
--- a/libchewing/build.sh
+++ b/libchewing/build.sh
@ -27,7 +27,7 @@ make -C test CFLAGS="$CFLAGS -Dmain=stress_main -Drand=get_fuzz_input" stress.o

 $CC $CFLAGS \
    -o /out/chewing_fuzzer \
-    /src/oss-fuzz/libchewing/chewing_fuzzer.c \
+    /src/chewing_fuzzer.c \
    test/stress.o test/.libs/libtesthelper.a src/.libs/libchewing.a \
    /work/libfuzzer/*.o $LDFLAGS

--- a/libpng/Dockerfile
+++ b/libpng/Dockerfile
@ -18,4 +18,4 @@ FROM ossfuzz/base-libfuzzer
 MAINTAINER mmoroz@chromium.org
 RUN apt-get install -y make autoconf automake libtool zlib1g-dev

-COPY build.sh /src/
+COPY build.sh libpng_read_fuzzer.* png.dict /src/
--- a/libpng/build.sh
+++ b/libpng/build.sh
@ -29,5 +29,7 @@ make clean all

 # build libpng_read_fuzzer
 $CXX $CXXFLAGS -std=c++11 -I. -lz \
-     /src/oss-fuzz/libpng/libpng_read_fuzzer.cc -o /out/libpng_read_fuzzer \
+     /src/libpng_read_fuzzer.cc -o /out/libpng_read_fuzzer \
     /work/libfuzzer/*.o .libs/libpng16.a $LDFLAGS
+
+cp /src/*.dict /src/*.options /out/
--- a/libxml2/Dockerfile
+++ b/libxml2/Dockerfile
@ -19,3 +19,6 @@ MAINTAINER ochang@chromium.org
 RUN apt-get install -y make autoconf automake libtool pkg-config

 COPY build.sh /src/
+COPY libxml2_xml_read_memory_fuzzer.* \
+     libxml2_xml_regexp_compile_fuzzer.* \
+     xml.dict /src/
--- a/libxml2/build.sh
+++ b/libxml2/build.sh
@ -23,6 +23,8 @@ make clean all

 for fuzzer in libxml2_xml_read_memory_fuzzer libxml2_xml_regexp_compile_fuzzer; do
  $CXX $CXXFLAGS -std=c++11 -Iinclude/ \
-      /src/oss-fuzz/libxml2/$fuzzer.cc -o /out/$fuzzer \
+      /src/$fuzzer.cc -o /out/$fuzzer \
      /work/libfuzzer/*.o .libs/libxml2.a $LDFLAGS
 done
+
+cp /src/*.dict /src/*.options /out/
--- a/nss/Dockerfile
+++ b/nss/Dockerfile
@ -18,6 +18,6 @@ FROM ossfuzz/base-libfuzzer
 MAINTAINER mmoroz@chromium.org
 RUN apt-get install -y make autoconf automake libtool mercurial zlib1g-dev

-COPY build.sh /src/
+COPY build.sh fuzzers /src/

 ENV LD_LIBRARY_PATH "$LD_LIBRARY_PATH:/out"
--- a/nss/build.sh
+++ b/nss/build.sh
@ -57,7 +57,7 @@ FUZZERS="asn1_algorithmid_fuzzer \


 for fuzzer in $FUZZERS; do
-  $CXX $CXXFLAGS -std=c++11 /src/oss-fuzz/nss/fuzzers/$fuzzer.cc \
+  $CXX $CXXFLAGS -std=c++11 /src/fuzzers/$fuzzer.cc \
     -I/work/nss/include \
     /work/libfuzzer/*.o \
     /work/nss/lib/libnss.a /work/nss/lib/libnssutil.a \
--- a/re2/Dockerfile
+++ b/re2/Dockerfile
@ -18,4 +18,4 @@ FROM ossfuzz/base-libfuzzer
 MAINTAINER wrengr@chromium.org
 RUN apt-get install -y make autoconf automake libtool

-COPY build.sh /src/
+COPY build.sh re2_fuzzer.* /src/
--- a/re2/build.sh
+++ b/re2/build.sh
@ -31,6 +31,7 @@ make obj/libre2.a

 # Second, build our fuzzers.
 $CXX $CXXFLAGS -std=c++11 -I. \
-	/src/oss-fuzz/re2/re2_fuzzer.cc -o /out/re2_fuzzer \
+	/src/re2_fuzzer.cc -o /out/re2_fuzzer \
 	/work/libfuzzer/*.o ./obj/libre2.a $LDFLAGS

+cp /src/*.options /src/*.dict /out/\
--- a/re2/re2_fuzzer.options
+++ b/re2/re2_fuzzer.options
--- a/scripts/helper.py
+++ b/scripts/helper.py
@ -272,7 +272,6 @@ def shell(shell_args):

  command = [
        'docker', 'run', '-i',
-        '-v', '%s:/src/oss-fuzz' % OSSFUZZ_DIR,
        '-v', '%s:/src/%s' % (checkout_dir, args.library_name),
        '-v', '%s:/out' % os.path.join(BUILD_DIR, 'out', args.library_name),
        '-t', 'ossfuzz/' + args.library_name,
--- a/sqlite3/Dockerfile
+++ b/sqlite3/Dockerfile
@ -18,4 +18,4 @@ FROM ossfuzz/base-libfuzzer
 MAINTAINER tanin@google.com
 RUN apt-get install -y make autoconf automake libtool fossil tcl

-COPY build.sh /src/
+COPY build.sh sqlite3_fuzzer.* sql.dict /src/
--- a/sqlite3/build.sh
+++ b/sqlite3/build.sh
@ -33,5 +33,7 @@ make
 make sqlite3.c

 $CXX $CXXFLAGS -std=c++11 -I. \
-    /src/oss-fuzz/sqlite3/sqlite3_fuzzer.cc -o /out/sqlite3_fuzzer \
+    /src/sqlite3_fuzzer.cc -o /out/sqlite3_fuzzer \
    /work/libfuzzer/*.o ./sqlite3.o $LDFLAGS
+
+cp /src/*.options /src/*.dict /out/
--- a/woff2/Dockerfile
+++ b/woff2/Dockerfile
@ -18,4 +18,4 @@ FROM ossfuzz/base-libfuzzer
 MAINTAINER mmoroz@chromium.org
 RUN apt-get install -y make autoconf automake libtool

-COPY build.sh /src/
+COPY build.sh convert_woff2ttf_fuzzer.* /src/
--- a/woff2/build.sh
+++ b/woff2/build.sh
@ -41,5 +41,7 @@ rm src/woff2_compress.o src/woff2_decompress.o
 # Build the fuzzer.
 fuzzer=convert_woff2ttf_fuzzer
 $CXX $CXXFLAGS -std=c++11 -Isrc \
-    /src/oss-fuzz/woff2/$fuzzer.cc -o /out/$fuzzer \
+    /src/$fuzzer.cc -o /out/$fuzzer \
    /work/libfuzzer/*.o src/*.o brotli/dec/*.o brotli/enc/*.o $LDFLAGS
+
+cp /src/*.options /out/
--- a/zlib/Dockerfile
+++ b/zlib/Dockerfile
@ -18,4 +18,4 @@ FROM ossfuzz/base-libfuzzer
 MAINTAINER inferno@chromium.org
 RUN apt-get install -y make autoconf automake libtool

-COPY build.sh /src/
+COPY build.sh zlib_uncompress_fuzzer.cc /src/
--- a/zlib/build.sh
+++ b/zlib/build.sh
@ -6,5 +6,5 @@ cd /src/zlib
 make clean all

 $CXX $CXXFLAGS -std=c++11 -I. \
-    /src/oss-fuzz/zlib/zlib_uncompress_fuzzer.cc -o /out/zlib_uncompress_fuzzer \
+    /src/zlib_uncompress_fuzzer.cc -o /out/zlib_uncompress_fuzzer \
    /work/libfuzzer/*.o ./libz.a $LDFLAGS