From dae20129807c9a15b9a59eb20431b98b84202714 Mon Sep 17 00:00:00 2001 From: Mike Aizatsky Date: Wed, 12 Oct 2016 16:25:06 -0700 Subject: [PATCH] [infra] do not checkout oss-fuzz (#23) Promising oss-fuzz in /src/oss-fuzz creates lots of confusion about where files come from. Let's make everything explicit. Fixes #20 --- docs/building_running_fuzzers_external.md | 2 +- docs/new_library.md | 46 ++++++++++++--------- expat/Dockerfile | 2 +- expat/README.md | 2 +- expat/build.sh | 4 +- infra/base-images/base-libfuzzer/Dockerfile | 2 - infra/libfuzzer-pipeline.groovy | 11 +---- libchewing/Dockerfile | 2 +- libchewing/build.sh | 2 +- libpng/Dockerfile | 2 +- libpng/build.sh | 4 +- libxml2/Dockerfile | 3 ++ libxml2/build.sh | 4 +- nss/Dockerfile | 2 +- nss/build.sh | 2 +- re2/Dockerfile | 2 +- re2/build.sh | 3 +- re2/{re2.options => re2_fuzzer.options} | 0 scripts/helper.py | 1 - sqlite3/Dockerfile | 2 +- sqlite3/build.sh | 4 +- woff2/Dockerfile | 2 +- woff2/build.sh | 4 +- zlib/Dockerfile | 2 +- zlib/build.sh | 2 +- 25 files changed, 60 insertions(+), 52 deletions(-) rename re2/{re2.options => re2_fuzzer.options} (100%) diff --git a/docs/building_running_fuzzers_external.md b/docs/building_running_fuzzers_external.md index 27639dd14..c9c3af762 100644 --- a/docs/building_running_fuzzers_external.md +++ b/docs/building_running_fuzzers_external.md @@ -29,7 +29,7 @@ docker build -t ossfuzz/$PROJECT_NAME oss-fuzz/$PROJECT_NAME ```` 2. Running a container: ````bash -docker run -ti -v $PWD/$PROJECT_NAME:/src/$PROJECT_NAME -v $PWD/oss-fuzz:/src/oss-fuzz -v /tmp/out:/out ossfuzz/$PROJECT_NAME +docker run -ti -v $PWD/$PROJECT_NAME:/src/$PROJECT_NAME -v /tmp/out:/out ossfuzz/$PROJECT_NAME ```` `/tmp/out` will contain fuzzers. diff --git a/docs/new_library.md b/docs/new_library.md index d6a4bfd26..9cd438bb3 100644 --- a/docs/new_library.md +++ b/docs/new_library.md @@ -43,14 +43,38 @@ Create a fuzzer and add it to the *library_name/* directory as well. This is the Docker image definition that build.sh will be executed in. It is very simple for most libraries: -```bash +```docker FROM ossfuzz/base-libfuzzer # base image with clang toolchain -MAINTAINER YOUR_EMAIL # each file should have a maintainer +MAINTAINER YOUR_EMAIL # each file should have a maintainer RUN apt-get install -y ... # install required packages to build a project COPY build.sh /src/ # install build script for the project. ``` Expat example: [expat/Dockerfile](../expat/Dockerfile) +## Create Fuzzer Source File + +Create a new .cc file, define a `LLVMFuzzerTestOneInput` function and call +your library: + +```c++ +#include +#include + +extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { + // put your fuzzing code here and use data+size as input. + return 0; +} +``` + +Make sure you add the file to your Docker image: +```docker +COPY build.sh my_fuzzer.cc /src/ # install build script & fuzzer. +``` + +There are [lots](../libxml2/libxml2_xml_read_memory_fuzzer.cc) +[of](../expat/parse_fuzzer.cc) [examples](../zlib/zlib_uncompress_fuzzer.cc) +in this project repository. + ## build.sh This is where most of the work is done to build fuzzers for your library. The script will @@ -105,24 +129,6 @@ These flags are provided in following environment variables: Many well-crafted build scripts will automatically use these variables. If not, passing them manually to a build tool might be required. -## Create Fuzzer Source File - -Create a new .cc file, define a `LLVMFuzzerTestOneInput` function and call -your library: - -```c++ -#include -#include - -extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { - // put your fuzzing code here and use data+size as input. - return 0; -} -``` - -There are [lots](../libxml2/libxml2_xml_read_memory_fuzzer.cc) -[of](../expat/parse_fuzzer.cc) [examples](../zlib/zlib_uncompress_fuzzer.cc) -in this project repository. ### Dictionaries and custom libfuzzer options diff --git a/expat/Dockerfile b/expat/Dockerfile index d60d4842b..acfe42670 100644 --- a/expat/Dockerfile +++ b/expat/Dockerfile @@ -21,4 +21,4 @@ RUN apt-get install -y make autoconf automake libtool docbook2x ENV GIT_CHECKOUT_DIR="expat" ENV GIT_URL="git://git.code.sf.net/p/expat/code_git" -COPY build.sh /src/ +COPY build.sh parse_fuzzer.* xml.dict /src/ diff --git a/expat/README.md b/expat/README.md index a1eaeb93d..cc561c339 100644 --- a/expat/README.md +++ b/expat/README.md @@ -10,7 +10,7 @@ git clone https://github.com/google/oss-fuzz.git git clone git://git.code.sf.net/p/expat/code_git expat # Build & run the image. docker build -t ossfuzz/expat oss-fizz/expat && \ -docker run -i -v $PWD/oss-fuzz:/src/oss-fuzz -v $PWD/expat:/src/expat -v $HOME/tmp/out:/out -t ossfuzz/expat +docker run -i -v $PWD/expat:/src/expat -v $HOME/tmp/out:/out -t ossfuzz/expat ```` Fuzzers will be in `$HOME/tmp/out`. diff --git a/expat/build.sh b/expat/build.sh index 922a35716..042c6ea66 100755 --- a/expat/build.sh +++ b/expat/build.sh @@ -6,5 +6,7 @@ cd /src/expat/expat make clean all $CXX $CXXFLAGS -std=c++11 -Ilib/ \ - /src/oss-fuzz/expat/parse_fuzzer.cc -o /out/expat_parse_fuzzer \ + /src/parse_fuzzer.cc -o /out/expat_parse_fuzzer \ /work/libfuzzer/*.o .libs/libexpat.a $LDFLAGS + +cp /src/*.dict /src/*.options /out/ \ No newline at end of file diff --git a/infra/base-images/base-libfuzzer/Dockerfile b/infra/base-images/base-libfuzzer/Dockerfile index 1d26e6e3a..bf41b3882 100644 --- a/infra/base-images/base-libfuzzer/Dockerfile +++ b/infra/base-images/base-libfuzzer/Dockerfile @@ -18,8 +18,6 @@ FROM ossfuzz/base-clang MAINTAINER mike.aizatsky@gmail.com RUN apt-get install -y git libc6-dev -RUN cd /src && git clone --depth 1 https://github.com/google/oss-fuzz.git - RUN mkdir -p /work/libfuzzer ENV SANITIZER_FLAGS="-fsanitize=address" diff --git a/infra/libfuzzer-pipeline.groovy b/infra/libfuzzer-pipeline.groovy index 0a2492ffa..e352b8d04 100644 --- a/infra/libfuzzer-pipeline.groovy +++ b/infra/libfuzzer-pipeline.groovy @@ -34,7 +34,6 @@ def call(body) { def date = java.time.format.DateTimeFormatter.ofPattern("yyyyMMddHHmm") .format(java.time.LocalDateTime.now()) - def ossFuzzUrl = 'https://github.com/google/oss-fuzz.git' node { def workspace = pwd() @@ -44,10 +43,6 @@ def call(body) { stage("docker image") { def revisions = [:] - dir('oss-fuzz') { - git url: ossFuzzUrl - } - dir(checkoutDir) { git url: gitUrl revisions[gitUrl] = sh(returnStdout: true, script: 'git rev-parse HEAD').trim() @@ -74,11 +69,7 @@ def call(body) { // Run image to produce fuzzers sh "rm -rf $out" sh "mkdir -p $out" - sh "docker run -v $workspace/$checkoutDir:/src/$checkoutDir -v $workspace/oss-fuzz:/src/oss-fuzz -v $out:/out -e SANITIZER_FLAGS=\"-fsanitize=$sanitizer\" -t $dockerTag" - - // Copy dict and options files - sh "cp $workspace/oss-fuzz/$projectName/*.dict $out/ || true" - sh "cp $workspace/oss-fuzz/$projectName/*.options $out/ || true" + sh "docker run -v $workspace/$checkoutDir:/src/$checkoutDir -v $out:/out -e SANITIZER_FLAGS=\"-fsanitize=$sanitizer\" -t $dockerTag" } } } diff --git a/libchewing/Dockerfile b/libchewing/Dockerfile index 9794dac93..bc52b667b 100644 --- a/libchewing/Dockerfile +++ b/libchewing/Dockerfile @@ -18,4 +18,4 @@ FROM ossfuzz/base-libfuzzer MAINTAINER kcwu@csie.org RUN apt-get install -y make autoconf automake libtool texinfo -COPY build.sh /src/ +COPY build.sh chewing_fuzzer.c /src/ diff --git a/libchewing/build.sh b/libchewing/build.sh index b046dd2bd..d7c029ea9 100755 --- a/libchewing/build.sh +++ b/libchewing/build.sh @@ -27,7 +27,7 @@ make -C test CFLAGS="$CFLAGS -Dmain=stress_main -Drand=get_fuzz_input" stress.o $CC $CFLAGS \ -o /out/chewing_fuzzer \ - /src/oss-fuzz/libchewing/chewing_fuzzer.c \ + /src/chewing_fuzzer.c \ test/stress.o test/.libs/libtesthelper.a src/.libs/libchewing.a \ /work/libfuzzer/*.o $LDFLAGS diff --git a/libpng/Dockerfile b/libpng/Dockerfile index 6e168612f..2d4e37df6 100644 --- a/libpng/Dockerfile +++ b/libpng/Dockerfile @@ -18,4 +18,4 @@ FROM ossfuzz/base-libfuzzer MAINTAINER mmoroz@chromium.org RUN apt-get install -y make autoconf automake libtool zlib1g-dev -COPY build.sh /src/ +COPY build.sh libpng_read_fuzzer.* png.dict /src/ diff --git a/libpng/build.sh b/libpng/build.sh index cf281c4a8..19d15eb5a 100755 --- a/libpng/build.sh +++ b/libpng/build.sh @@ -29,5 +29,7 @@ make clean all # build libpng_read_fuzzer $CXX $CXXFLAGS -std=c++11 -I. -lz \ - /src/oss-fuzz/libpng/libpng_read_fuzzer.cc -o /out/libpng_read_fuzzer \ + /src/libpng_read_fuzzer.cc -o /out/libpng_read_fuzzer \ /work/libfuzzer/*.o .libs/libpng16.a $LDFLAGS + +cp /src/*.dict /src/*.options /out/ \ No newline at end of file diff --git a/libxml2/Dockerfile b/libxml2/Dockerfile index 3eb50f0f5..a1aef69ae 100644 --- a/libxml2/Dockerfile +++ b/libxml2/Dockerfile @@ -19,3 +19,6 @@ MAINTAINER ochang@chromium.org RUN apt-get install -y make autoconf automake libtool pkg-config COPY build.sh /src/ +COPY libxml2_xml_read_memory_fuzzer.* \ + libxml2_xml_regexp_compile_fuzzer.* \ + xml.dict /src/ diff --git a/libxml2/build.sh b/libxml2/build.sh index e1edbc216..1643543b6 100755 --- a/libxml2/build.sh +++ b/libxml2/build.sh @@ -23,6 +23,8 @@ make clean all for fuzzer in libxml2_xml_read_memory_fuzzer libxml2_xml_regexp_compile_fuzzer; do $CXX $CXXFLAGS -std=c++11 -Iinclude/ \ - /src/oss-fuzz/libxml2/$fuzzer.cc -o /out/$fuzzer \ + /src/$fuzzer.cc -o /out/$fuzzer \ /work/libfuzzer/*.o .libs/libxml2.a $LDFLAGS done + +cp /src/*.dict /src/*.options /out/ \ No newline at end of file diff --git a/nss/Dockerfile b/nss/Dockerfile index 5cf55e00a..1fd6b0d90 100644 --- a/nss/Dockerfile +++ b/nss/Dockerfile @@ -18,6 +18,6 @@ FROM ossfuzz/base-libfuzzer MAINTAINER mmoroz@chromium.org RUN apt-get install -y make autoconf automake libtool mercurial zlib1g-dev -COPY build.sh /src/ +COPY build.sh fuzzers /src/ ENV LD_LIBRARY_PATH "$LD_LIBRARY_PATH:/out" diff --git a/nss/build.sh b/nss/build.sh index d20764cae..9cc1a1a00 100755 --- a/nss/build.sh +++ b/nss/build.sh @@ -57,7 +57,7 @@ FUZZERS="asn1_algorithmid_fuzzer \ for fuzzer in $FUZZERS; do - $CXX $CXXFLAGS -std=c++11 /src/oss-fuzz/nss/fuzzers/$fuzzer.cc \ + $CXX $CXXFLAGS -std=c++11 /src/fuzzers/$fuzzer.cc \ -I/work/nss/include \ /work/libfuzzer/*.o \ /work/nss/lib/libnss.a /work/nss/lib/libnssutil.a \ diff --git a/re2/Dockerfile b/re2/Dockerfile index aec23fb91..492b40d43 100644 --- a/re2/Dockerfile +++ b/re2/Dockerfile @@ -18,4 +18,4 @@ FROM ossfuzz/base-libfuzzer MAINTAINER wrengr@chromium.org RUN apt-get install -y make autoconf automake libtool -COPY build.sh /src/ +COPY build.sh re2_fuzzer.* /src/ \ No newline at end of file diff --git a/re2/build.sh b/re2/build.sh index 27bc60799..1a3569d8c 100755 --- a/re2/build.sh +++ b/re2/build.sh @@ -31,6 +31,7 @@ make obj/libre2.a # Second, build our fuzzers. $CXX $CXXFLAGS -std=c++11 -I. \ - /src/oss-fuzz/re2/re2_fuzzer.cc -o /out/re2_fuzzer \ + /src/re2_fuzzer.cc -o /out/re2_fuzzer \ /work/libfuzzer/*.o ./obj/libre2.a $LDFLAGS +cp /src/*.options /src/*.dict /out/\ \ No newline at end of file diff --git a/re2/re2.options b/re2/re2_fuzzer.options similarity index 100% rename from re2/re2.options rename to re2/re2_fuzzer.options diff --git a/scripts/helper.py b/scripts/helper.py index 9f39ba16c..5721cd5e4 100644 --- a/scripts/helper.py +++ b/scripts/helper.py @@ -272,7 +272,6 @@ def shell(shell_args): command = [ 'docker', 'run', '-i', - '-v', '%s:/src/oss-fuzz' % OSSFUZZ_DIR, '-v', '%s:/src/%s' % (checkout_dir, args.library_name), '-v', '%s:/out' % os.path.join(BUILD_DIR, 'out', args.library_name), '-t', 'ossfuzz/' + args.library_name, diff --git a/sqlite3/Dockerfile b/sqlite3/Dockerfile index b034e8a15..b4ec3a991 100644 --- a/sqlite3/Dockerfile +++ b/sqlite3/Dockerfile @@ -18,4 +18,4 @@ FROM ossfuzz/base-libfuzzer MAINTAINER tanin@google.com RUN apt-get install -y make autoconf automake libtool fossil tcl -COPY build.sh /src/ \ No newline at end of file +COPY build.sh sqlite3_fuzzer.* sql.dict /src/ \ No newline at end of file diff --git a/sqlite3/build.sh b/sqlite3/build.sh index 808711566..d1a7b9707 100755 --- a/sqlite3/build.sh +++ b/sqlite3/build.sh @@ -33,5 +33,7 @@ make make sqlite3.c $CXX $CXXFLAGS -std=c++11 -I. \ - /src/oss-fuzz/sqlite3/sqlite3_fuzzer.cc -o /out/sqlite3_fuzzer \ + /src/sqlite3_fuzzer.cc -o /out/sqlite3_fuzzer \ /work/libfuzzer/*.o ./sqlite3.o $LDFLAGS + +cp /src/*.options /src/*.dict /out/ \ No newline at end of file diff --git a/woff2/Dockerfile b/woff2/Dockerfile index 24a2b0805..56f1a95f6 100644 --- a/woff2/Dockerfile +++ b/woff2/Dockerfile @@ -18,4 +18,4 @@ FROM ossfuzz/base-libfuzzer MAINTAINER mmoroz@chromium.org RUN apt-get install -y make autoconf automake libtool -COPY build.sh /src/ +COPY build.sh convert_woff2ttf_fuzzer.* /src/ diff --git a/woff2/build.sh b/woff2/build.sh index 28cc2fe0a..6440766d2 100755 --- a/woff2/build.sh +++ b/woff2/build.sh @@ -41,5 +41,7 @@ rm src/woff2_compress.o src/woff2_decompress.o # Build the fuzzer. fuzzer=convert_woff2ttf_fuzzer $CXX $CXXFLAGS -std=c++11 -Isrc \ - /src/oss-fuzz/woff2/$fuzzer.cc -o /out/$fuzzer \ + /src/$fuzzer.cc -o /out/$fuzzer \ /work/libfuzzer/*.o src/*.o brotli/dec/*.o brotli/enc/*.o $LDFLAGS + +cp /src/*.options /out/ diff --git a/zlib/Dockerfile b/zlib/Dockerfile index 439f2fd5e..3cdfdbf80 100644 --- a/zlib/Dockerfile +++ b/zlib/Dockerfile @@ -18,4 +18,4 @@ FROM ossfuzz/base-libfuzzer MAINTAINER inferno@chromium.org RUN apt-get install -y make autoconf automake libtool -COPY build.sh /src/ \ No newline at end of file +COPY build.sh zlib_uncompress_fuzzer.cc /src/ \ No newline at end of file diff --git a/zlib/build.sh b/zlib/build.sh index c7013a2dd..caa809106 100755 --- a/zlib/build.sh +++ b/zlib/build.sh @@ -6,5 +6,5 @@ cd /src/zlib make clean all $CXX $CXXFLAGS -std=c++11 -I. \ - /src/oss-fuzz/zlib/zlib_uncompress_fuzzer.cc -o /out/zlib_uncompress_fuzzer \ + /src/zlib_uncompress_fuzzer.cc -o /out/zlib_uncompress_fuzzer \ /work/libfuzzer/*.o ./libz.a $LDFLAGS