diff --git a/boringssl/Dockerfile b/boringssl/Dockerfile index ac3e7c512..d54c5fa1e 100644 --- a/boringssl/Dockerfile +++ b/boringssl/Dockerfile @@ -18,6 +18,5 @@ FROM ossfuzz/base-libfuzzer MAINTAINER mike.aizatsky@gmail.com RUN apt-get install -y cmake ninja-build golang -ENV GIT_URL "https://boringssl.googlesource.com/boringssl" - +RUN git clone https://boringssl.googlesource.com/boringssl COPY build.sh /src/ diff --git a/curl/Dockerfile b/curl/Dockerfile index cb9ebd57a..3ea8e9f53 100644 --- a/curl/Dockerfile +++ b/curl/Dockerfile @@ -18,6 +18,6 @@ FROM ossfuzz/base-libfuzzer MAINTAINER dvyukov@google.com RUN apt-get install -y make autoconf automake libtool libssl-dev zlib1g-dev +RUN git clone https://github.com/curl/curl.git COPY build.sh fuzzer.cc /src/ -ENV GIT_URL "" diff --git a/docs/new_library.md b/docs/new_library.md index 5815d9366..6e636e83c 100644 --- a/docs/new_library.md +++ b/docs/new_library.md @@ -26,7 +26,8 @@ general, check out [this page](http://llvm.org/docs/LibFuzzer.html). To add a new OSS library to oss-fuzz, 3 supporting files have to be added to oss-fuzz source code repository: -* `library_name/Dockerfile` - defines an container environment with all the dependencies needed to build the project and the fuzzer. +* `library_name/Dockerfile` - defines an container environment with all the dependencies +needed to build the project and the fuzzer. * `library_name/build.sh` - build script that will be executed inside the container. * `library_name/Jenkinsfile` - will be needed to integrate fuzzers with ClusterFuzz build and distributed execution system. Specify your library VCS location in it. @@ -49,6 +50,7 @@ It is very simple for most libraries: FROM ossfuzz/base-libfuzzer # base image with clang toolchain MAINTAINER YOUR_EMAIL # each file should have a maintainer RUN apt-get install -y ... # install required packages to build a project +RUN git checkout # checkout all sources needed to build your library COPY build.sh /src/ # install build script and other files. ``` Expat example: [expat/Dockerfile](../expat/Dockerfile) @@ -111,10 +113,9 @@ $CXX $CXXFLAGS -std=c++11 -Ilib/ \ When build.sh script is executed, the following locations are available within the image: -| Path | Description -| ------ | ----- -| `/src/$LIB_NAME` | Source code for your library. -| `/src/oss-fuzz` | Checked out oss-fuzz source tree. +| Path | Description +| ------ | ----- +| `/src/` | Source code needed to build your library. | `/usr/lib/libfuzzer.a` | Prebuilt libFuzzer library that need to be linked into all fuzzers (`-lfuzzer`). You *must* use special compiler flags to build your library and fuzzers. diff --git a/expat/Dockerfile b/expat/Dockerfile index acfe42670..63c88698c 100644 --- a/expat/Dockerfile +++ b/expat/Dockerfile @@ -18,7 +18,5 @@ FROM ossfuzz/base-libfuzzer MAINTAINER mike.aizatsky@gmail.com RUN apt-get install -y make autoconf automake libtool docbook2x -ENV GIT_CHECKOUT_DIR="expat" -ENV GIT_URL="git://git.code.sf.net/p/expat/code_git" - +RUN git clone git://git.code.sf.net/p/expat/code_git expat COPY build.sh parse_fuzzer.* xml.dict /src/ diff --git a/freetype2/Dockerfile b/freetype2/Dockerfile index e0cefab1f..8e9e2ec2e 100644 --- a/freetype2/Dockerfile +++ b/freetype2/Dockerfile @@ -18,6 +18,5 @@ FROM ossfuzz/base-libfuzzer MAINTAINER mike.aizatsky@gmail.com RUN apt-get install -y make autoconf libtool libarchive-dev -ENV GIT_URL="git://git.sv.nongnu.org/freetype/freetype2.git" - +RUN git clone git://git.sv.nongnu.org/freetype/freetype2.git COPY build.sh /src/ diff --git a/harfbuzz/Dockerfile b/harfbuzz/Dockerfile index 9179d7646..5392a96ff 100644 --- a/harfbuzz/Dockerfile +++ b/harfbuzz/Dockerfile @@ -18,6 +18,5 @@ FROM ossfuzz/base-libfuzzer MAINTAINER mmoroz@chromium.org RUN apt-get install -y make autoconf automake libtool ragel pkg-config -ENV GIT_URL="https://anongit.freedesktop.org/git/harfbuzz.git" - +RUN git clone https://anongit.freedesktop.org/git/harfbuzz.git COPY build.sh harfbuzz_fuzzer.cc /src/ diff --git a/infra/base-images/base-libfuzzer/Dockerfile b/infra/base-images/base-libfuzzer/Dockerfile index 25a8e42eb..9006b66c4 100644 --- a/infra/base-images/base-libfuzzer/Dockerfile +++ b/infra/base-images/base-libfuzzer/Dockerfile @@ -26,11 +26,12 @@ ENV COV_FLAGS="-fsanitize-coverage=edge,indirect-calls,8bit-counters" ENV ASAN_OPTIONS="symbolize=1:detect_leaks=0" ENV FUZZER_LDFLAGS "-Wl,-whole-archive /usr/local/lib/libc++.a /usr/local/lib/libc++abi.a -Wl,-no-whole-archive" - RUN mkdir /out VOLUME /out RUN mkdir /src/bin -COPY compile checkout_and_compile run /src/bin/ +COPY compile run /src/bin/ ENV PATH=/src/bin:$PATH +WORKDIR /src CMD ["compile"] + diff --git a/infra/base-images/base-libfuzzer/README.md b/infra/base-images/base-libfuzzer/README.md index 7ac316e79..086d03ce9 100644 --- a/infra/base-images/base-libfuzzer/README.md +++ b/infra/base-images/base-libfuzzer/README.md @@ -3,21 +3,19 @@ Supported commands: -* `docker run -ti [compile]` - compiles everything. Expects /src/ paths - to be mounted. -* `docker run -ti checkout_and_compile` - checks projects sources out - if its location is defined and compiles. +* `docker run -ti [compile]` - builds fuzzers. * `docker run -ti run ` - build fuzzers and start specified one with given options. * `docker run -ti /bin/bash` - drop into shell. Run `compile` script - to start build. + to start build. # Image Files Layout | Location | Description | | -------- | ---------- | -| `/out/` | build artifacts should be copied here | -| `/work/` | used to store intermediate files | +| `/out/` | build artifacts should be copied here | +| `/src/` | place to checkout source files | +| `/work/` | used to store intermediate files | | `/usr/lib/libfuzzer.a` | libfuzzer static library | # Provided Environment Variables @@ -38,22 +36,17 @@ passing them manually to a build tool might be required. # Child Image Interface -## Required Files +## Sources + +Child image has to checkout all sources it needs to compile fuzzers into +`/src/` directory. When the image is executed, a directory could be mounted +on top of these with local checkouts using +`docker run -v $HOME/my_library:/src/my_library ...`. + +## Other Required Files Following files have to be added by child images: -| File Location | Description | -| ------------- | ----------- | -| `/src/build.sh` | build script to build the library and its fuzzers | - -## Optional Environment Variables - -Child image can define following environment variables: - -| Variable | Description | -| -------- | ----------- | -| `GIT_URL` (optional) | git url for sources | -| `SVN_URL` (optional) | svn url for sources | -| `GIT_CHECKOUT_DIR` (optional) | directory (under `/src/`) to checkout into | -| `SVN_CHECKOUT_DIR` (optional) | directory (under `/src/`) to checkout into | - +| File Location | Description | +| ------------- | ----------- | +| `/src/build.sh` | build script to build the library and its fuzzers | \ No newline at end of file diff --git a/infra/base-images/base-libfuzzer/checkout_and_compile b/infra/base-images/base-libfuzzer/checkout_and_compile deleted file mode 100755 index b829d94d3..000000000 --- a/infra/base-images/base-libfuzzer/checkout_and_compile +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/bash -eux -# Copyright 2016 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -################################################################################ - -cd /src - -if [[ ! -z "${GIT_URL-}" ]]; then - git clone --recursive $GIT_URL ${GIT_CHECKOUT_DIR-} -elif [[ ! -z "${SVN_URL-}" ]]; then - svn co $SVN_URL ${SVN_CHECKOUT_DIR-} -else - echo "ERROR: VCS URL Not Defined. Expected ($GET_URL|$SVN_URL)." - exit 1 -fi - -compile diff --git a/infra/base-images/base-libfuzzer/compile b/infra/base-images/base-libfuzzer/compile index bbf8a0f6d..1cc3f6760 100755 --- a/infra/base-images/base-libfuzzer/compile +++ b/infra/base-images/base-libfuzzer/compile @@ -15,7 +15,6 @@ # ################################################################################ - echo "---------------------------------------------------------------" @@ -37,5 +36,6 @@ echo "FUZZER_LDFLAGS=$FUZZER_LDFLAGS" echo "---------------------------------------------------------------" -/src/build.sh +cd /src +/bin/bash /src/build.sh diff --git a/infra/base-images/base-libfuzzer/run b/infra/base-images/base-libfuzzer/run index dbcd356dc..aa09f4504 100755 --- a/infra/base-images/base-libfuzzer/run +++ b/infra/base-images/base-libfuzzer/run @@ -15,7 +15,7 @@ # ################################################################################ -checkout_and_compile +compile export PATH=/out:$PATH cd /out $@ diff --git a/json/Dockerfile b/json/Dockerfile index ef9f9fcf3..81a90ce35 100644 --- a/json/Dockerfile +++ b/json/Dockerfile @@ -18,4 +18,5 @@ FROM ossfuzz/base-libfuzzer MAINTAINER vitalybuka@gmail.com RUN apt-get install -y binutils gcc +RUN git clone https://github.com/nlohmann/json.git COPY build.sh parse_fuzzer.* /src/ diff --git a/libass/Dockerfile b/libass/Dockerfile index 25b04be6f..94ff335e6 100644 --- a/libass/Dockerfile +++ b/libass/Dockerfile @@ -18,6 +18,7 @@ FROM ossfuzz/base-libfuzzer MAINTAINER eugeni.stepanov@gmail.com RUN apt-get install -y make autoconf automake libtool pkg-config libfreetype6-dev libfontconfig1-dev -ENV GIT_URL="https://github.com/libass/libass.git" +RUN git clone https://github.com/libass/libass.git +RUN git clone https://github.com/behdad/fribidi.git COPY build.sh libass_fuzzer.cc *.dict *.options /src/ diff --git a/libass/build.sh b/libass/build.sh index 3404aa7b2..515484700 100755 --- a/libass/build.sh +++ b/libass/build.sh @@ -17,8 +17,6 @@ cd /src -rm -rf fribidi -git clone https://github.com/behdad/fribidi.git cd fribidi ./bootstrap ./configure --enable-static=yes --enable-shared=no --with-pic=yes diff --git a/libchewing/Dockerfile b/libchewing/Dockerfile index 2e80438ea..389edaadd 100644 --- a/libchewing/Dockerfile +++ b/libchewing/Dockerfile @@ -18,6 +18,5 @@ FROM ossfuzz/base-libfuzzer MAINTAINER kcwu@csie.org RUN apt-get install -y make autoconf automake libtool texinfo -ENV GIT_URL="https://github.com/chewing/libchewing.git" - +RUN git clone https://github.com/chewing/libchewing.git COPY build.sh chewing_fuzzer.c /src/ diff --git a/libpng/Dockerfile b/libpng/Dockerfile index 2d2dac73b..2122ba988 100644 --- a/libpng/Dockerfile +++ b/libpng/Dockerfile @@ -18,6 +18,5 @@ FROM ossfuzz/base-libfuzzer MAINTAINER mmoroz@chromium.org RUN apt-get install -y make autoconf automake libtool zlib1g-dev -ENV GIT_URL="git://git.code.sf.net/p/libpng/code" - +RUN git clone git://git.code.sf.net/p/libpng/code libpng COPY build.sh libpng_read_fuzzer.* png.dict /src/ diff --git a/libxml2/Dockerfile b/libxml2/Dockerfile index 225ce9985..9bf723181 100644 --- a/libxml2/Dockerfile +++ b/libxml2/Dockerfile @@ -18,7 +18,7 @@ FROM ossfuzz/base-libfuzzer MAINTAINER ochang@chromium.org RUN apt-get install -y make autoconf automake libtool pkg-config -ENV GIT_URL="git://git.gnome.org/libxml2" +RUN git clone git://git.gnome.org/libxml2 COPY build.sh /src/ COPY libxml2_xml_read_memory_fuzzer.* \ diff --git a/nss/Dockerfile b/nss/Dockerfile index 5ac8c1446..48edb00e3 100644 --- a/nss/Dockerfile +++ b/nss/Dockerfile @@ -18,4 +18,6 @@ FROM ossfuzz/base-libfuzzer MAINTAINER mmoroz@chromium.org RUN apt-get install -y make autoconf automake libtool mercurial zlib1g-dev +RUN hg clone https://hg.mozilla.org/projects/nspr +RUN hg clone https://hg.mozilla.org/projects/nss COPY build.sh fuzzers/* /src/ diff --git a/nss/build.sh b/nss/build.sh index acdde030c..2a49936be 100755 --- a/nss/build.sh +++ b/nss/build.sh @@ -16,11 +16,6 @@ ################################################################################ # Build the library. -mkdir -p /work/nss -cd /work/nss -hg clone https://hg.mozilla.org/projects/nspr -hg clone https://hg.mozilla.org/projects/nss - cd /work/nss/nss make BUILD_OPT=1 USE_64=1 NSS_DISABLE_GTESTS=1 CC="$CC $CFLAGS" \ CXX="$CXX $CXXFLAGS" LD="$CC $CFLAGS" ZDEFS_FLAG= clean nss_build_all diff --git a/pcre2/Dockerfile b/pcre2/Dockerfile index 4671b4974..180d2e3dd 100644 --- a/pcre2/Dockerfile +++ b/pcre2/Dockerfile @@ -18,4 +18,5 @@ FROM ossfuzz/base-libfuzzer MAINTAINER kcc@google.com RUN apt-get install -y make autoconf automake libtool subversion +RUN svn co svn://vcs.exim.org/pcre2/code/trunk pcre2 COPY build.sh pcre2_fuzzer.cc /src/ diff --git a/pcre2/build.sh b/pcre2/build.sh index e7518d492..4b7783a21 100755 --- a/pcre2/build.sh +++ b/pcre2/build.sh @@ -15,8 +15,6 @@ # ################################################################################ -cd /src/pcre2 -svn co svn://vcs.exim.org/pcre2/code/trunk pcre2 cd pcre2 # build the library. diff --git a/re2/Dockerfile b/re2/Dockerfile index fa6d3710d..5c3cd1241 100644 --- a/re2/Dockerfile +++ b/re2/Dockerfile @@ -18,6 +18,5 @@ FROM ossfuzz/base-libfuzzer MAINTAINER wrengr@chromium.org RUN apt-get install -y make autoconf automake libtool -ENV GIT_URL="https://code.googlesource.com/re2" - +RUN git clone https://code.googlesource.com/re2 COPY build.sh re2_fuzzer.* /src/ \ No newline at end of file diff --git a/scripts/templates.py b/scripts/templates.py index 4c7aa2216..6c26c422e 100755 --- a/scripts/templates.py +++ b/scripts/templates.py @@ -59,7 +59,7 @@ DOCKER_TEMPLATE = """\ FROM ossfuzz/base-libfuzzer MAINTAINER your@email.com RUN apt-get install -y make autoconf automake libtool - +RUN git checkout put git url here COPY build.sh /src/ """ diff --git a/sqlite3/Dockerfile b/sqlite3/Dockerfile index b4ec3a991..dc3474327 100644 --- a/sqlite3/Dockerfile +++ b/sqlite3/Dockerfile @@ -18,4 +18,8 @@ FROM ossfuzz/base-libfuzzer MAINTAINER tanin@google.com RUN apt-get install -y make autoconf automake libtool fossil tcl +RUN mkdir /src/sqlite3 && \ + cd /src/sqlite3 && \ + fossil clone https://www.sqlite.org/src sqlite --user `whoami` && \ + fossil open sqlite COPY build.sh sqlite3_fuzzer.* sql.dict /src/ \ No newline at end of file diff --git a/sqlite3/build.sh b/sqlite3/build.sh index 086f23dcc..b8cb93e12 100755 --- a/sqlite3/build.sh +++ b/sqlite3/build.sh @@ -17,13 +17,6 @@ cd /src/sqlite3 -rm -rf fossil -mkdir fossil -cd fossil - -fossil clone https://www.sqlite.org/src sqlite --user `whoami` -fossil open sqlite - mkdir bld cd bld diff --git a/woff2/Dockerfile b/woff2/Dockerfile index ca4b406ed..00f7e755e 100644 --- a/woff2/Dockerfile +++ b/woff2/Dockerfile @@ -18,6 +18,5 @@ FROM ossfuzz/base-libfuzzer MAINTAINER mmoroz@chromium.org RUN apt-get install -y make autoconf automake libtool -ENV GIT_URL="https://github.com/google/woff2" - +RUN git clone --recursive https://github.com/google/woff2 COPY build.sh convert_woff2ttf_fuzzer.* /src/ diff --git a/woff2/build.sh b/woff2/build.sh index 7d4535061..6b7a2d9a0 100755 --- a/woff2/build.sh +++ b/woff2/build.sh @@ -15,10 +15,7 @@ # ################################################################################ -# Checkout woff2 repository with submodules. -mkdir -p /work/woff2 -cd /work/woff2 -git clone https://github.com/google/woff2 --recursive +cd /src/woff2 # Build the library. Actually there is no 'library' target, so we use .o files. # '-no-canonical-prefixes' flag makes clang crazy. Need to avoid it. @@ -31,7 +28,7 @@ cat Makefile | sed -e "s/-no-canonical-prefixes//" \ > Makefile.temp mv Makefile.temp Makefile -# woff2 uses LFLAGS instead of LDFLAGS. +# woff2 uses LFLAGS instead of LDFLAGS. export LFLAGS=$FUZZER_LDFLAGS make CC="$CC $CFLAGS" CXX="$CXX $CXXFLAGS" clean all diff --git a/zlib/Dockerfile b/zlib/Dockerfile index 27da63057..c6196f463 100644 --- a/zlib/Dockerfile +++ b/zlib/Dockerfile @@ -18,6 +18,5 @@ FROM ossfuzz/base-libfuzzer MAINTAINER inferno@chromium.org RUN apt-get install -y make autoconf automake libtool -ENV GIT_URL="https://github.com/madler/zlib.git" - +RUN git clone https://github.com/madler/zlib.git COPY build.sh zlib_uncompress_fuzzer.cc /src/ \ No newline at end of file