[infra] checking out library sources in the image (#40)

fixes #33

boringssl/Dockerfile

@@ -18,6 +18,5 @@ FROM ossfuzz/base-libfuzzer
 MAINTAINER mike.aizatsky@gmail.com
 RUN apt-get install -y cmake ninja-build golang
 
-ENV GIT_URL "https://boringssl.googlesource.com/boringssl"
-
+RUN git clone https://boringssl.googlesource.com/boringssl
 COPY build.sh /src/

curl/Dockerfile

@@ -18,6 +18,6 @@ FROM ossfuzz/base-libfuzzer
 MAINTAINER dvyukov@google.com
 RUN apt-get install -y make autoconf automake libtool libssl-dev zlib1g-dev
 
+RUN git clone https://github.com/curl/curl.git
 COPY build.sh fuzzer.cc /src/
 
-ENV GIT_URL ""

docs/new_library.md

@@ -26,7 +26,8 @@ general, check out [this page](http://llvm.org/docs/LibFuzzer.html).
 
 To add a new OSS library to oss-fuzz, 3 supporting files have to be added to oss-fuzz source code repository:
 
-* `library_name/Dockerfile` - defines an container environment with all the dependencies needed to build the project and the fuzzer.
+* `library_name/Dockerfile` - defines an container environment with all the dependencies
+needed to build the project and the fuzzer.
 * `library_name/build.sh` - build script that will be executed inside the container.
 * `library_name/Jenkinsfile` - will be needed to integrate fuzzers with ClusterFuzz build and distributed execution system. 
   Specify your library VCS location in it.
@@ -49,6 +50,7 @@ It is very simple for most libraries:
 FROM ossfuzz/base-libfuzzer             # base image with clang toolchain
 MAINTAINER YOUR_EMAIL                   # each file should have a maintainer
 RUN apt-get install -y ...              # install required packages to build a project
+RUN git checkout <git_url>              # checkout all sources needed to build your library
 COPY build.sh <additional_files> /src/  # install build script and other files.
 ```
 Expat example: [expat/Dockerfile](../expat/Dockerfile)
@@ -113,8 +115,7 @@ When build.sh script is executed, the following locations are available within t
 
 | Path                   | Description
 | ------                 | -----
-| `/src/$LIB_NAME`      | Source code for your library.
-| `/src/oss-fuzz`       | Checked out oss-fuzz source tree.
+| `/src/<some_dir>`      | Source code needed to build your library.
 | `/usr/lib/libfuzzer.a` | Prebuilt libFuzzer library that need to be linked into all fuzzers (`-lfuzzer`).
 
 You *must* use special compiler flags to build your library and fuzzers.

expat/Dockerfile

@@ -18,7 +18,5 @@ FROM ossfuzz/base-libfuzzer
 MAINTAINER mike.aizatsky@gmail.com
 RUN apt-get install -y make autoconf automake libtool docbook2x
 
-ENV GIT_CHECKOUT_DIR="expat"
-ENV GIT_URL="git://git.code.sf.net/p/expat/code_git"
-
+RUN git clone git://git.code.sf.net/p/expat/code_git expat 
 COPY build.sh parse_fuzzer.*  xml.dict /src/

freetype2/Dockerfile

@@ -18,6 +18,5 @@ FROM ossfuzz/base-libfuzzer
 MAINTAINER mike.aizatsky@gmail.com
 RUN apt-get install -y make autoconf libtool libarchive-dev
 
-ENV GIT_URL="git://git.sv.nongnu.org/freetype/freetype2.git"
-
+RUN git clone git://git.sv.nongnu.org/freetype/freetype2.git
 COPY build.sh /src/

harfbuzz/Dockerfile

@@ -18,6 +18,5 @@ FROM ossfuzz/base-libfuzzer
 MAINTAINER mmoroz@chromium.org
 RUN apt-get install -y make autoconf automake libtool ragel pkg-config
 
-ENV GIT_URL="https://anongit.freedesktop.org/git/harfbuzz.git"
-
+RUN git clone https://anongit.freedesktop.org/git/harfbuzz.git
 COPY build.sh harfbuzz_fuzzer.cc /src/

infra/base-images/base-libfuzzer/Dockerfile

@@ -26,11 +26,12 @@ ENV COV_FLAGS="-fsanitize-coverage=edge,indirect-calls,8bit-counters"
 ENV ASAN_OPTIONS="symbolize=1:detect_leaks=0"
 ENV FUZZER_LDFLAGS "-Wl,-whole-archive /usr/local/lib/libc++.a /usr/local/lib/libc++abi.a -Wl,-no-whole-archive"
 
-
 RUN mkdir /out
 VOLUME /out
 
 RUN mkdir /src/bin
-COPY compile checkout_and_compile run /src/bin/
+COPY compile run /src/bin/
 ENV PATH=/src/bin:$PATH
+WORKDIR /src
 CMD ["compile"]
+

infra/base-images/base-libfuzzer/README.md

@@ -3,10 +3,7 @@
 
 Supported commands:
 
-* `docker run -ti <image_name> [compile]` - compiles everything. Expects /src/ paths
-  to be mounted.
-* `docker run -ti <image_name> checkout_and_compile` - checks projects sources out 
-  if its location is defined and compiles.
+* `docker run -ti <image_name> [compile]` - builds fuzzers.
 * `docker run -ti <image_name> run <fuzzer_name> <fuzzer_options...>` - build fuzzers and start
   specified one with given options.
 * `docker run -ti <image_name> /bin/bash` - drop into shell. Run `compile` script
@@ -17,6 +14,7 @@ Supported commands:
 | Location | Description |
 | -------- | ----------  |
 | `/out/`                | build artifacts should be copied here  |
+| `/src/`                | place to checkout source files |
 | `/work/`               | used to store intermediate files |
 | `/usr/lib/libfuzzer.a` | libfuzzer static library |
 
@@ -38,22 +36,17 @@ passing them manually to a build tool might be required.
 
 # Child Image Interface
 
-## Required Files
+## Sources
+
+Child image has to checkout all sources it needs to compile fuzzers into
+`/src/` directory. When the image is executed, a directory could be mounted
+on top of these with local checkouts using
+`docker run -v $HOME/my_library:/src/my_library ...`.
+
+## Other Required Files
 
 Following files have to be added by child images:
 
 | File Location   | Description |
 | -------------   | ----------- |
 | `/src/build.sh` | build script to build the library and its fuzzers |
-
-## Optional Environment Variables
-
-Child image can define following environment variables:
-
-| Variable | Description |
-| -------- | ----------- |
-| `GIT_URL` (optional) | git url for sources |
-| `SVN_URL` (optional) | svn url for sources |
-| `GIT_CHECKOUT_DIR` (optional) | directory (under `/src/`) to checkout into |
-| `SVN_CHECKOUT_DIR` (optional) | directory (under `/src/`) to checkout into |
-

infra/base-images/base-libfuzzer/checkout_and_compile

@@ -1,29 +0,0 @@
-#!/bin/bash -eux
-# Copyright 2016 Google Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-################################################################################
-
-cd /src
-
-if [[ ! -z "${GIT_URL-}" ]]; then
-  git clone --recursive $GIT_URL ${GIT_CHECKOUT_DIR-}
-elif [[ ! -z "${SVN_URL-}" ]]; then
-  svn co $SVN_URL ${SVN_CHECKOUT_DIR-}
-else
-  echo "ERROR: VCS URL Not Defined. Expected ($GET_URL|$SVN_URL)."
-  exit 1
-fi
-
-compile

infra/base-images/base-libfuzzer/compile

@@ -15,7 +15,6 @@
 #
 ################################################################################
 
-
 echo "---------------------------------------------------------------"
 
 
@@ -37,5 +36,6 @@ echo "FUZZER_LDFLAGS=$FUZZER_LDFLAGS"
 
 echo "---------------------------------------------------------------"
 
-/src/build.sh
+cd /src
+/bin/bash /src/build.sh
 

infra/base-images/base-libfuzzer/run

@@ -15,7 +15,7 @@
 #
 ################################################################################
 
-checkout_and_compile
+compile
 export PATH=/out:$PATH
 cd /out
 $@

json/Dockerfile

@@ -18,4 +18,5 @@ FROM ossfuzz/base-libfuzzer
 MAINTAINER vitalybuka@gmail.com
 RUN apt-get install -y binutils gcc
 
+RUN git clone https://github.com/nlohmann/json.git
 COPY build.sh parse_fuzzer.* /src/

libass/Dockerfile

@@ -18,6 +18,7 @@ FROM ossfuzz/base-libfuzzer
 MAINTAINER eugeni.stepanov@gmail.com
 RUN apt-get install -y make autoconf automake libtool pkg-config libfreetype6-dev libfontconfig1-dev
 
-ENV GIT_URL="https://github.com/libass/libass.git"
+RUN git clone https://github.com/libass/libass.git
+RUN git clone https://github.com/behdad/fribidi.git
 
 COPY build.sh libass_fuzzer.cc *.dict *.options /src/

libass/build.sh

@@ -17,8 +17,6 @@
 
 cd /src
 
-rm -rf fribidi
-git clone https://github.com/behdad/fribidi.git
 cd fribidi
 ./bootstrap
 ./configure --enable-static=yes --enable-shared=no --with-pic=yes

libchewing/Dockerfile

@@ -18,6 +18,5 @@ FROM ossfuzz/base-libfuzzer
 MAINTAINER kcwu@csie.org
 RUN apt-get install -y make autoconf automake libtool texinfo
 
-ENV GIT_URL="https://github.com/chewing/libchewing.git"
-
+RUN git clone https://github.com/chewing/libchewing.git
 COPY build.sh chewing_fuzzer.c /src/

libpng/Dockerfile

@@ -18,6 +18,5 @@ FROM ossfuzz/base-libfuzzer
 MAINTAINER mmoroz@chromium.org
 RUN apt-get install -y make autoconf automake libtool zlib1g-dev
 
-ENV GIT_URL="git://git.code.sf.net/p/libpng/code"
-
+RUN git clone git://git.code.sf.net/p/libpng/code libpng
 COPY build.sh libpng_read_fuzzer.* png.dict /src/

libxml2/Dockerfile

@@ -18,7 +18,7 @@ FROM ossfuzz/base-libfuzzer
 MAINTAINER ochang@chromium.org
 RUN apt-get install -y make autoconf automake libtool pkg-config
 
-ENV GIT_URL="git://git.gnome.org/libxml2"
+RUN git clone git://git.gnome.org/libxml2
 
 COPY build.sh /src/
 COPY libxml2_xml_read_memory_fuzzer.* \

nss/Dockerfile

@@ -18,4 +18,6 @@ FROM ossfuzz/base-libfuzzer
 MAINTAINER mmoroz@chromium.org
 RUN apt-get install -y make autoconf automake libtool mercurial zlib1g-dev
 
+RUN hg clone https://hg.mozilla.org/projects/nspr
+RUN hg clone https://hg.mozilla.org/projects/nss
 COPY build.sh fuzzers/* /src/

nss/build.sh

@@ -16,11 +16,6 @@
 ################################################################################
 
 # Build the library.
-mkdir -p /work/nss
-cd /work/nss
-hg clone https://hg.mozilla.org/projects/nspr
-hg clone https://hg.mozilla.org/projects/nss
-
 cd /work/nss/nss
 make BUILD_OPT=1 USE_64=1 NSS_DISABLE_GTESTS=1 CC="$CC $CFLAGS" \
     CXX="$CXX $CXXFLAGS" LD="$CC $CFLAGS" ZDEFS_FLAG= clean nss_build_all

pcre2/Dockerfile

@@ -18,4 +18,5 @@ FROM ossfuzz/base-libfuzzer
 MAINTAINER kcc@google.com
 RUN apt-get install -y make autoconf automake libtool subversion
 
+RUN svn co svn://vcs.exim.org/pcre2/code/trunk pcre2
 COPY build.sh pcre2_fuzzer.cc /src/

pcre2/build.sh

@@ -15,8 +15,6 @@
 #
 ################################################################################
 
-cd /src/pcre2
-svn co svn://vcs.exim.org/pcre2/code/trunk pcre2
 cd pcre2
 
 # build the library.

re2/Dockerfile

@@ -18,6 +18,5 @@ FROM ossfuzz/base-libfuzzer
 MAINTAINER wrengr@chromium.org
 RUN apt-get install -y make autoconf automake libtool
 
-ENV GIT_URL="https://code.googlesource.com/re2"
-
+RUN git clone https://code.googlesource.com/re2
 COPY build.sh re2_fuzzer.* /src/

scripts/templates.py

@@ -59,7 +59,7 @@ DOCKER_TEMPLATE = """\
 FROM ossfuzz/base-libfuzzer
 MAINTAINER your@email.com
 RUN apt-get install -y make autoconf automake libtool
-
+RUN git checkout put git url here
 COPY build.sh /src/
 """
 

sqlite3/Dockerfile

@@ -18,4 +18,8 @@ FROM ossfuzz/base-libfuzzer
 MAINTAINER tanin@google.com
 RUN apt-get install -y make autoconf automake libtool fossil tcl
 
+RUN mkdir /src/sqlite3 && \
+    cd /src/sqlite3 && \
+    fossil clone https://www.sqlite.org/src sqlite --user `whoami` && \
+    fossil open sqlite
 COPY build.sh sqlite3_fuzzer.* sql.dict /src/

sqlite3/build.sh

@@ -17,13 +17,6 @@
 
 cd /src/sqlite3
 
-rm -rf fossil
-mkdir fossil
-cd fossil
-
-fossil clone https://www.sqlite.org/src sqlite --user `whoami`
-fossil open sqlite
-
 mkdir bld
 cd bld
 

woff2/Dockerfile

@@ -18,6 +18,5 @@ FROM ossfuzz/base-libfuzzer
 MAINTAINER mmoroz@chromium.org
 RUN apt-get install -y make autoconf automake libtool
 
-ENV GIT_URL="https://github.com/google/woff2"
-
+RUN git clone --recursive https://github.com/google/woff2
 COPY build.sh convert_woff2ttf_fuzzer.* /src/

woff2/build.sh

@@ -15,10 +15,7 @@
 #
 ################################################################################
 
-# Checkout woff2 repository with submodules.
-mkdir -p /work/woff2
-cd /work/woff2
-git clone https://github.com/google/woff2 --recursive
+cd /src/woff2
 
 # Build the library. Actually there is no 'library' target, so we use .o files.
 # '-no-canonical-prefixes' flag makes clang crazy. Need to avoid it.

zlib/Dockerfile

@@ -18,6 +18,5 @@ FROM ossfuzz/base-libfuzzer
 MAINTAINER inferno@chromium.org
 RUN apt-get install -y make autoconf automake libtool
 
-ENV GIT_URL="https://github.com/madler/zlib.git"
-
+RUN git clone https://github.com/madler/zlib.git
 COPY build.sh zlib_uncompress_fuzzer.cc /src/