From a95b41b0b311d559c2f984f3ebee533572303d13 Mon Sep 17 00:00:00 2001 From: Mike Aizatsky Date: Tue, 25 Oct 2016 16:36:29 -0700 Subject: [PATCH] replacing library with target --- README.md | 12 ++--- docs/README_template | 2 +- docs/debugging.md | 2 +- docs/faq.md | 2 +- docs/{new_library.md => new_target.md} | 54 +++++++++---------- infra/base-images/base-libfuzzer/README.md | 6 +-- infra/helper.py | 60 +++++++++++----------- infra/templates.py | 2 +- 8 files changed, 71 insertions(+), 69 deletions(-) rename docs/{new_library.md => new_target.md} (80%) diff --git a/README.md b/README.md index 153f4da71..383374efa 100644 --- a/README.md +++ b/README.md @@ -4,8 +4,8 @@ > *Status*: Beta. We are preparing the project for the first public release. Documentation and smoothing the process is our main priority. -[FAQ](docs/faq.md) | -[New Library Guide](docs/new_library.md) +[FAQ](docs/faq.md) +| [New Target Guide](docs/new_target.md) | [Project List](docs/projects.md) @@ -81,7 +81,7 @@ To submit a new target to oss-fuzz: If this is not you or address differs from VCS, an informal e-mail verification will be required. This e-mail will also be publicly listed in our [Projects](docs/projects.md) page. -- once accepted by an oss-fuzz project member, follow the [New Library Guide](docs/new_library.md) +- once accepted by an oss-fuzz project member, follow the [New Target Guide](docs/new_target.md) to write the code. @@ -89,19 +89,19 @@ To submit a new target to oss-fuzz: Following Google's standard [disclosure policy](https://googleprojectzero.blogspot.com/2015/02/feedback-and-data-driven-updates-to.html) oss-fuzz will adhere to following disclosure principles: - - **90-day deadline**. After notifying library authors, we will open reported + - **90-day deadline**. After notifying target authors, we will open reported issues in 90 days, or sooner if the fix is released. - **Weekends and holidays**. If a deadline is due to expire on a weekend or US public holiday, the deadline will be moved to the next normal work day. - **Grace period**. We will have a 14-day grace period. If a 90-day deadline - will expire but library engineers let us know before the deadline that a + will expire but upstream engineers let us know before the deadline that a patch is scheduled for release on a specific day within 14 days following the deadline, the public disclosure will be delayed until the availability of the patch. ## Documentation -* [New Library Guide](docs/new_library.md) walks through steps necessary to add fuzzers to an open source project. +* [New Target Guide](docs/new_target.md) walks through steps necessary to add new targets to oss-fuzz. * [Running and Building Fuzzers](docs/building_running_fuzzers.md) documents the process for fuzzers that are *part of target project* source code repository. * [Running and Building External Fuzzers](docs/building_running_fuzzers_external.md) documents the process for fuzzers that are diff --git a/docs/README_template b/docs/README_template index 212ff3bea..6ddb3bce8 100644 --- a/docs/README_template +++ b/docs/README_template @@ -1,4 +1,4 @@ -# A template for a README file inside 'fuzz/' directory in a library project. +# A template for a README file inside 'fuzz/' directory in a target project. This directory contains source code and build scripts for coverage-guided fuzzers. diff --git a/docs/debugging.md b/docs/debugging.md index 4c09c3d5b..ab970bb6a 100644 --- a/docs/debugging.md +++ b/docs/debugging.md @@ -4,7 +4,7 @@ While developing your build script, it may be useful to run bash within the container: ```bash -$ python infra/helper.py shell $LIB_NAME # runs /bin/bash within container +$ python infra/helper.py shell $TARGET_NAME # runs /bin/bash within container $ compile # run compilation manually ``` diff --git a/docs/faq.md b/docs/faq.md index 3ecc01b7e..8578fa2db 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -8,7 +8,7 @@ features available. ## Why do you use Docker? -Building fuzzers requires building your library with a fresh Clang compiler and special compiler flags. +Building fuzzers requires building your target with a fresh Clang compiler and special compiler flags. An easy-to-use Docker image is provided to simplify toolchain distribution. This also limits our exposure to multitude of Linux varieties and provides a reproducible and (more) secure environment for fuzzer building and execution. diff --git a/docs/new_library.md b/docs/new_target.md similarity index 80% rename from docs/new_library.md rename to docs/new_target.md index aec5990fc..478fcffae 100644 --- a/docs/new_library.md +++ b/docs/new_target.md @@ -1,8 +1,9 @@ -# Setting up fuzzers for a new library +# Setting up fuzzers for a new target -Fuzzer build configurations are placed into a `targets/` subdirectory for the -library in the [oss-fuzz repo] on GitHub. For example, fuzzers for the expat -library go into . +Fuzzer build configurations are placed into [`targets/`](https://github.com/google/oss-fuzz/tree/master/targets) +subdirectories of the [oss-fuzz repo] on GitHub. +For example, fuzzers for the expat library are located in +[`targets/expat/`](https://github.com/google/oss-fuzz/tree/master/targets/expat). ## Prerequisites @@ -17,23 +18,23 @@ library go into . ## Overview -To add a new OSS library to oss-fuzz, 3 supporting files have to be added to oss-fuzz source code repository: +To add a new OSS target to oss-fuzz, 3 supporting files have to be added to oss-fuzz source code repository: -* `targets/library_name/Dockerfile` - defines an container environment with all the dependencies +* `targets/target_name/Dockerfile` - defines an container environment with all the dependencies needed to build the project and the fuzzer. -* `targets/library_name/build.sh` - build script that will be executed inside the container. -* `targets/library_name/Jenkinsfile` - will be needed to integrate fuzzers with ClusterFuzz build and distributed execution system. - Specify your library VCS location in it. +* `targets/target_name/build.sh` - build script that will be executed inside the container. +* `targets/target_name/Jenkinsfile` - will be needed to integrate fuzzers with ClusterFuzz build and distributed execution system. + Specify your target VCS location in it. -To create a new directory for the library and *automatically generate* these 3 files a python script can be used: +To create a new directory for the target and *automatically generate* these 3 files a python script can be used: ```bash $ cd /path/to/oss-fuzz -$ export LIB_NAME=name_of_the_library -$ python infra/helper.py generate $LIB_NAME +$ export TARGET_NAME=target_name +$ python infra/helper.py generate $TARGET_NAME ``` -Create a fuzzer and add it to the *library_name/* directory as well. +Create a fuzzer and add it to the *target_name/* directory as well. ## Dockerfile @@ -43,7 +44,7 @@ It is very simple for most libraries: FROM ossfuzz/base-libfuzzer # base image with clang toolchain MAINTAINER YOUR_EMAIL # each file should have a maintainer RUN apt-get install -y ... # install required packages to build a project -RUN git checkout # checkout all sources needed to build your library +RUN git checkout # checkout all sources needed to build your target COPY build.sh fuzzer.cc /src/ # install build script and other source files. ``` Expat example: [expat/Dockerfile](../expat/Dockerfile) @@ -51,7 +52,7 @@ Expat example: [expat/Dockerfile](../expat/Dockerfile) ## Create Fuzzer Source File Create a new .cc file, define a `LLVMFuzzerTestOneInput` function and call -your library: +your target: ```c++ #include @@ -74,13 +75,14 @@ in this project repository. ## build.sh -This is where most of the work is done to build fuzzers for your library. The script will +This is where most of the work is done to build fuzzers for your target. The script will be executed within an image built from a `Dockerfile`. In general, this script will need to: -1. Build the library using its build system *with* correct compiler and its flags provided as *environment variables* (see below). -2. Build the fuzzers, linking with the library and libFuzzer. Built fuzzers +1. Build the target using its build system *with* correct compiler and its flags provided as + *environment variables* (see below). +2. Build the fuzzers, linking with the target and libFuzzer. Resulting fuzzers should be placed in `/out`. For expat, this looks like: @@ -108,10 +110,10 @@ When build.sh script is executed, the following locations are available within t | Path | Description | ------ | ----- -| `/src/` | Source code needed to build your library. +| `/src/` | Source code needed to build your target. | `/usr/lib/libfuzzer.a` | Prebuilt libFuzzer library that need to be linked into all fuzzers (`-lfuzzer`). -You *must* use special compiler flags to build your library and fuzzers. +You *must* use special compiler flags to build your target and fuzzers. These flags are provided in following environment variables: | Env Variable | Description @@ -179,7 +181,7 @@ libfuzzerBuild { } ``` -Simply replace the the "git" entry with the correct git url for the library. +Simply replace the the "git" entry with the correct git url for the target. *Note*: only git is supported right now. @@ -190,16 +192,16 @@ version using docker commands directly is documented [here](building_running_fuz ```bash $ cd /path/to/oss-fuzz -$ python infra/helper.py build_image $LIB_NAME -$ python infra/helper.py build_fuzzers $LIB_NAME +$ python infra/helper.py build_image $TARGET_NAME +$ python infra/helper.py build_fuzzers $TARGET_NAME ``` -This should place the built fuzzers into `/path/to/oss-fuzz/build/out/$LIB_NAME` +This should place the built fuzzers into `/path/to/oss-fuzz/build/out/$TARGET_NAME` on your machine (`/out` in the container). You can then try to run these fuzzers inside the container to make sure that they work properly: ```bash -$ python infra/helper.py run_fuzzer $LIB_NAME name_of_a_fuzzer +$ python infra/helper.py run_fuzzer $TARGET_NAME name_of_a_fuzzer ``` If everything works locally, then it should also work on our automated builders @@ -208,7 +210,7 @@ and ClusterFuzz. It's recommended to look at coverage as a sanity check to make sure that fuzzer gets to the code you expect. ```bash -$ python infra/helper.py coverage $LIB_NAME name_of_a_fuzzer +$ python infra/helper.py coverage $TARGET_NAME name_of_a_fuzzer ``` ## Debugging Problems diff --git a/infra/base-images/base-libfuzzer/README.md b/infra/base-images/base-libfuzzer/README.md index 30d07b331..a63bac54e 100644 --- a/infra/base-images/base-libfuzzer/README.md +++ b/infra/base-images/base-libfuzzer/README.md @@ -22,7 +22,7 @@ Supported commands: # Provided Environment Variables -You *must* use special compiler flags to build your library and fuzzers. +You *must* use special compiler flags to build your target and fuzzers. These flags are provided in following environment variables: | Env Variable | Description @@ -43,7 +43,7 @@ passing them manually to a build tool might be required. Child image has to checkout all sources it needs to compile fuzzers into `/src/` directory. When the image is executed, a directory could be mounted on top of these with local checkouts using -`docker run -v $HOME/my_library:/src/my_library ...`. +`docker run -v $HOME/my_target:/src/my_target ...`. ## Other Required Files @@ -51,4 +51,4 @@ Following files have to be added by child images: | File Location | Description | | ------------- | ----------- | -| `/src/build.sh` | build script to build the library and its fuzzers | +| `/src/build.sh` | build script to build the target and its fuzzers | diff --git a/infra/helper.py b/infra/helper.py index d43d5fe43..3182a1579 100755 --- a/infra/helper.py +++ b/infra/helper.py @@ -66,10 +66,10 @@ def main(): return 0 -def _check_library_exists(library_name): - """Checks if a library exists.""" - if not os.path.exists(os.path.join(OSSFUZZ_DIR, "targets", library_name)): - print(library_name, 'does not exist', file=sys.stderr) +def _check_target_exists(target_name): + """Checks if a target exists.""" + if not os.path.exists(os.path.join(OSSFUZZ_DIR, "targets", target_name)): + print(target_name, 'does not exist', file=sys.stderr) return False return True @@ -80,11 +80,11 @@ def _get_command_string(command): return ' '.join(pipes.quote(part) for part in command) -def _get_version_control_url(library_name): - """Returns (url, type) for the library.""" +def _get_version_control_url(target_name): + """Returns (url, type) for the target.""" git_regex = re.compile(r'.*git\s*=\s*"(.*?)"\s*') - with open(os.path.join(OSSFUZZ_DIR, library_name, 'Jenkinsfile')) as f: + with open(os.path.join(OSSFUZZ_DIR, target_name, 'Jenkinsfile')) as f: for line in f: match = git_regex.match(line) if match: @@ -96,15 +96,15 @@ def _get_version_control_url(library_name): def build_image(build_args): """Build docker image.""" parser = argparse.ArgumentParser('helper.py build_image') - parser.add_argument('library_name') + parser.add_argument('target_name') args = parser.parse_args(build_args) - if not _check_library_exists(args.library_name): + if not _check_target_exists(args.target_name): return 1 command = [ - 'docker', 'build', '--pull', '-t', 'ossfuzz/' + args.library_name, - os.path.join("targets", args.library_name) + 'docker', 'build', '--pull', '-t', 'ossfuzz/' + args.target_name, + os.path.join("targets", args.target_name) ] print('Running:', _get_command_string(command)) @@ -120,7 +120,7 @@ def build_image(build_args): def build_fuzzers(build_args): """Build fuzzers.""" parser = argparse.ArgumentParser('helper.py build_fuzzers') - parser.add_argument('library_name') + parser.add_argument('target_name') args = parser.parse_args(build_args) if build_image(build_args): @@ -128,8 +128,8 @@ def build_fuzzers(build_args): command = [ 'docker', 'run', '-i', - '-v', '%s:/out' % os.path.join(BUILD_DIR, 'out', args.library_name), - '-t', 'ossfuzz/' + args.library_name, + '-v', '%s:/out' % os.path.join(BUILD_DIR, 'out', args.target_name), + '-t', 'ossfuzz/' + args.target_name, ] print('Running:', _get_command_string(command)) @@ -146,16 +146,16 @@ def build_fuzzers(build_args): def run_fuzzer(run_args): """Runs a fuzzer in the container.""" parser = argparse.ArgumentParser('helper.py run_fuzzer') - parser.add_argument('library_name', help='name of the library') + parser.add_argument('target_name', help='name of the target') parser.add_argument('fuzzer_name', help='name of the fuzzer') parser.add_argument('fuzzer_args', help='arguments to pass to the fuzzer', nargs=argparse.REMAINDER) args = parser.parse_args(run_args) - if not _check_library_exists(args.library_name): + if not _check_target_exists(args.target_name): return 1 - if not os.path.exists(os.path.join(BUILD_DIR, 'out', args.library_name, + if not os.path.exists(os.path.join(BUILD_DIR, 'out', args.target_name, args.fuzzer_name)): print(args.fuzzer_name, 'does not seem to exist. Please run build_fuzzers first.', @@ -166,7 +166,7 @@ def run_fuzzer(run_args): 'docker', 'run', '-i', '-v', '%s:/out' % os.path.join(BUILD_DIR, 'out'), '-t', 'ossfuzz/libfuzzer-runner', - '/out/%s/%s' %(args.library_name, args.fuzzer_name) + '/out/%s/%s' %(args.target_name, args.fuzzer_name) ] + args.fuzzer_args print('Running:', _get_command_string(command)) @@ -177,16 +177,16 @@ def coverage(run_args): """Runs a fuzzer in the container.""" parser = argparse.ArgumentParser('helper.py coverage') parser.add_argument('--run_time', default=60, help='time in seconds to run fuzzer') - parser.add_argument('library_name', help='name of the library') + parser.add_argument('target_name', help='name of the target') parser.add_argument('fuzzer_name', help='name of the fuzzer') parser.add_argument('fuzzer_args', help='arguments to pass to the fuzzer', nargs=argparse.REMAINDER) args = parser.parse_args(run_args) - if not _check_library_exists(args.library_name): + if not _check_target_exists(args.target_name): return 1 - if not os.path.exists(os.path.join(BUILD_DIR, 'out', args.library_name, + if not os.path.exists(os.path.join(BUILD_DIR, 'out', args.target_name, args.fuzzer_name)): print(args.fuzzer_name, 'does not seem to exist. Please run build_fuzzers first.', @@ -203,7 +203,7 @@ def coverage(run_args): '-w', '/cov', '-e', 'ASAN_OPTIONS=coverage=1,detect_leaks=0', '-t', 'ossfuzz/libfuzzer-runner', - '/out/%s/%s' % (args.library_name, args.fuzzer_name), + '/out/%s/%s' % (args.target_name, args.fuzzer_name), '-max_total_time=%s' % args.run_time ] + args.fuzzer_args @@ -213,7 +213,7 @@ def coverage(run_args): command = [ 'docker', 'run', '-i', - '-v', '%s:/out' % os.path.join(BUILD_DIR, 'out', args.library_name), + '-v', '%s:/out' % os.path.join(BUILD_DIR, 'out', args.target_name), '-v', '%s:/cov' % temp_dir, '-v', '%s:/scripts' % os.path.join(OSSFUZZ_DIR, 'scripts'), '-w', '/cov', @@ -228,11 +228,11 @@ def coverage(run_args): def generate(generate_args): - """Generate empty library files.""" + """Generate empty target files.""" parser = argparse.ArgumentParser('helper.py generate') - parser.add_argument('library_name') + parser.add_argument('target_name') args = parser.parse_args(generate_args) - dir = os.path.join("targets", args.library_name) + dir = os.path.join("targets", args.target_name) try: os.mkdir(dir) @@ -248,7 +248,7 @@ def generate(generate_args): build_sh_path = os.path.join(dir, 'build.sh') with open(build_sh_path, 'w') as f: - f.write(templates.BUILD_TEMPLATE % args.library_name) + f.write(templates.BUILD_TEMPLATE % args.target_name) os.chmod(build_sh_path, 0755) return 0 @@ -257,7 +257,7 @@ def generate(generate_args): def shell(shell_args): """Runs a shell within a docker image.""" parser = argparse.ArgumentParser('helper.py shell') - parser.add_argument('library_name', help='name of the library') + parser.add_argument('target_name', help='name of the target') args = parser.parse_args(shell_args) if build_image(shell_args): @@ -265,8 +265,8 @@ def shell(shell_args): command = [ 'docker', 'run', '-i', - '-v', '%s:/out' % os.path.join(BUILD_DIR, 'out', args.library_name), - '-t', 'ossfuzz/' + args.library_name, + '-v', '%s:/out' % os.path.join(BUILD_DIR, 'out', args.target_name), + '-t', 'ossfuzz/' + args.target_name, '/bin/bash' ] print('Running:', _get_command_string(command)) diff --git a/infra/templates.py b/infra/templates.py index 6c26c422e..3fa7a6d71 100755 --- a/infra/templates.py +++ b/infra/templates.py @@ -83,7 +83,7 @@ BUILD_TEMPLATE = """\ cd /src/%s -# build the library. +# build the target. # e.g. # # ./autogen.sh