diff --git a/infra/base-images/base-builder-fuzzbench/Dockerfile b/infra/base-images/base-builder-fuzzbench/Dockerfile index 781bf4e46..4349b50a6 100644 --- a/infra/base-images/base-builder-fuzzbench/Dockerfile +++ b/infra/base-images/base-builder-fuzzbench/Dockerfile @@ -20,4 +20,6 @@ FROM gcr.io/oss-fuzz-base/base-builder COPY fuzzbench_install_dependencies /usr/local/bin RUN fuzzbench_install_dependencies -COPY fuzzbench_build fuzzbench_run_fuzzer /usr/local/bin \ No newline at end of file +ENV OSS_FUZZ_ON_DEMAND=1 + +COPY fuzzbench_build fuzzbench_run_fuzzer fuzzbench_measure /usr/local/bin/ \ No newline at end of file diff --git a/infra/base-images/base-builder-fuzzbench/fuzzbench_build b/infra/base-images/base-builder-fuzzbench/fuzzbench_build index 05fb427da..a00d0d1b1 100755 --- a/infra/base-images/base-builder-fuzzbench/fuzzbench_build +++ b/infra/base-images/base-builder-fuzzbench/fuzzbench_build @@ -17,10 +17,12 @@ # TODO(metzman): Do this in a docket image so we don't need to waste time # reinstalling. -PYTHONPATH=$FUZZBENCH python3 -B -u -c "from fuzzers.$FUZZING_ENGINE import fuzzer; fuzzer.build()" +PYTHONPATH=$FUZZBENCH_PATH python3 -B -u -c "from fuzzers.$FUZZING_ENGINE import fuzzer; fuzzer.build()" if [ "$FUZZING_ENGINE" = "coverage" ]; then cd /out mkdir -p filestore/oss-fuzz-on-demand/coverage-binaries - tar -czvf filestore/oss-fuzz-on-demand/coverage-binaries/coverage-build-$PROJECT.tar.gz * /src /work + # We expect an error regarding leading slashes. Just assume this step succeeds. + # TODO(metzman): Fix this when I get a chance. + tar -czvf filestore/oss-fuzz-on-demand/coverage-binaries/coverage-build-$PROJECT.tar.gz * /src /work || exit 0 fi diff --git a/infra/base-images/base-builder-fuzzbench/fuzzbench_measure b/infra/base-images/base-builder-fuzzbench/fuzzbench_measure new file mode 100755 index 000000000..ba9a413a0 --- /dev/null +++ b/infra/base-images/base-builder-fuzzbench/fuzzbench_measure @@ -0,0 +1,32 @@ +#! /bin/bash -eux +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +################################################################################ + +# TODO(metzman): Make these configurable. +export DB_PATH=$OUT/experiment.db +export SNAPSHOT_PERIOD=30 +export EXPERIMENT_FILESTORE=$OUT/filestore +export MAX_TOTAL_TIME=120 +export EXPERIMENT=oss-fuzz-on-demand + +rm -f $DB_PATH + +# FUZZER=mopt BENCHMARK=skcms + +export SQL_DATABASE_URL=sqlite:///$DB_PATH + +cd $FUZZBENCH_PATH +PYTHONPATH=. python3 -B experiment/measurer/standalone.py $MAX_TOTAL_TIME diff --git a/infra/base-images/base-builder-fuzzbench/fuzzbench_run_fuzzer b/infra/base-images/base-builder-fuzzbench/fuzzbench_run_fuzzer index 523aa82d2..5bb38fd6f 100755 --- a/infra/base-images/base-builder-fuzzbench/fuzzbench_run_fuzzer +++ b/infra/base-images/base-builder-fuzzbench/fuzzbench_run_fuzzer @@ -19,13 +19,21 @@ export RUNNER_NICENESS="-5" export EXPERIMENT_FILESTORE=/out/filestore export EXPERIMENT=oss-fuzz-on-demand export OSS_FUZZ_ON_DEMAND=1 -export OUTPUT_CORPUS_DIR=$OUT/out-corpus +export OUTPUT_CORPUS_DIR=/output-corpus export SEED_CORPUS_DIR=/input-corpus mkdir $SEED_CORPUS_DIR rm -rf $OUTPUT_CORPUS_DIR mkdir $OUTPUT_CORPUS_DIR export FUZZER=$FUZZING_ENGINE +# TODO(metzman): Make this configurable. export MAX_TOTAL_TIME=120 export SNAPSHOT_PERIOD=30 +export TRIAL_ID=1 + +# BENCHMARK, FUZZ_TARGET cd $OUT -PYTHONPATH=$FUZZBENCH nice -n $RUNNER_NICENESS python3 -B -u $FUZZBENCH/experiment/runner.py + +# Prevent permissions issues with pyc files and docker. +cp -r $FUZZBENCH_PATH /tmp/fuzzbench + +PYTHONPATH=/tmp/fuzzbench nice -n $RUNNER_NICENESS python3 -B -u /tmp/fuzzbench/experiment/runner.py diff --git a/infra/helper.py b/infra/helper.py index 8de4f27e7..05bedca7e 100755 --- a/infra/helper.py +++ b/infra/helper.py @@ -187,6 +187,8 @@ def main(): # pylint: disable=too-many-branches,too-many-return-statements result = fuzzbench_build_fuzzers(args) elif args.command == 'fuzzbench_run_fuzzer': result = fuzzbench_run_fuzzer(args) + elif args.command == 'fuzzbench_measure': + result = fuzzbench_measure(args) elif args.command == 'check_build': result = check_build(args) elif args.command == 'download_corpora': @@ -245,7 +247,7 @@ def _add_external_project_args(parser): ) -def get_parser(): # pylint: disable=too-many-statements +def get_parser(): # pylint: disable=too-many-statements,too-many-locals """Returns an argparse parser.""" parser = argparse.ArgumentParser('helper.py', description='oss-fuzz helpers') subparsers = parser.add_subparsers(dest='command') @@ -356,6 +358,15 @@ def get_parser(): # pylint: disable=too-many-statements fuzzbench_run_fuzzer_parser.add_argument( 'fuzzer_args', help='arguments to pass to the fuzzer', nargs='*') + fuzzbench_measure_parser = subparsers.add_parser( + 'fuzzbench_measure', help='Measure results from fuzzing.') + fuzzbench_measure_parser.add_argument( + 'project', help='name of the project or path (external)') + fuzzbench_measure_parser.add_argument('engine_name', + help='name of the fuzzer') + fuzzbench_measure_parser.add_argument('fuzz_target_name', + help='name of the fuzzer') + coverage_parser = subparsers.add_parser( 'coverage', help='Generate code coverage report for the project.') coverage_parser.add_argument('--no-corpus-download', @@ -936,8 +947,8 @@ def fuzzbench_build_fuzzers(args): ], check=True) env = [ - f'FUZZBENCH={fuzzbench_path}', 'OSS_FUZZ_ON_DEMAND=1', - f'PROJECT={args.project}' + f'FUZZBENCH_PATH={fuzzbench_path}', 'OSS_FUZZ_ON_DEMAND=1', + f'PROJECT={args.project.name}' ] tag = f'gcr.io/oss-fuzz/{args.project.name}' subprocess.run([ @@ -950,7 +961,7 @@ def fuzzbench_build_fuzzers(args): '--tag', tag, '--build-arg', f'parent_image={tag}', '--file', os.path.join(fuzzbench_path, 'fuzzers', args.engine, 'builder.Dockerfile'), - os.path.join(fuzzbench_path, 'fuzzers') + os.path.join(fuzzbench_path, 'fuzzers', args.engine) ]) return build_fuzzers_impl(args.project, @@ -1384,10 +1395,10 @@ def fuzzbench_run_fuzzer(args): return False env = [ - 'FUZZING_ENGINE=' + args.engine, - 'SANITIZER=' + args.sanitizer, - 'RUN_FUZZER_MODE=interactive', - 'HELPER=True', + 'FUZZING_ENGINE=' + args.engine, 'SANITIZER=' + args.sanitizer, + 'RUN_FUZZER_MODE=interactive', 'HELPER=True', + f'FUZZ_TARGET={args.fuzzer_name}', f'BENCHMARK={args.project.name}', + 'TRIAL_ID=1' ] if args.e: @@ -1420,7 +1431,7 @@ def fuzzbench_run_fuzzer(args): '-v', f'{fuzzbench_path}:{fuzzbench_path}', '-e', - f'FUZZBENCH={fuzzbench_path}', + f'FUZZBENCH_PATH={fuzzbench_path}', f'gcr.io/oss-fuzz/{args.project.name}', 'fuzzbench_run_fuzzer', args.fuzzer_name, @@ -1429,6 +1440,31 @@ def fuzzbench_run_fuzzer(args): return docker_run(run_args, architecture=args.architecture) +def fuzzbench_measure(args): + """Measure results from fuzzing with fuzzbench.""" + if not check_project_exists(args.project): + return False + + with tempfile.TemporaryDirectory() as tmp_dir: + tmp_dir = os.path.abspath(tmp_dir) + fuzzbench_path = os.path.join(tmp_dir, 'fuzzbench') + subprocess.run([ + 'git', 'clone', 'https://github.com/google/fuzzbench', '--depth', '1', + fuzzbench_path + ], + check=True) + run_args = [ + '-v', f'{args.project.out}:/out', '-v', + f'{fuzzbench_path}:{fuzzbench_path}', '-e', + f'FUZZBENCH_PATH={fuzzbench_path}', '-e', + f'FUZZ_TARGET={args.fuzz_target_name}', '-e', + f'FUZZER={args.engine_name}', '-e', f'BENCHMARK={args.project.name}', + f'gcr.io/oss-fuzz/{args.project.name}', 'fuzzbench_measure' + ] + + return docker_run(run_args, 'x86_64') + + def reproduce(args): """Reproduces a specific test case from a specific project.""" return reproduce_impl(args.project, args.fuzzer_name, args.valgrind, args.e,