[fuzzbench] Support measurement (#9655)

Save to a SQL database for now.

infra/base-images/base-builder-fuzzbench/Dockerfile

@@ -20,4 +20,6 @@ FROM gcr.io/oss-fuzz-base/base-builder
 COPY fuzzbench_install_dependencies /usr/local/bin
 RUN fuzzbench_install_dependencies
 
-COPY fuzzbench_build fuzzbench_run_fuzzer /usr/local/bin
+ENV OSS_FUZZ_ON_DEMAND=1
+
+COPY fuzzbench_build fuzzbench_run_fuzzer fuzzbench_measure /usr/local/bin/

infra/base-images/base-builder-fuzzbench/fuzzbench_build

@@ -17,10 +17,12 @@
 
 # TODO(metzman): Do this in a docket image so we don't need to waste time
 # reinstalling.
-PYTHONPATH=$FUZZBENCH python3 -B -u -c "from fuzzers.$FUZZING_ENGINE import fuzzer; fuzzer.build()"
+PYTHONPATH=$FUZZBENCH_PATH python3 -B -u -c "from fuzzers.$FUZZING_ENGINE import fuzzer; fuzzer.build()"
 
 if [ "$FUZZING_ENGINE" = "coverage" ]; then
   cd /out
   mkdir -p filestore/oss-fuzz-on-demand/coverage-binaries
-  tar -czvf filestore/oss-fuzz-on-demand/coverage-binaries/coverage-build-$PROJECT.tar.gz * /src /work
+  # We expect an error regarding leading slashes. Just assume this step succeeds.
+  # TODO(metzman): Fix this when I get a chance.
+  tar -czvf filestore/oss-fuzz-on-demand/coverage-binaries/coverage-build-$PROJECT.tar.gz * /src /work || exit 0
 fi

infra/base-images/base-builder-fuzzbench/fuzzbench_measure

@@ -0,0 +1,32 @@
+#! /bin/bash -eux
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+################################################################################
+
+# TODO(metzman): Make these configurable.
+export DB_PATH=$OUT/experiment.db
+export SNAPSHOT_PERIOD=30
+export EXPERIMENT_FILESTORE=$OUT/filestore
+export MAX_TOTAL_TIME=120
+export EXPERIMENT=oss-fuzz-on-demand
+
+rm -f $DB_PATH
+
+# FUZZER=mopt BENCHMARK=skcms
+
+export SQL_DATABASE_URL=sqlite:///$DB_PATH
+
+cd $FUZZBENCH_PATH
+PYTHONPATH=. python3 -B experiment/measurer/standalone.py $MAX_TOTAL_TIME

infra/base-images/base-builder-fuzzbench/fuzzbench_run_fuzzer

@@ -19,13 +19,21 @@ export RUNNER_NICENESS="-5"
 export EXPERIMENT_FILESTORE=/out/filestore
 export EXPERIMENT=oss-fuzz-on-demand
 export OSS_FUZZ_ON_DEMAND=1
-export OUTPUT_CORPUS_DIR=$OUT/out-corpus
+export OUTPUT_CORPUS_DIR=/output-corpus
 export SEED_CORPUS_DIR=/input-corpus
 mkdir $SEED_CORPUS_DIR
 rm -rf $OUTPUT_CORPUS_DIR
 mkdir $OUTPUT_CORPUS_DIR
 export FUZZER=$FUZZING_ENGINE
+# TODO(metzman): Make this configurable.
 export MAX_TOTAL_TIME=120
 export SNAPSHOT_PERIOD=30
+export TRIAL_ID=1
+
+# BENCHMARK, FUZZ_TARGET
 cd $OUT
-PYTHONPATH=$FUZZBENCH nice -n $RUNNER_NICENESS python3 -B -u $FUZZBENCH/experiment/runner.py
+
+# Prevent permissions issues with pyc files and docker.
+cp -r $FUZZBENCH_PATH /tmp/fuzzbench
+
+PYTHONPATH=/tmp/fuzzbench nice -n $RUNNER_NICENESS python3 -B -u /tmp/fuzzbench/experiment/runner.py

infra/helper.py

@@ -187,6 +187,8 @@ def main():  # pylint: disable=too-many-branches,too-many-return-statements
     result = fuzzbench_build_fuzzers(args)
   elif args.command == 'fuzzbench_run_fuzzer':
     result = fuzzbench_run_fuzzer(args)
+  elif args.command == 'fuzzbench_measure':
+    result = fuzzbench_measure(args)
   elif args.command == 'check_build':
     result = check_build(args)
   elif args.command == 'download_corpora':
@@ -245,7 +247,7 @@ def _add_external_project_args(parser):
   )
 
 
-def get_parser():  # pylint: disable=too-many-statements
+def get_parser():  # pylint: disable=too-many-statements,too-many-locals
   """Returns an argparse parser."""
   parser = argparse.ArgumentParser('helper.py', description='oss-fuzz helpers')
   subparsers = parser.add_subparsers(dest='command')
@@ -356,6 +358,15 @@ def get_parser():  # pylint: disable=too-many-statements
   fuzzbench_run_fuzzer_parser.add_argument(
       'fuzzer_args', help='arguments to pass to the fuzzer', nargs='*')
 
+  fuzzbench_measure_parser = subparsers.add_parser(
+      'fuzzbench_measure', help='Measure results from fuzzing.')
+  fuzzbench_measure_parser.add_argument(
+      'project', help='name of the project or path (external)')
+  fuzzbench_measure_parser.add_argument('engine_name',
+                                        help='name of the fuzzer')
+  fuzzbench_measure_parser.add_argument('fuzz_target_name',
+                                        help='name of the fuzzer')
+
   coverage_parser = subparsers.add_parser(
       'coverage', help='Generate code coverage report for the project.')
   coverage_parser.add_argument('--no-corpus-download',
@@ -936,8 +947,8 @@ def fuzzbench_build_fuzzers(args):
     ],
                    check=True)
     env = [
-        f'FUZZBENCH={fuzzbench_path}', 'OSS_FUZZ_ON_DEMAND=1',
-        f'PROJECT={args.project}'
+        f'FUZZBENCH_PATH={fuzzbench_path}', 'OSS_FUZZ_ON_DEMAND=1',
+        f'PROJECT={args.project.name}'
     ]
     tag = f'gcr.io/oss-fuzz/{args.project.name}'
     subprocess.run([
@@ -950,7 +961,7 @@ def fuzzbench_build_fuzzers(args):
         '--tag', tag, '--build-arg', f'parent_image={tag}', '--file',
         os.path.join(fuzzbench_path, 'fuzzers', args.engine,
                      'builder.Dockerfile'),
-        os.path.join(fuzzbench_path, 'fuzzers')
+        os.path.join(fuzzbench_path, 'fuzzers', args.engine)
     ])
 
     return build_fuzzers_impl(args.project,
@@ -1384,10 +1395,10 @@ def fuzzbench_run_fuzzer(args):
     return False
 
   env = [
-      'FUZZING_ENGINE=' + args.engine,
-      'SANITIZER=' + args.sanitizer,
-      'RUN_FUZZER_MODE=interactive',
-      'HELPER=True',
+      'FUZZING_ENGINE=' + args.engine, 'SANITIZER=' + args.sanitizer,
+      'RUN_FUZZER_MODE=interactive', 'HELPER=True',
+      f'FUZZ_TARGET={args.fuzzer_name}', f'BENCHMARK={args.project.name}',
+      'TRIAL_ID=1'
   ]
 
   if args.e:
@@ -1420,7 +1431,7 @@ def fuzzbench_run_fuzzer(args):
         '-v',
         f'{fuzzbench_path}:{fuzzbench_path}',
         '-e',
-        f'FUZZBENCH={fuzzbench_path}',
+        f'FUZZBENCH_PATH={fuzzbench_path}',
         f'gcr.io/oss-fuzz/{args.project.name}',
         'fuzzbench_run_fuzzer',
         args.fuzzer_name,
@@ -1429,6 +1440,31 @@ def fuzzbench_run_fuzzer(args):
     return docker_run(run_args, architecture=args.architecture)
 
 
+def fuzzbench_measure(args):
+  """Measure results from fuzzing with fuzzbench."""
+  if not check_project_exists(args.project):
+    return False
+
+  with tempfile.TemporaryDirectory() as tmp_dir:
+    tmp_dir = os.path.abspath(tmp_dir)
+    fuzzbench_path = os.path.join(tmp_dir, 'fuzzbench')
+    subprocess.run([
+        'git', 'clone', 'https://github.com/google/fuzzbench', '--depth', '1',
+        fuzzbench_path
+    ],
+                   check=True)
+    run_args = [
+        '-v', f'{args.project.out}:/out', '-v',
+        f'{fuzzbench_path}:{fuzzbench_path}', '-e',
+        f'FUZZBENCH_PATH={fuzzbench_path}', '-e',
+        f'FUZZ_TARGET={args.fuzz_target_name}', '-e',
+        f'FUZZER={args.engine_name}', '-e', f'BENCHMARK={args.project.name}',
+        f'gcr.io/oss-fuzz/{args.project.name}', 'fuzzbench_measure'
+    ]
+
+    return docker_run(run_args, 'x86_64')
+
+
 def reproduce(args):
   """Reproduces a specific test case from a specific project."""
   return reproduce_impl(args.project, args.fuzzer_name, args.valgrind, args.e,