oss-fuzz/infra/helper.py

#!/usr/bin/env python
# Copyright 2016 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
################################################################################

from __future__ import print_function
from multiprocessing.dummy import Pool as ThreadPool
import argparse
import datetime
import errno
import multiprocessing
import os
import pipes
import re
import subprocess
import sys
import tempfile
import templates

OSSFUZZ_DIR = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
BUILD_DIR = os.path.join(OSSFUZZ_DIR, 'build')

BASE_IMAGES = [
    'gcr.io/oss-fuzz-base/base-image',
    'gcr.io/oss-fuzz-base/base-clang',
    'gcr.io/oss-fuzz-base/base-builder',
    'gcr.io/oss-fuzz-base/base-runner',
    'gcr.io/oss-fuzz-base/base-runner-debug',
    'gcr.io/oss-fuzz-base/base-msan-builder',
    'gcr.io/oss-fuzz-base/msan-builder',
]

VALID_PROJECT_NAME_REGEX = re.compile(r'^[a-zA-Z0-9_-]+$')
MAX_PROJECT_NAME_LENGTH = 26

if sys.version_info[0] >= 3:
    raw_input = input

CORPUS_URL_FORMAT = (
    'gs://{project_name}-corpus.clusterfuzz-external.appspot.com/libFuzzer/'
    '{project_name}_{fuzz_target}/')
CORPUS_BACKUP_URL_FORMAT = (
    'gs://{project_name}-backup.clusterfuzz-external.appspot.com/corpus/'
    'libFuzzer/{project_name}_{fuzz_target}/')


def main():
  os.chdir(OSSFUZZ_DIR)
  if not os.path.exists(BUILD_DIR):
    os.mkdir(BUILD_DIR)

  parser = argparse.ArgumentParser('helper.py', description='oss-fuzz helpers')
  subparsers = parser.add_subparsers(dest='command')

  generate_parser = subparsers.add_parser(
      'generate', help='Generate files for new project.')
  generate_parser.add_argument('project_name')

  build_image_parser = subparsers.add_parser(
      'build_image', help='Build an image.')
  build_image_parser.add_argument('project_name')
  build_image_parser.add_argument('--pull', action='store_true',
                                  help='Pull latest base image.')
  build_image_parser.add_argument('--no-pull', action='store_true',
                                  help='Do not pull latest base image.')

  build_fuzzers_parser = subparsers.add_parser(
      'build_fuzzers', help='Build fuzzers for a project.')
  _add_engine_args(build_fuzzers_parser)
  _add_sanitizer_args(build_fuzzers_parser)
  _add_environment_args(build_fuzzers_parser)
  build_fuzzers_parser.add_argument('project_name')
  build_fuzzers_parser.add_argument('source_path', help='path of local source',
                                    nargs='?')
  build_fuzzers_parser.add_argument('--clean', dest='clean',
                                    action='store_true',
                                    help='clean existing artifacts.')
  build_fuzzers_parser.add_argument('--no-clean', dest='clean',
                                    action='store_false',
                                    help='do not clean existing artifacts '
                                    '(default).')
  build_fuzzers_parser.set_defaults(clean=False)

  check_build_parser = subparsers.add_parser(
      'check_build', help='Checks that fuzzers execute without errors.')
  _add_engine_args(check_build_parser)
  _add_sanitizer_args(check_build_parser)
  _add_environment_args(check_build_parser)
  check_build_parser.add_argument('project_name', help='name of the project')
  check_build_parser.add_argument('fuzzer_name', help='name of the fuzzer',
                                  nargs='?')

  run_fuzzer_parser = subparsers.add_parser(
      'run_fuzzer', help='Run a fuzzer.')
  _add_engine_args(run_fuzzer_parser)
  _add_environment_args(run_fuzzer_parser)
  run_fuzzer_parser.add_argument('project_name', help='name of the project')
  run_fuzzer_parser.add_argument('fuzzer_name', help='name of the fuzzer')
  run_fuzzer_parser.add_argument('fuzzer_args', help='arguments to pass to the fuzzer',
                                 nargs=argparse.REMAINDER)

  coverage_parser = subparsers.add_parser(
      'coverage', help='Run a fuzzer for a while and generate coverage.')
  coverage_parser.add_argument('--run_time', default=60,
                      help='time in seconds to run fuzzer')
  coverage_parser.add_argument('project_name', help='name of the project')
  coverage_parser.add_argument('fuzzer_name', help='name of the fuzzer')
  coverage_parser.add_argument('fuzzer_args', help='arguments to pass to the fuzzer',
                               nargs=argparse.REMAINDER)

  profile_parser = subparsers.add_parser(
      'profile', help='Generate code coverage report for the project.')
  profile_parser.add_argument('project_name', help='name of the project')
  profile_parser.add_argument('--no-corpus-download', action='store_true',
                              help='do not download corpus backup from OSS-Fuzz; '
                              'use corpus located in build/corpus/<project>/<fuzz_target>/')

  reproduce_parser = subparsers.add_parser(
      'reproduce', help='Reproduce a crash.')
  reproduce_parser.add_argument('--valgrind', action='store_true',
                       help='run with valgrind')
  reproduce_parser.add_argument('project_name', help='name of the project')
  reproduce_parser.add_argument('fuzzer_name', help='name of the fuzzer')
  reproduce_parser.add_argument('testcase_path', help='path of local testcase')
  reproduce_parser.add_argument('fuzzer_args', help='arguments to pass to the fuzzer',
                                nargs=argparse.REMAINDER)
  _add_environment_args(reproduce_parser)

  shell_parser = subparsers.add_parser(
      'shell', help='Run /bin/bash in an image.')
  shell_parser.add_argument('project_name', help='name of the project')
  _add_engine_args(shell_parser)
  _add_sanitizer_args(shell_parser)
  _add_environment_args(shell_parser)

  pull_images_parser = subparsers.add_parser('pull_images',
                                             help='Pull base images.')

  args = parser.parse_args()
  if args.command == 'generate':
    return generate(args)
  elif args.command == 'build_image':
    return build_image(args)
  elif args.command == 'build_fuzzers':
    return build_fuzzers(args)
  elif args.command == 'check_build':
    return check_build(args)
  elif args.command == 'run_fuzzer':
    return run_fuzzer(args)
  elif args.command == 'coverage':
    return coverage(args)
  elif args.command == 'profile':
    return profile(args)
  elif args.command == 'reproduce':
    return reproduce(args)
  elif args.command == 'shell':
    return shell(args)
  elif args.command == 'pull_images':
    return pull_images(args)

  return 0


def _is_base_image(image_name):
  """Checks if the image name is a base image."""
  return os.path.exists(os.path.join('infra', 'base-images', image_name))


def _check_project_exists(project_name):
  """Checks if a project exists."""
  if not os.path.exists(os.path.join(OSSFUZZ_DIR, 'projects', project_name)):
    print(project_name, 'does not exist', file=sys.stderr)
    return False

  return True


def _check_fuzzer_exists(project_name, fuzzer_name):
  """Checks if a fuzzer exists."""
  command = ['docker', 'run', '--rm']
  command.extend(['-v', '%s:/out' % _get_output_dir(project_name)])
  command.append('ubuntu:16.04')

  command.extend(['/bin/bash', '-c', 'test -f /out/%s' % fuzzer_name])

  try:
    subprocess.check_call(command)
  except subprocess.CalledProcessError:
    print(fuzzer_name,
          'does not seem to exist. Please run build_fuzzers first.',
          file=sys.stderr)
    return False

  return True


def _get_absolute_path(path):
  """Returns absolute path with user expansion."""
  return os.path.abspath(os.path.expanduser(path))


def _get_command_string(command):
  """Returns a shell escaped command string."""
  return ' '.join(pipes.quote(part) for part in command)


def _get_corpus_dir(project_name=''):
  """Returns path to /corpus directory for the given project (if specified)."""
  return os.path.join(BUILD_DIR, 'corpus', project_name)


def _get_output_dir(project_name=''):
  """Returns path to /out directory for the given project (if specified)."""
  return os.path.join(BUILD_DIR, 'out', project_name)


def _get_work_dir(project_name=''):
  """Returns path to /work directory for the given project (if specified)."""
  return os.path.join(BUILD_DIR, 'work', project_name)


def _add_engine_args(parser):
  """Add common engine args."""
  parser.add_argument('--engine', default='libfuzzer',
                      choices=['libfuzzer', 'afl', 'honggfuzz', 'none'])


def _add_sanitizer_args(parser):
  """Add common sanitizer args."""
  parser.add_argument('--sanitizer', default='address',
                      choices=['address', 'memory', 'undefined', 'coverage', 'profile'])


def _add_environment_args(parser):
  """Add common environment args."""
  parser.add_argument('-e', action='append',
                      help="set environment variable e.g. VAR=value")


def _build_image(image_name, no_cache=False, pull=False):
  """Build image."""

  is_base_image = _is_base_image(image_name)
  if is_base_image:
    image_project = 'oss-fuzz-base'
    dockerfile_dir = os.path.join('infra', 'base-images', image_name)
  else:
    image_project = 'oss-fuzz'
    if not _check_project_exists(image_name):
      return False

    dockerfile_dir = os.path.join('projects', image_name)

  build_args = []
  if no_cache:
    build_args.append('--no-cache')

  build_args += ['-t', 'gcr.io/%s/%s' % (image_project, image_name), dockerfile_dir]

  return docker_build(build_args, pull=pull)


def _env_to_docker_args(env_list):
  """Turn envirnoment variable list into docker arguments."""
  return sum([['-e', v] for v in env_list], [])


def docker_run(run_args, print_output=True):
  """Call `docker run`."""
  command = ['docker', 'run', '--rm', '-i', '--privileged']
  command.extend(run_args)

  print('Running:', _get_command_string(command))
  stdout = None
  if not print_output:
    stdout = open(os.devnull, 'w')

  try:
    subprocess.check_call(command, stdout=stdout, stderr=subprocess.STDOUT)
  except subprocess.CalledProcessError as e:
    return e.returncode

  return 0


def docker_build(build_args, pull=False):
  """Call `docker build`."""
  command = ['docker', 'build']
  if pull:
    command.append('--pull')

  command.extend(build_args)
  print('Running:', _get_command_string(command))

  try:
    subprocess.check_call(command)
  except subprocess.CalledProcessError:
    print('docker build failed.', file=sys.stderr)
    return False

  return True


def docker_pull(image, pull=False):
  """Call `docker pull`."""
  command = ['docker', 'pull', image]
  print('Running:', _get_command_string(command))

  try:
    subprocess.check_call(command)
  except subprocess.CalledProcessError:
    print('docker pull failed.', file=sys.stderr)
    return False

  return True


def build_image(args):
  """Build docker image."""
  if args.pull and args.no_pull:
    print('Incompatible arguments --pull and --no-pull.')
    return 1

  if args.pull:
    pull = True
  elif args.no_pull:
    pull = False
  else:
    y_or_n = raw_input('Pull latest base images (compiler/runtime)? (y/N): ')
    pull = y_or_n.lower() == 'y'

  if pull:
    print('Pulling latest base images...')
  else:
    print('Using cached base images...')

  # If build_image is called explicitly, don't use cache.
  if _build_image(args.project_name, no_cache=True, pull=pull):
    return 0

  return 1


def build_fuzzers(args):
  """Build fuzzers."""
  project_name = args.project_name
  if not _build_image(args.project_name):
    return 1

  project_out_dir = _get_output_dir(project_name)
  if args.clean:
    print('Cleaning existing build artifacts.')

    # Clean old and possibly conflicting artifacts in project's out directory.
    docker_run([
        '-v', '%s:/out' % project_out_dir,
        '-t', 'gcr.io/oss-fuzz/%s' % project_name,
        '/bin/bash', '-c', 'rm -rf /out/*'
    ])
  else:
    print('Keeping existing build artifacts as-is (if any).')

  env = [
      'FUZZING_ENGINE=' + args.engine,
      'SANITIZER=' + args.sanitizer
  ]
  if args.e:
    env += args.e

  project_work_dir = _get_work_dir(project_name)

  # Copy instrumented libraries.
  if args.sanitizer == 'memory':
    docker_run([
        '-v', '%s:/work' % project_work_dir,
        'gcr.io/oss-fuzz-base/msan-builder',
        'bash', '-c', 'cp -r /msan /work'])
    env.append('MSAN_LIBS_PATH=' + '/work/msan')

  command = (
      ['docker', 'run', '--rm', '-i', '--cap-add', 'SYS_PTRACE'] +
      _env_to_docker_args(env))
  if args.source_path:
    command += [
        '-v',
        '%s:/src/%s' % (_get_absolute_path(args.source_path), args.project_name)
    ]
  command += [
      '-v', '%s:/out' % project_out_dir,
      '-v', '%s:/work' % project_work_dir,
      '-t', 'gcr.io/oss-fuzz/%s' % project_name
  ]

  print('Running:', _get_command_string(command))

  try:
    subprocess.check_call(command)
  except subprocess.CalledProcessError:
    print('fuzzers build failed.', file=sys.stderr)
    return 1

  # Patch MSan builds to use instrumented shared libraries.
  if args.sanitizer == 'memory':
    docker_run([
        '-v', '%s:/out' % project_out_dir,
        '-v', '%s:/work' % project_work_dir
    ] + _env_to_docker_args(env) + [
        'gcr.io/oss-fuzz-base/base-msan-builder',
        'patch_build.py', '/out'
    ])

  return 0


def check_build(args):
  """Checks that fuzzers in the container execute without errors."""
  if not _check_project_exists(args.project_name):
    return 1

  if (args.fuzzer_name and
      not _check_fuzzer_exists(args.project_name, args.fuzzer_name)):
    return 1

  env = [
      'FUZZING_ENGINE=' + args.engine,
      'SANITIZER=' + args.sanitizer
  ]
  if args.e:
    env += args.e

  run_args = _env_to_docker_args(env) + [
      '-v', '%s:/out' % _get_output_dir(args.project_name),
      '-t', 'gcr.io/oss-fuzz-base/base-runner'
  ]

  if args.fuzzer_name:
    run_args += [
        'bad_build_check',
        os.path.join('/out', args.fuzzer_name)
    ]
  else:
    run_args.append('test_all')

  exit_code = docker_run(run_args)
  if exit_code == 0:
    print('Check build passed.')
  else:
    print('Check build failed.')

  return exit_code


def _get_fuzz_targets(project_name):
  """Return names of fuzz targest build in the project's /out directory."""
  return [
    p
    for p in os.listdir(_get_output_dir(project_name))
    if os.access(p, os.X_OK)
  ]


def _get_latest_corpus(project_name, fuzz_target, base_corpus_dir):
  """Download the latest corpus for the given fuzz target."""
  corpus_dir = os.path.join(base_corpus_dir, fuzz_target)
  if not os.path.exists(corpus_dir):
    os.makedirs(corpus_dir)

  corpus_backup_url = CORPUS_BACKUP_URL_FORMAT.format(project_name=project_name,
                                                      fuzz_target=fuzz_target)
  command = [
      'gsutil',
      'ls',
      corpus_backup_url
  ]
  output = subprocess.check_output(command).splitlines()
  if output:
    latest_backup_url = output[-1]
    archive_path = corpus_dir + '.zip'
    command = [
        'gsutil',
        'cp',
        latest_backup_url,
        archive_path
    ]
    subprocess.check_call(command)
    command = [
        'unzip',
        '-q',
        '-o',
        archive_path,
        '-d',
        corpus_dir
    ]
    subprocess.check_call(command)
    os.remove(archive_path)
  else:
    # Sync the working corpus copy if a minimized backup is not available.
    corpus_url = CORPUS_URL_FORMAT.format(project_name=project_name,
                                          fuzz_target=fuzz_target)
    command = [
        'gsutil',
        '-m',
        '-q',
        'rsync',
        '-R',
        corpus_url,
        corpus_dir
    ]
    subprocess.check_call(command)


def download_corpus(project_name):
  """Download most recent corpus from GCS for the given project."""
  fuzz_targets = _get_fuzz_targets(project_name)
  corpus_dir = _get_corpus_dir(project_name)
  if not os.path.exists(corpus_dir):
    os.makedirs(corpus_dir)

  def _download_for_single_target(fuzz_target):
    _get_latest_corpus(project_name, fuzz_target, corpus_dir)

  print('Downloading corpus for %s' % project_name)
  thread_pool = ThreadPool(multiprocessing.cpu_count())
  thread_pool.map(_download_for_single_target, fuzz_targets)


def profile(args):
  """Generate code coverage using clang source based code coverage."""
  if not _check_project_exists(args.project_name):
    return 1
  if not args.no_corpus_download:
    download_corpus(args.project_name)

  env = [
      'FUZZING_ENGINE=libfuzzer',
      'PROJECT=%s' % args.project_name,
      'SANITIZER=profile'
  ]

  run_args = _env_to_docker_args(env) + [
      '-v', '%s:/out' % _get_output_dir(args.project_name),
      '-v', '%s:/corpus' % _get_corpus_dir(args.project_name),
      '-p', '8008:8008',
      '-t', 'gcr.io/oss-fuzz-base/base-runner',
  ]

  run_args.append('coverage')

  exit_code = docker_run(run_args)
  if exit_code == 0:
    print('Successfully generated clang code coverage report.')
  else:
    print('Failed to generate clang code coverage report.')

  return exit_code


def run_fuzzer(args):
  """Runs a fuzzer in the container."""
  if not _check_project_exists(args.project_name):
    return 1

  if not _check_fuzzer_exists(args.project_name, args.fuzzer_name):
    return 1

  env = ['FUZZING_ENGINE=' + args.engine]
  if args.e:
    env += args.e

  run_args = _env_to_docker_args(env) + [
      '-v', '%s:/out' % _get_output_dir(args.project_name),
      '-t', 'gcr.io/oss-fuzz-base/base-runner',
      'run_fuzzer',
      args.fuzzer_name,
  ] + args.fuzzer_args

  return docker_run(run_args)


def coverage(args):
  """Runs a fuzzer in the container."""
  if not _check_project_exists(args.project_name):
    return 1

  if not _check_fuzzer_exists(args.project_name, args.fuzzer_name):
    return 1

  temp_dir = tempfile.mkdtemp()

  run_args = [
      '-e', 'FUZZING_ENGINE=libfuzzer',
      '-e', 'ASAN_OPTIONS=coverage_dir=/cov',
      '-e', 'MSAN_OPTIONS=coverage_dir=/cov',
      '-e', 'UBSAN_OPTIONS=coverage_dir=/cov',
      '-v', '%s:/out' % _get_output_dir(args.project_name),
      '-v', '%s:/cov' % temp_dir,
      '-w', '/cov',
      '-t', 'gcr.io/oss-fuzz-base/base-runner',
      'run_fuzzer',
      args.fuzzer_name,
      '-dump_coverage=1',
      '-max_total_time=%s' % args.run_time
  ] + args.fuzzer_args

  print('This may take a while (running your fuzzer for %s seconds)...' %
        args.run_time)
  docker_run(run_args, print_output=False)

  run_args = [
      '-v', '%s:/out' % _get_output_dir(args.project_name),
      '-v', '%s:/cov' % temp_dir,
      '-w', '/cov',
      '-p', '8001:8001',
      '-t', 'gcr.io/oss-fuzz/%s' % args.project_name,
      'coverage_report', '/out/%s' % args.fuzzer_name,
  ]

  docker_run(run_args)


def reproduce(args):
  """Reproduces a testcase in the container."""
  if not _check_project_exists(args.project_name):
    return 1

  if not _check_fuzzer_exists(args.project_name, args.fuzzer_name):
    return 1

  debugger = ''
  env = []
  image_name = 'base-runner'

  if args.valgrind:
    debugger = 'valgrind --tool=memcheck --track-origins=yes --leak-check=full'

  if debugger:
    image_name = 'base-runner-debug'
    env += ['DEBUGGER=' + debugger]

  if args.e:
    env += args.e

  run_args = _env_to_docker_args(env) + [
      '-v', '%s:/out' % _get_output_dir(args.project_name),
      '-v', '%s:/testcase' % _get_absolute_path(args.testcase_path),
      '-t', 'gcr.io/oss-fuzz-base/%s' % image_name,
      'reproduce',
      args.fuzzer_name,
      '-runs=100',
  ] + args.fuzzer_args

  return docker_run(run_args)


def generate(args):
  """Generate empty project files."""
  if len(args.project_name) > MAX_PROJECT_NAME_LENGTH:
    print('Project name needs to be less than or equal to %d characters.' %
          MAX_PROJECT_NAME_LENGTH, file=sys.stderr)
    return 1

  if not VALID_PROJECT_NAME_REGEX.match(args.project_name):
    print('Invalid project name.', file=sys.stderr)
    return 1

  dir = os.path.join('projects', args.project_name)

  try:
    os.mkdir(dir)
  except OSError as e:
    if e.errno != errno.EEXIST:
      raise
    print(dir, 'already exists.', file=sys.stderr)
    return 1

  print('Writing new files to', dir)

  template_args = {
    'project_name': args.project_name,
    'year': datetime.datetime.now().year
  }
  with open(os.path.join(dir, 'project.yaml'), 'w') as f:
    f.write(templates.PROJECT_YAML_TEMPLATE % template_args)

  with open(os.path.join(dir, 'Dockerfile'), 'w') as f:
    f.write(templates.DOCKER_TEMPLATE % template_args)

  build_sh_path = os.path.join(dir, 'build.sh')
  with open(build_sh_path, 'w') as f:
    f.write(templates.BUILD_TEMPLATE % template_args)

  os.chmod(build_sh_path, 0o755)
  return 0


def shell(args):
  """Runs a shell within a docker image."""
  if not _build_image(args.project_name):
    return 1

  env = [
      'FUZZING_ENGINE=' + args.engine,
      'SANITIZER=' + args.sanitizer
  ]

  if args.e:
    env += args.e

  if _is_base_image(args.project_name):
    image_project = 'oss-fuzz-base'
    out_dir = _get_output_dir()
  else:
    image_project = 'oss-fuzz'
    out_dir = _get_output_dir(args.project_name)

  run_args = _env_to_docker_args(env) + [
      '-v', '%s:/out' % out_dir,
      '-v', '%s:/work' % _get_work_dir(args.project_name),
      '-t', 'gcr.io/%s/%s' % (image_project, args.project_name),
      '/bin/bash'
  ]

  docker_run(run_args)
  return 0


def pull_images(args):
  """Pull base images."""
  for base_image in BASE_IMAGES:
    if not docker_pull(base_image):
      return 1

  return 0


if __name__ == '__main__':
  sys.exit(main())