From d88b5e4aa728515ab1a668fcc60dbb17c1658286 Mon Sep 17 00:00:00 2001 From: jonathanmetzman <31354670+jonathanmetzman@users.noreply.github.com> Date: Wed, 4 Aug 2021 06:42:17 -0700 Subject: [PATCH] [CIFuzz] Add support for external projects to helper.py (#6141) Add support to helper.py for external projects for all subcommands except `download_corpora`. External users just need to specify `build_integration_path`, `project_src_path` and `external`. Also do a big refactor to pass a project object as part of args instead of a string containing the name of the project or its path. Related: #6125 --- infra/bisector.py | 17 +- infra/build_specified_commit.py | 5 +- infra/build_specified_commit_test.py | 12 +- infra/helper.py | 430 +++++++++++++++++---------- infra/helper_test.py | 103 ++++++- infra/templates.py | 33 ++ 6 files changed, 419 insertions(+), 181 deletions(-) diff --git a/infra/bisector.py b/infra/bisector.py index a9df522a2..9afdc7c50 100644 --- a/infra/bisector.py +++ b/infra/bisector.py @@ -105,7 +105,7 @@ def main(): architecture=args.architecture) result = bisect(args.type, args.old_commit, args.new_commit, - args.testcase_path, args.fuzz_target, build_data) + args.test_case_path, args.fuzz_target, build_data) if not result.commit: logging.error('No error was found in commit range %s:%s', args.old_commit, args.new_commit) @@ -142,12 +142,15 @@ def _check_for_crash(project_name, fuzz_target, testcase_path): return utils.execute(command + args) logging.info('Checking for crash') - out, err, return_code = helper.reproduce_impl(project_name, - fuzz_target, - False, [], [], - testcase_path, - run_function=docker_run, - err_result=(None, None, None)) + out, err, return_code = helper.reproduce_impl( + project=helper.Project(project_name), + fuzzer_name=fuzz_target, + valgrind=False, + env_to_add=[], + fuzzer_args=[], + testcase_path=testcase_path, + run_function=docker_run, + err_result=(None, None, None)) if return_code is None: return None diff --git a/infra/build_specified_commit.py b/infra/build_specified_commit.py index 798010b7b..4768275e1 100644 --- a/infra/build_specified_commit.py +++ b/infra/build_specified_commit.py @@ -147,7 +147,7 @@ def copy_src_from_docker(project_name, host_dir): @retry.wrap(_IMAGE_BUILD_TRIES, 2) def _build_image_with_retries(project_name): """Build image with retries.""" - return helper.build_image_impl(project_name) + return helper.build_image_impl(helper.Project(project_name)) def get_required_post_checkout_steps(dockerfile_path): @@ -223,7 +223,8 @@ def build_fuzzers_from_commit(commit, post_checkout_step, ]) - result = helper.build_fuzzers_impl(project_name=build_data.project_name, + project = helper.Project(build_data.project_name) + result = helper.build_fuzzers_impl(project=project, clean=True, engine=build_data.engine, sanitizer=build_data.sanitizer, diff --git a/infra/build_specified_commit_test.py b/infra/build_specified_commit_test.py index 7aa27509f..42adae5d0 100644 --- a/infra/build_specified_commit_test.py +++ b/infra/build_specified_commit_test.py @@ -61,7 +61,8 @@ class BuildImageIntegrationTest(unittest.TestCase): build_specified_commit.build_fuzzers_from_commit(test_repo.old_commit, test_repo_manager, host_src_dir, build_data) - old_result = helper.reproduce_impl(project_name=test_repo.project_name, + project = helper.Project(test_repo.project_name) + old_result = helper.reproduce_impl(project=project, fuzzer_name=test_repo.fuzz_target, valgrind=False, env_to_add=[], @@ -70,9 +71,12 @@ class BuildImageIntegrationTest(unittest.TestCase): build_specified_commit.build_fuzzers_from_commit(test_repo.project_name, test_repo_manager, host_src_dir, build_data) - new_result = helper.reproduce_impl(test_repo.project_name, - test_repo.fuzz_target, False, [], [], - test_repo.testcase_path) + new_result = helper.reproduce_impl(project=project, + fuzzer_name=test_repo.fuzz_target, + valgrind=False, + env_to_add=[], + fuzzer_args=[], + testcase_path=test_repo.testcase_path) self.assertNotEqual(new_result, old_result) def test_detect_main_repo_from_commit(self): diff --git a/infra/helper.py b/infra/helper.py index 9ae49954b..246c9e775 100755 --- a/infra/helper.py +++ b/infra/helper.py @@ -63,9 +63,70 @@ LANGUAGES_WITH_COVERAGE_SUPPORT = ['c', 'c++', 'go', 'jvm', 'rust'] WORKDIR_REGEX = re.compile(r'\s*WORKDIR\s*([^\s]+)') +# TODO(jonathanmetzman): Enforce this. +DEFAULT_RELATIVE_BUILD_INTEGRATION_PATH = '.cifuzz' + # pylint: disable=too-many-lines +class Project: + """Class representing a project that is in OSS-Fuzz or an external project + (ClusterFuzzLite user).""" + + def __init__(self, + project_name_or_path, + is_external=False, + build_integration_path=DEFAULT_RELATIVE_BUILD_INTEGRATION_PATH): + self.is_external = is_external + if self.is_external: + self.name = os.path.basename(os.path.abspath(project_name_or_path)) + self.path = project_name_or_path + self.build_integration_path = os.path.join(self.path, + build_integration_path) + else: + self.name = project_name_or_path + self.path = os.path.join(OSS_FUZZ_DIR, 'projects', self.name) + self.build_integration_path = self.path + + @property + def dockerfile_path(self): + """Returns path to the project Dockerfile.""" + return os.path.join(self.path, 'Dockerfile') + + @property + def language(self): + """Returns project language.""" + if self.is_external: + # TODO(metzman): Handle this properly. + return 'c++' + + project_yaml_path = os.path.join(self.path, 'project.yaml') + with open(project_yaml_path) as file_handle: + content = file_handle.read() + for line in content.splitlines(): + match = PROJECT_LANGUAGE_REGEX.match(line) + if match: + return match.group(1) + + logging.warning('Language not specified in project.yaml.') + return None + + @property + def out(self): + """Returns the out dir for the project. Creates it if needed.""" + return _get_out_dir(self.name) + + @property + def work(self): + """Returns the out dir for the project. Creates it if needed.""" + return _get_project_build_subdir(self.name, 'work') + + @property + def corpus(self): + """Returns the out dir for the project. Creates it if needed.""" + return _get_project_build_subdir(self.name, 'corpus') + + def main(): # pylint: disable=too-many-branches,too-many-return-statements """Gets subcommand from program arguments and does it. Returns 0 on success 1 on error.""" @@ -121,10 +182,37 @@ def bool_to_retcode(boolean): def parse_args(parser, args=None): - """Parses |args| using |parser| and returns parsed args.""" + """Parses |args| using |parser| and returns parsed args. Also changes + |args.build_integration_path| to have correct default behavior.""" # Use default argument None for args so that in production, argparse does its # normal behavior, but unittesting is easier. - return parser.parse_args(args) + parsed_args = parser.parse_args(args) + project = getattr(parsed_args, 'project', None) + if not project: + return parsed_args + + # Use hacky method for extracting attributes so that ShellTest works. + # TODO(metzman): Fix this. + is_external = getattr(parsed_args, 'is_external', False) + build_integration_path = getattr(parsed_args, 'build_integration_path', False) + + parsed_args.project = Project(parsed_args.project, is_external, + build_integration_path) + return parsed_args + + +def _add_external_project_args(parser): + parser.add_argument('--build-integration-path', + help=('Path to the build integration for non-OSS-Fuzz ' + 'projects.'), + default=DEFAULT_RELATIVE_BUILD_INTEGRATION_PATH) + + parser.add_argument( + '--external', + help='Is project external?', + default=False, + action='store_true', + ) def get_parser(): # pylint: disable=too-many-statements @@ -134,11 +222,12 @@ def get_parser(): # pylint: disable=too-many-statements generate_parser = subparsers.add_parser( 'generate', help='Generate files for new project.') - generate_parser.add_argument('project_name') + generate_parser.add_argument('project') + _add_external_project_args(generate_parser) build_image_parser = subparsers.add_parser('build_image', help='Build an image.') - build_image_parser.add_argument('project_name') + build_image_parser.add_argument('project') build_image_parser.add_argument('--pull', action='store_true', help='Pull latest base image.') @@ -149,6 +238,7 @@ def get_parser(): # pylint: disable=too-many-statements build_image_parser.add_argument('--no-pull', action='store_true', help='Do not pull latest base image.') + _add_external_project_args(build_image_parser) build_fuzzers_parser = subparsers.add_parser( 'build_fuzzers', help='Build fuzzers for a project.') @@ -156,7 +246,8 @@ def get_parser(): # pylint: disable=too-many-statements _add_engine_args(build_fuzzers_parser) _add_sanitizer_args(build_fuzzers_parser) _add_environment_args(build_fuzzers_parser) - build_fuzzers_parser.add_argument('project_name') + _add_external_project_args(build_fuzzers_parser) + build_fuzzers_parser.add_argument('project') build_fuzzers_parser.add_argument('source_path', help='path of local source', nargs='?') @@ -185,19 +276,23 @@ def get_parser(): # pylint: disable=too-many-statements check_build_parser, choices=['address', 'memory', 'undefined', 'dataflow', 'thread']) _add_environment_args(check_build_parser) - check_build_parser.add_argument('project_name', help='name of the project') + check_build_parser.add_argument('project', + help='name of the project or path (external)') check_build_parser.add_argument('fuzzer_name', help='name of the fuzzer', nargs='?') + _add_external_project_args(check_build_parser) run_fuzzer_parser = subparsers.add_parser( 'run_fuzzer', help='Run a fuzzer in the emulated fuzzing environment.') _add_engine_args(run_fuzzer_parser) _add_sanitizer_args(run_fuzzer_parser) _add_environment_args(run_fuzzer_parser) + _add_external_project_args(run_fuzzer_parser) run_fuzzer_parser.add_argument( '--corpus-dir', help='directory to store corpus for the fuzz target') - run_fuzzer_parser.add_argument('project_name', help='name of the project') + run_fuzzer_parser.add_argument('project', + help='name of the project or path (external)') run_fuzzer_parser.add_argument('fuzzer_name', help='name of the fuzzer') run_fuzzer_parser.add_argument('fuzzer_args', help='arguments to pass to the fuzzer', @@ -221,35 +316,40 @@ def get_parser(): # pylint: disable=too-many-statements coverage_parser.add_argument('--corpus-dir', help='specify location of corpus' ' to be used (requires --fuzz-target argument)') - coverage_parser.add_argument('project_name', help='name of the project') + coverage_parser.add_argument('project', + help='name of the project or path (external)') coverage_parser.add_argument('extra_args', help='additional arguments to ' 'pass to llvm-cov utility.', nargs='*') + _add_external_project_args(coverage_parser) download_corpora_parser = subparsers.add_parser( 'download_corpora', help='Download all corpora for a project.') download_corpora_parser.add_argument('--fuzz-target', help='specify name of a fuzz target') - download_corpora_parser.add_argument('project_name', - help='name of the project') + download_corpora_parser.add_argument( + 'project', help='name of the project or path (external)') reproduce_parser = subparsers.add_parser('reproduce', help='Reproduce a crash.') reproduce_parser.add_argument('--valgrind', action='store_true', help='run with valgrind') - reproduce_parser.add_argument('project_name', help='name of the project') + reproduce_parser.add_argument('project', + help='name of the project or path (external)') reproduce_parser.add_argument('fuzzer_name', help='name of the fuzzer') reproduce_parser.add_argument('testcase_path', help='path of local testcase') reproduce_parser.add_argument('fuzzer_args', help='arguments to pass to the fuzzer', nargs=argparse.REMAINDER) _add_environment_args(reproduce_parser) + _add_external_project_args(reproduce_parser) shell_parser = subparsers.add_parser( 'shell', help='Run /bin/bash within the builder container.') - shell_parser.add_argument('project_name', help='name of the project') + shell_parser.add_argument('project', + help='name of the project or path (external)') shell_parser.add_argument('source_path', help='path of local source', nargs='?') @@ -257,6 +357,7 @@ def get_parser(): # pylint: disable=too-many-statements _add_engine_args(shell_parser) _add_sanitizer_args(shell_parser) _add_environment_args(shell_parser) + _add_external_project_args(shell_parser) subparsers.add_parser('pull_images', help='Pull base images.') return parser @@ -267,19 +368,19 @@ def is_base_image(image_name): return os.path.exists(os.path.join('infra', 'base-images', image_name)) -def check_project_exists(project_name): +def check_project_exists(project): """Checks if a project exists.""" - if not os.path.exists(_get_project_dir(project_name)): - logging.error('%s does not exist.', project_name) + if not os.path.exists(project.path): + logging.error('%s does not exist.', project.name) return False return True -def _check_fuzzer_exists(project_name, fuzzer_name): +def _check_fuzzer_exists(project, fuzzer_name): """Checks if a fuzzer exists.""" command = ['docker', 'run', '--rm'] - command.extend(['-v', '%s:/out' % _get_out_dir(project_name)]) + command.extend(['-v', '%s:/out' % project.out]) command.append('ubuntu:16.04') command.extend(['/bin/bash', '-c', 'test -f /out/%s' % fuzzer_name]) @@ -304,56 +405,20 @@ def _get_command_string(command): return ' '.join(pipes.quote(part) for part in command) -def _get_project_dir(project_name): - """Returns path to the project.""" - return os.path.join(OSS_FUZZ_DIR, 'projects', project_name) - - -def get_dockerfile_path(project_name): - """Returns path to the project Dockerfile.""" - return os.path.join(_get_project_dir(project_name), 'Dockerfile') - - -def _get_project_build_subdir(project_name, subdir_name): - """Creates the |subdir_name| subdirectory of the |project_name| subdirectory - in |BUILD_DIR| and returns its path.""" - directory = os.path.join(BUILD_DIR, subdir_name, project_name) +def _get_project_build_subdir(project, subdir_name): + """Creates the |subdir_name| subdirectory of the |project| subdirectory in + |BUILD_DIR| and returns its path.""" + directory = os.path.join(BUILD_DIR, subdir_name, project) if not os.path.exists(directory): os.makedirs(directory) return directory -def _get_corpus_dir(project_name=''): - """Creates and returns path to /corpus directory for the given project (if - specified).""" - return _get_project_build_subdir(project_name, 'corpus') - - -def _get_out_dir(project_name=''): +def _get_out_dir(project=''): """Creates and returns path to /out directory for the given project (if specified).""" - return _get_project_build_subdir(project_name, 'out') - - -def _get_work_dir(project_name=''): - """Creates and returns path to /work directory for the given project (if - specified).""" - return _get_project_build_subdir(project_name, 'work') - - -def _get_project_language(project_name): - """Returns project language.""" - project_yaml_path = os.path.join(OSS_FUZZ_DIR, 'projects', project_name, - 'project.yaml') - with open(project_yaml_path) as file_handle: - content = file_handle.read() - for line in content.splitlines(): - match = PROJECT_LANGUAGE_REGEX.match(line) - if match: - return match.group(1) - - return None + return _get_project_build_subdir(project, 'out') def _add_architecture_args(parser, choices=('x86_64', 'i386')): @@ -386,17 +451,27 @@ def _add_environment_args(parser): help="set environment variable e.g. VAR=value") -def build_image_impl(image_name, cache=True, pull=False): +def build_image_impl(project, cache=True, pull=False): """Builds image.""" - proj_is_base_image = is_base_image(image_name) - if proj_is_base_image: + image_name = project.name + + if is_base_image(image_name): image_project = 'oss-fuzz-base' - dockerfile_dir = os.path.join('infra', 'base-images', image_name) - else: + docker_build_dir = os.path.join(OSS_FUZZ_DIR, 'infra', 'base-images', + image_name) + docker_file_path = None + elif project.is_external: + # External projects need to use the repo root as the build directory. + docker_file_path = os.path.join(project.build_integration_path, + 'Dockerfile') + docker_build_dir = project.path image_project = 'oss-fuzz' - if not check_project_exists(image_name): + else: + if not check_project_exists(project): return False - dockerfile_dir = os.path.join('projects', image_name) + docker_file_path = None + docker_build_dir = project.path + image_project = 'oss-fuzz' if pull and not pull_images(): return False @@ -405,9 +480,13 @@ def build_image_impl(image_name, cache=True, pull=False): if not cache: build_args.append('--no-cache') - build_args += [ - '-t', 'gcr.io/%s/%s' % (image_project, image_name), dockerfile_dir - ] + build_args += ['-t', 'gcr.io/%s/%s' % (image_project, image_name)] + if docker_file_path: + build_args += [ + '--file', + docker_file_path, + ] + build_args.append(docker_build_dir) return docker_build(build_args) @@ -432,14 +511,12 @@ def workdir_from_lines(lines, default='/src'): return default -def _workdir_from_dockerfile(project_name): +def _workdir_from_dockerfile(project): """Parses WORKDIR from the Dockerfile for the given project.""" - dockerfile_path = get_dockerfile_path(project_name) - - with open(dockerfile_path) as file_handle: + with open(project.dockerfile_path) as file_handle: lines = file_handle.readlines() - return workdir_from_lines(lines, default=os.path.join('/src', project_name)) + return workdir_from_lines(lines, default=os.path.join('/src', project.name)) def docker_run(run_args, print_output=True): @@ -514,14 +591,14 @@ def build_image(args): logging.error('Using cached base images...') # If build_image is called explicitly, don't use cache. - if build_image_impl(args.project_name, cache=args.cache, pull=pull): + if build_image_impl(args.project, cache=args.cache, pull=pull): return True return False def build_fuzzers_impl( # pylint: disable=too-many-arguments,too-many-locals,too-many-branches - project_name, + project, clean, engine, sanitizer, @@ -530,29 +607,23 @@ def build_fuzzers_impl( # pylint: disable=too-many-arguments,too-many-locals,to source_path, mount_path=None): """Builds fuzzers.""" - if not build_image_impl(project_name): + if not build_image_impl(project): return False - project_out_dir = _get_out_dir(project_name) - project_work_dir = _get_work_dir(project_name) - project_language = _get_project_language(project_name) - if not project_language: - logging.warning('Language not specified in project.yaml. Build may fail.') - if clean: logging.info('Cleaning existing build artifacts.') # Clean old and possibly conflicting artifacts in project's out directory. docker_run([ '-v', - '%s:/out' % project_out_dir, '-t', - 'gcr.io/oss-fuzz/%s' % project_name, '/bin/bash', '-c', 'rm -rf /out/*' + '%s:/out' % project.out, '-t', + 'gcr.io/oss-fuzz/%s' % project.name, '/bin/bash', '-c', 'rm -rf /out/*' ]) docker_run([ '-v', - '%s:/work' % project_work_dir, '-t', - 'gcr.io/oss-fuzz/%s' % project_name, '/bin/bash', '-c', 'rm -rf /work/*' + '%s:/work' % project.work, '-t', + 'gcr.io/oss-fuzz/%s' % project.name, '/bin/bash', '-c', 'rm -rf /work/*' ]) else: @@ -565,8 +636,8 @@ def build_fuzzers_impl( # pylint: disable=too-many-arguments,too-many-locals,to _add_oss_fuzz_ci_if_needed(env) - if project_language: - env.append('FUZZING_LANGUAGE=' + project_language) + if project.language: + env.append('FUZZING_LANGUAGE=' + project.language) if env_to_add: env += env_to_add @@ -575,14 +646,14 @@ def build_fuzzers_impl( # pylint: disable=too-many-arguments,too-many-locals,to if sanitizer == 'memory': docker_run([ '-v', - '%s:/work' % project_work_dir, 'gcr.io/oss-fuzz-base/msan-libs-builder', + '%s:/work' % project.work, 'gcr.io/oss-fuzz-base/msan-libs-builder', 'bash', '-c', 'cp -r /msan /work' ]) env.append('MSAN_LIBS_PATH=' + '/work/msan') command = ['--cap-add', 'SYS_PTRACE'] + _env_to_docker_args(env) if source_path: - workdir = _workdir_from_dockerfile(project_name) + workdir = _workdir_from_dockerfile(project) if mount_path: command += [ '-v', @@ -600,9 +671,9 @@ def build_fuzzers_impl( # pylint: disable=too-many-arguments,too-many-locals,to command += [ '-v', - '%s:/out' % project_out_dir, '-v', - '%s:/work' % project_work_dir, '-t', - 'gcr.io/oss-fuzz/%s' % project_name + '%s:/out' % project.out, '-v', + '%s:/work' % project.work, '-t', + 'gcr.io/oss-fuzz/%s' % project.name ] result = docker_run(command) @@ -612,21 +683,19 @@ def build_fuzzers_impl( # pylint: disable=too-many-arguments,too-many-locals,to # Patch MSan builds to use instrumented shared libraries. if sanitizer == 'memory': - docker_run([ - '-v', - '%s:/out' % project_out_dir, '-v', - '%s:/work' % project_work_dir - ] + _env_to_docker_args(env) + [ - 'gcr.io/oss-fuzz-base/base-sanitizer-libs-builder', 'patch_build.py', - '/out' - ]) + docker_run( + ['-v', '%s:/out' % project.out, '-v', + '%s:/work' % project.work] + _env_to_docker_args(env) + [ + 'gcr.io/oss-fuzz-base/base-sanitizer-libs-builder', + 'patch_build.py', '/out' + ]) return True def build_fuzzers(args): """Builds fuzzers.""" - return build_fuzzers_impl(args.project_name, + return build_fuzzers_impl(args.project, args.clean, args.engine, args.sanitizer, @@ -645,15 +714,15 @@ def _add_oss_fuzz_ci_if_needed(env): def check_build(args): """Checks that fuzzers in the container execute without errors.""" - if not check_project_exists(args.project_name): + if not check_project_exists(args.project): return False if (args.fuzzer_name and - not _check_fuzzer_exists(args.project_name, args.fuzzer_name)): + not _check_fuzzer_exists(args.project, args.fuzzer_name)): return False - fuzzing_language = _get_project_language(args.project_name) - if fuzzing_language is None: + fuzzing_language = args.project.language + if not fuzzing_language: logging.warning( 'Language not specified in project.yaml. Defaulting to C++.') fuzzing_language = 'c++' @@ -670,8 +739,7 @@ def check_build(args): run_args = _env_to_docker_args(env) + [ '-v', - '%s:/out' % _get_out_dir(args.project_name), '-t', - 'gcr.io/oss-fuzz-base/base-runner' + '%s:/out' % args.project.out, '-t', 'gcr.io/oss-fuzz-base/base-runner' ] if args.fuzzer_name: @@ -688,10 +756,10 @@ def check_build(args): return result -def _get_fuzz_targets(project_name): +def _get_fuzz_targets(project): """Returns names of fuzz targest build in the project's /out directory.""" fuzz_targets = [] - for name in os.listdir(_get_out_dir(project_name)): + for name in os.listdir(project.out): if name.startswith('afl-'): continue if name.startswith('jazzer_'): @@ -699,7 +767,7 @@ def _get_fuzz_targets(project_name): if name == 'llvm-symbolizer': continue - path = os.path.join(_get_out_dir(project_name), name) + path = os.path.join(project.out, name) # Python and JVM fuzz targets are only executable for the root user, so # we can't use os.access. if os.path.isfile(path) and (os.stat(path).st_mode & 0o111): @@ -750,7 +818,7 @@ def _get_latest_corpus(project_name, fuzz_target, base_corpus_dir): def download_corpora(args): """Downloads most recent corpora from GCS for the given project.""" - if not check_project_exists(args.project_name): + if not check_project_exists(args.project): return False try: @@ -764,22 +832,20 @@ def download_corpora(args): if args.fuzz_target: fuzz_targets = [args.fuzz_target] else: - fuzz_targets = _get_fuzz_targets(args.project_name) + fuzz_targets = _get_fuzz_targets(args.project) - corpus_dir = _get_corpus_dir(args.project_name) - if not os.path.exists(corpus_dir): - os.makedirs(corpus_dir) + corpus_dir = args.project.corpus def _download_for_single_target(fuzz_target): try: - _get_latest_corpus(args.project_name, fuzz_target, corpus_dir) + _get_latest_corpus(args.project, fuzz_target, corpus_dir) return True except Exception as error: # pylint:disable=broad-except logging.error('Corpus download for %s failed: %s.', fuzz_target, str(error)) return False - logging.info('Downloading corpora for %s project to %s.', args.project_name, + logging.info('Downloading corpora for %s project to %s.', args.project.name, corpus_dir) thread_pool = ThreadPool() return all(thread_pool.map(_download_for_single_target, fuzz_targets)) @@ -793,24 +859,24 @@ def coverage(args): '--fuzz-target') return False - if not check_project_exists(args.project_name): + if not check_project_exists(args.project): return False - project_language = _get_project_language(args.project_name) - if project_language not in LANGUAGES_WITH_COVERAGE_SUPPORT: + if args.project.language not in LANGUAGES_WITH_COVERAGE_SUPPORT: logging.error( 'Project is written in %s, coverage for it is not supported yet.', - project_language) + args.project.language) return False - if not args.no_corpus_download and not args.corpus_dir: + if (not args.no_corpus_download and not args.corpus_dir and + not args.project.is_external): if not download_corpora(args): return False env = [ 'FUZZING_ENGINE=libfuzzer', - 'FUZZING_LANGUAGE=%s' % project_language, - 'PROJECT=%s' % args.project_name, + 'FUZZING_LANGUAGE=%s' % args.project.language, + 'PROJECT=%s' % args.project.name, 'SANITIZER=coverage', 'HTTP_PORT=%s' % args.port, 'COVERAGE_EXTRA_ARGS=%s' % ' '.join(args.extra_args), @@ -832,11 +898,11 @@ def coverage(args): corpus_dir = os.path.realpath(args.corpus_dir) run_args.extend(['-v', '%s:/corpus/%s' % (corpus_dir, args.fuzz_target)]) else: - run_args.extend(['-v', '%s:/corpus' % _get_corpus_dir(args.project_name)]) + run_args.extend(['-v', '%s:/corpus' % args.project.corpus]) run_args.extend([ '-v', - '%s:/out' % _get_out_dir(args.project_name), + '%s:/out' % args.project.out, '-t', 'gcr.io/oss-fuzz-base/base-runner', ]) @@ -856,10 +922,10 @@ def coverage(args): def run_fuzzer(args): """Runs a fuzzer in the container.""" - if not check_project_exists(args.project_name): + if not check_project_exists(args.project): return False - if not _check_fuzzer_exists(args.project_name, args.fuzzer_name): + if not _check_fuzzer_exists(args.project, args.fuzzer_name): return False env = [ @@ -886,7 +952,7 @@ def run_fuzzer(args): run_args.extend([ '-v', - '%s:/out' % _get_out_dir(args.project_name), + '%s:/out' % args.project.out, '-t', 'gcr.io/oss-fuzz-base/base-runner', 'run_fuzzer', @@ -898,12 +964,12 @@ def run_fuzzer(args): def reproduce(args): """Reproduces a specific test case from a specific project.""" - return reproduce_impl(args.project_name, args.fuzzer_name, args.valgrind, - args.e, args.fuzzer_args, args.testcase_path) + return reproduce_impl(args.project, args.fuzzer_name, args.valgrind, args.e, + args.fuzzer_args, args.testcase_path) def reproduce_impl( # pylint: disable=too-many-arguments - project_name, + project, fuzzer_name, valgrind, env_to_add, @@ -912,10 +978,10 @@ def reproduce_impl( # pylint: disable=too-many-arguments run_function=docker_run, err_result=False): """Reproduces a testcase in the container.""" - if not check_project_exists(project_name): + if not check_project_exists(project): return err_result - if not _check_fuzzer_exists(project_name, fuzzer_name): + if not _check_fuzzer_exists(project, fuzzer_name): return err_result debugger = '' @@ -934,7 +1000,7 @@ def reproduce_impl( # pylint: disable=too-many-arguments run_args = _env_to_docker_args(env) + [ '-v', - '%s:/out' % _get_out_dir(project_name), + '%s:/out' % project.out, '-v', '%s:/testcase' % _get_absolute_path(testcase_path), '-t', @@ -947,51 +1013,85 @@ def reproduce_impl( # pylint: disable=too-many-arguments return run_function(run_args) -def generate(args): - """Generates empty project files.""" - if len(args.project_name) > MAX_PROJECT_NAME_LENGTH: +def _validate_project_name(project_name): + """Validates |project_name| is a valid OSS-Fuzz project name.""" + if len(project_name) > MAX_PROJECT_NAME_LENGTH: logging.error( 'Project name needs to be less than or equal to %d characters.', MAX_PROJECT_NAME_LENGTH) return False - if not VALID_PROJECT_NAME_REGEX.match(args.project_name): - logging.error('Invalid project name.') + if not VALID_PROJECT_NAME_REGEX.match(project_name): + logging.info('Invalid project name: %s.', project_name) return False - directory = os.path.join('projects', args.project_name) + return True + +def _create_build_integration_directory(directory): + """Returns True on successful creation of a build integration directory. + Suitable for OSS-Fuzz and external projects.""" try: - os.mkdir(directory) + os.makedirs(directory) except OSError as error: if error.errno != errno.EEXIST: raise logging.error('%s already exists.', directory) return False + return True - logging.info('Writing new files to %s.', directory) + +def _template_project_file(filename, template, template_args, directory): + """Templates |template| using |template_args| and writes the result to + |directory|/|filename|. Sets the file to executable if |filename| is + build.sh.""" + file_path = os.path.join(directory, filename) + with open(file_path, 'w') as file_handle: + file_handle.write(template % template_args) + + if filename == 'build.sh': + os.chmod(file_path, 0o755) + + +def generate(args): + """Generates empty project files.""" + return _generate_impl(args.project) + + +def _get_current_datetime(): + """Returns this year. Needed for mocking.""" + return datetime.datetime.now() + + +def _generate_impl(project): + """Implementation of generate(). Useful for testing.""" + if project.is_external: + # External project. + project_templates = templates.EXTERNAL_TEMPLATES + else: + # Internal project. + if not _validate_project_name(project.name): + return False + project_templates = templates.TEMPLATES + + directory = project.build_integration_path + if not _create_build_integration_directory(directory): + return False + + logging.info('Writing new files to: %s.', directory) template_args = { - 'project_name': args.project_name, - 'year': datetime.datetime.now().year + 'project_name': project.name, + 'year': _get_current_datetime().year } - with open(os.path.join(directory, 'project.yaml'), 'w') as file_handle: - file_handle.write(templates.PROJECT_YAML_TEMPLATE % template_args) - - with open(os.path.join(directory, 'Dockerfile'), 'w') as file_handle: - file_handle.write(templates.DOCKER_TEMPLATE % template_args) - - build_sh_path = os.path.join(directory, 'build.sh') - with open(build_sh_path, 'w') as file_handle: - file_handle.write(templates.BUILD_TEMPLATE % template_args) - - os.chmod(build_sh_path, 0o755) + for filename, template in project_templates.items(): + _template_project_file(filename, template, template_args, directory) return True def shell(args): """Runs a shell within a docker image.""" - if not build_image_impl(args.project_name): + if not build_image_impl(args.project): return False env = [ @@ -1000,18 +1100,18 @@ def shell(args): 'ARCHITECTURE=' + args.architecture, ] - if args.project_name != 'base-runner-debug': - env.append('FUZZING_LANGUAGE=' + _get_project_language(args.project_name)) + if args.project.name != 'base-runner-debug': + env.append('FUZZING_LANGUAGE=' + args.project.language) if args.e: env += args.e - if is_base_image(args.project_name): + if is_base_image(args.project.name): image_project = 'oss-fuzz-base' out_dir = _get_out_dir() else: image_project = 'oss-fuzz' - out_dir = _get_out_dir(args.project_name) + out_dir = args.project.out run_args = _env_to_docker_args(env) if args.source_path: @@ -1023,8 +1123,8 @@ def shell(args): run_args.extend([ '-v', '%s:/out' % out_dir, '-v', - '%s:/work' % _get_work_dir(args.project_name), '-t', - 'gcr.io/%s/%s' % (image_project, args.project_name), '/bin/bash' + '%s:/work' % args.project.work, '-t', + 'gcr.io/%s/%s' % (image_project, args.project.name), '/bin/bash' ]) docker_run(run_args) diff --git a/infra/helper_test.py b/infra/helper_test.py index 7ebbc3adf..cb9e6a659 100644 --- a/infra/helper_test.py +++ b/infra/helper_test.py @@ -13,18 +13,25 @@ # limitations under the License. """Tests for helper.py""" +import datetime +import os import unittest from unittest import mock +from pyfakefs import fake_filesystem_unittest + import helper +import templates + +# pylint: disable=no-self-use,protected-access -class TestShell(unittest.TestCase): +class ShellTest(unittest.TestCase): """Tests 'shell' command.""" @mock.patch('helper.docker_run') @mock.patch('helper.build_image_impl') - def test_base_runner_debug(self, mocked_build_image_impl, _): + def test_base_runner_debug(self, _, __): """Tests that shell base-runner-debug works as intended.""" image_name = 'base-runner-debug' unparsed_args = ['shell', image_name] @@ -32,5 +39,95 @@ class TestShell(unittest.TestCase): args = helper.parse_args(parser, unparsed_args) args.sanitizer = 'address' result = helper.shell(args) - mocked_build_image_impl.assert_called_with(image_name) self.assertTrue(result) + + +class BuildImageImplTest(unittest.TestCase): + """Tests for build_image_impl.""" + + @mock.patch('helper.docker_build') + def test_no_cache(self, mocked_docker_build): + """Tests that cache=False is handled properly.""" + image_name = 'base-image' + helper.build_image_impl(helper.Project(image_name), cache=False) + self.assertIn('--no-cache', mocked_docker_build.call_args_list[0][0][0]) + + @mock.patch('helper.docker_build') + @mock.patch('helper.pull_images') + def test_pull(self, mocked_pull_images, _): + """Tests that pull=True is handled properly.""" + image_name = 'base-image' + helper.build_image_impl(helper.Project(image_name), pull=True) + mocked_pull_images.assert_called_with() + + @mock.patch('helper.docker_build') + def test_base_image(self, mocked_docker_build): + """Tests that build_image_impl works as intended with a base-image.""" + image_name = 'base-image' + helper.build_image_impl(helper.Project(image_name)) + mocked_docker_build.assert_called_with([ + '-t', 'gcr.io/oss-fuzz-base/base-image', + os.path.join(helper.OSS_FUZZ_DIR, 'infra/base-images/base-image') + ]) + + @mock.patch('helper.docker_build') + def test_oss_fuzz_project(self, mocked_docker_build): + """Tests that build_image_impl works as intended with an OSS-Fuzz + project.""" + project_name = 'example' + helper.build_image_impl(helper.Project(project_name)) + mocked_docker_build.assert_called_with([ + '-t', 'gcr.io/oss-fuzz/example', + os.path.join(helper.OSS_FUZZ_DIR, 'projects/example') + ]) + + @mock.patch('helper.docker_build') + def test_external_project(self, mocked_docker_build): + """Tests that build_image_impl works as intended with a non-OSS-Fuzz + project.""" + project_src_path = '/example' + build_integration_path = 'build-integration' + project = helper.Project(project_src_path, + is_external=True, + build_integration_path=build_integration_path) + helper.build_image_impl(project) + mocked_docker_build.assert_called_with([ + '-t', 'gcr.io/oss-fuzz/example', '--file', + '/example/build-integration/Dockerfile', project_src_path + ]) + + +class GenerateImplTest(fake_filesystem_unittest.TestCase): + """Tests for _generate_impl.""" + PROJECT_NAME = 'newfakeproject' + + def setUp(self): + self.setUpPyfakefs() + self.fs.add_real_directory(helper.OSS_FUZZ_DIR) + + def _verify_templated_files(self, template_dict, directory): + template_args = {'project_name': self.PROJECT_NAME, 'year': 2021} + for filename, template in template_dict.items(): + file_path = os.path.join(directory, filename) + with open(file_path, 'r') as file_handle: + contents = file_handle.read() + self.assertEqual(contents, template % template_args) + + @mock.patch('helper._get_current_datetime', + return_value=datetime.datetime(year=2021, month=1, day=1)) + def test_generate_oss_fuzz_project(self, _): + """Tests that the correct files are generated for an OSS-Fuzz project.""" + helper._generate_impl(helper.Project(self.PROJECT_NAME)) + self._verify_templated_files( + templates.TEMPLATES, + os.path.join(helper.OSS_FUZZ_DIR, 'projects', self.PROJECT_NAME)) + + def test_generate_external_project(self): + """Tests that the correct files are generated for a non-OSS-Fuzz project.""" + build_integration_path = '/newfakeproject/build-integration' + helper._generate_impl( + helper.Project('/newfakeproject/', + is_external=True, + build_integration_path=build_integration_path)) + self._verify_templated_files(templates.EXTERNAL_TEMPLATES, + build_integration_path) diff --git a/infra/templates.py b/infra/templates.py index f16da924f..66e3e78c2 100755 --- a/infra/templates.py +++ b/infra/templates.py @@ -46,6 +46,14 @@ WORKDIR %(project_name)s COPY build.sh $SRC/ """ +EXTERNAL_DOCKER_TEMPLATE = """\ +FROM gcr.io/oss-fuzz-base/base-builder +RUN apt-get update && apt-get install -y make autoconf automake libtool +RUN git clone --depth 1 %(project_name)s # or use other version control +WORKDIR %(project_name)s +COPY build.sh $SRC/ +""" + BUILD_TEMPLATE = """\ #!/bin/bash -eu # Copyright %(year)d Google LLC @@ -76,3 +84,28 @@ BUILD_TEMPLATE = """\ # /path/to/name_of_fuzzer.cc -o $OUT/name_of_fuzzer \\ # $LIB_FUZZING_ENGINE /path/to/library.a """ + +EXTERNAL_BUILD_TEMPLATE = """\ +# build project +# e.g. +# ./autogen.sh +# ./configure +# make -j$(nproc) all + +# build fuzzers +# e.g. +# $CXX $CXXFLAGS -std=c++11 -Iinclude \\ +# /path/to/name_of_fuzzer.cc -o $OUT/name_of_fuzzer \\ +# $LIB_FUZZING_ENGINE /path/to/library.a +""" + +TEMPLATES = { + 'build.sh': BUILD_TEMPLATE, + 'Dockerfile': DOCKER_TEMPLATE, + 'project.yaml': PROJECT_YAML_TEMPLATE +} + +EXTERNAL_TEMPLATES = { + 'build.sh': EXTERNAL_BUILD_TEMPLATE, + 'Dockerfile': EXTERNAL_DOCKER_TEMPLATE +}