From 1ab5e408e5253db7058c5050a7ee15216c078698 Mon Sep 17 00:00:00 2001 From: jonathanmetzman <31354670+jonathanmetzman@users.noreply.github.com> Date: Sun, 18 Sep 2022 08:04:33 -0400 Subject: [PATCH] [trial builds] Support forks (#8519) Fix https://github.com/google/oss-fuzz/issues/8273 --- infra/build/functions/build_and_run_coverage.py | 10 ++++------ .../functions/build_and_run_coverage_test.py | 2 +- infra/build/functions/build_lib.py | 9 ++++----- infra/build/functions/build_project.py | 16 +++++++++------- infra/build/functions/build_project_test.py | 4 ++-- infra/build/functions/ci_trial_build.py | 8 ++++++-- infra/build/functions/request_build.py | 2 +- infra/build/functions/trial_build.py | 5 +++++ .../build/functions/trial_build/cloudbuild.yaml | 1 + 9 files changed, 33 insertions(+), 24 deletions(-) diff --git a/infra/build/functions/build_and_run_coverage.py b/infra/build/functions/build_and_run_coverage.py index 004403c1f..e93680a84 100755 --- a/infra/build/functions/build_and_run_coverage.py +++ b/infra/build/functions/build_and_run_coverage.py @@ -100,12 +100,10 @@ def get_build_steps( # pylint: disable=too-many-locals, too-many-arguments report_date = build_project.get_datetime_now().strftime('%Y%m%d') bucket = CoverageBucket(project.name, report_date, PLATFORM, config.testing) - build_steps = build_lib.get_project_image_steps( - project.name, - project.image, - project.fuzzing_language, - branch=config.branch, - test_image_suffix=config.test_image_suffix) + build_steps = build_lib.get_project_image_steps(project.name, + project.image, + project.fuzzing_language, + config=config) build = build_project.Build(FUZZING_ENGINE, 'coverage', ARCHITECTURE) env = build_project.get_env(project.fuzzing_language, build) diff --git a/infra/build/functions/build_and_run_coverage_test.py b/infra/build/functions/build_and_run_coverage_test.py index 2951cab41..29fb1fa5c 100644 --- a/infra/build/functions/build_and_run_coverage_test.py +++ b/infra/build/functions/build_and_run_coverage_test.py @@ -67,7 +67,7 @@ class TestRequestCoverageBuilds(fake_filesystem_unittest.TestCase): with open(expected_build_steps_file_path) as expected_build_steps_file: expected_coverage_build_steps = json.load(expected_build_steps_file) - config = build_project.Config(False, False, None, False, False) + config = build_project.Config(upload=False) project_yaml, dockerfile = build_project.get_project_data( test_utils.PROJECT) build_steps = build_and_run_coverage.get_build_steps( diff --git a/infra/build/functions/build_lib.py b/infra/build/functions/build_lib.py index 065a30e72..45298a0e5 100644 --- a/infra/build/functions/build_lib.py +++ b/infra/build/functions/build_lib.py @@ -416,18 +416,17 @@ def get_project_image_steps( # pylint: disable=too-many-arguments name, image, language, - branch=None, - test_image_suffix=None, + config, architectures=None): """Returns GCB steps to build OSS-Fuzz project image.""" if architectures is None: architectures = [] # TODO(metzman): Pass the URL to clone. - clone_step = get_git_clone_step(branch=branch) + clone_step = get_git_clone_step(repo_url=config.repo, branch=config.branch) steps = [clone_step] - if test_image_suffix: - steps.extend(get_pull_test_images_steps(test_image_suffix)) + if config.test_image_suffix: + steps.extend(get_pull_test_images_steps(config.test_image_suffix)) docker_build_step = get_docker_build_step([image], os.path.join('projects', name)) steps.append(docker_build_step) diff --git a/infra/build/functions/build_project.py b/infra/build/functions/build_project.py index 18f3d69aa..23ec94810 100755 --- a/infra/build/functions/build_project.py +++ b/infra/build/functions/build_project.py @@ -54,8 +54,11 @@ PROJECTS_DIR = os.path.abspath( os.path.join(__file__, os.path.pardir, os.path.pardir, os.path.pardir, os.path.pardir, 'projects')) +DEFAULT_OSS_FUZZ_REPO = 'https://github.com/google/oss-fuzz.git' Config = collections.namedtuple( - 'Config', ['testing', 'test_image_suffix', 'branch', 'parallel', 'upload']) + 'Config', + ['testing', 'test_image_suffix', 'repo', 'branch', 'parallel', 'upload'], + defaults=(False, None, DEFAULT_OSS_FUZZ_REPO, None, False, True)) WORKDIR_REGEX = re.compile(r'\s*WORKDIR\s*([^\s]+)') @@ -289,8 +292,7 @@ def get_build_steps( # pylint: disable=too-many-locals, too-many-statements, to project.name, project.image, project.fuzzing_language, - branch=config.branch, - test_image_suffix=config.test_image_suffix, + config=config, architectures=project.architectures) # Sort engines to make AFL first to test if libFuzzer has an advantage in @@ -528,10 +530,10 @@ def build_script_main(script_description, get_build_steps_func, build_type): credentials = oauth2client.client.GoogleCredentials.get_application_default() error = False - config = Config(args.testing, - args.test_image_suffix, - args.branch, - args.parallel, + config = Config(testing=args.testing, + test_image_suffix=args.test_image_suffix, + branch=args.branch, + parallel=args.parallel, upload=True) for project_name in args.projects: logging.info('Getting steps for: "%s".', project_name) diff --git a/infra/build/functions/build_project_test.py b/infra/build/functions/build_project_test.py index 33961c1e3..91be5793a 100644 --- a/infra/build/functions/build_project_test.py +++ b/infra/build/functions/build_project_test.py @@ -65,7 +65,7 @@ class TestRequestCoverageBuilds(fake_filesystem_unittest.TestCase): with open(expected_build_steps_file_path) as expected_build_steps_file: expected_build_steps = json.load(expected_build_steps_file) - config = build_project.Config(False, False, None, False, True) + config = build_project.Config(upload=True) project_yaml, dockerfile = build_project.get_project_data( test_utils.PROJECT) build_steps = build_project.get_build_steps(test_utils.PROJECT, @@ -101,7 +101,7 @@ class TestRequestCoverageBuilds(fake_filesystem_unittest.TestCase): with open(expected_build_steps_file_path) as expected_build_steps_file: expected_build_steps = json.load(expected_build_steps_file) - config = build_project.Config(False, False, None, False, True) + config = build_project.Config(upload=True) project_yaml, dockerfile = build_project.get_project_data( test_utils.PROJECT) build_steps = build_project.get_build_steps(test_utils.PROJECT, diff --git a/infra/build/functions/ci_trial_build.py b/infra/build/functions/ci_trial_build.py index abb494e5f..0359b2a7d 100644 --- a/infra/build/functions/ci_trial_build.py +++ b/infra/build/functions/ci_trial_build.py @@ -57,11 +57,12 @@ def get_latest_gcbrun_command(comments): return None -def exec_command_from_github(pull_request_number, branch): +def exec_command_from_github(pull_request_number, repo, branch): """Executes the gcbrun command for trial_build.py in the most recent command on |pull_request_number|.""" comments = get_comments(pull_request_number) command = get_latest_gcbrun_command(comments) + command.extend(['--repo', repo]) logging.info('Command: %s.', command) if command is None: logging.info('Trial build not requested.') @@ -79,7 +80,10 @@ def main(): logging.basicConfig(level=logging.INFO) pull_request_number = int(os.environ['PULL_REQUEST_NUMBER']) branch = os.environ['BRANCH'] - return 0 if exec_command_from_github(pull_request_number, branch) else 1 + repo = os.environ['REPO'] + if exec_command_from_github(pull_request_number, repo, branch): + return 0 + return 1 if __name__ == '__main__': diff --git a/infra/build/functions/request_build.py b/infra/build/functions/request_build.py index f5dc4b454..b4574f616 100644 --- a/infra/build/functions/request_build.py +++ b/infra/build/functions/request_build.py @@ -61,7 +61,7 @@ def get_project_data(project_name): def get_empty_config(): """Returns an empty build config.""" - return build_project.Config(False, None, None, False, True) + return build_project.Config() def get_build_steps(project_name, image_project, base_images_project): diff --git a/infra/build/functions/trial_build.py b/infra/build/functions/trial_build.py index 6676b2866..7ba0e1e98 100644 --- a/infra/build/functions/trial_build.py +++ b/infra/build/functions/trial_build.py @@ -112,6 +112,10 @@ def get_args(args=None): default=['afl', 'libfuzzer', 'honggfuzz', 'centipede'], nargs='+', help='Fuzzing engines.') + parser.add_argument('--repo', + required=False, + default=build_project.DEFAULT_OSS_FUZZ_REPO, + help='Use specified OSS-Fuzz repo.') parser.add_argument('--branch', required=False, default=None, @@ -275,6 +279,7 @@ def _do_test_builds(args, test_image_suffix): args.force_build) config = build_project.Config(testing=True, test_image_suffix=test_image_suffix, + repo=args.repo, branch=args.branch, parallel=False, upload=False) diff --git a/infra/build/functions/trial_build/cloudbuild.yaml b/infra/build/functions/trial_build/cloudbuild.yaml index 5a5a7d049..e6ce90f0a 100644 --- a/infra/build/functions/trial_build/cloudbuild.yaml +++ b/infra/build/functions/trial_build/cloudbuild.yaml @@ -12,6 +12,7 @@ steps: env: - 'PULL_REQUEST_NUMBER=${_PR_NUMBER}' - 'BRANCH=${_HEAD_BRANCH}' + - 'REPO=${_HEAD_REPO_URL}' timeout: 21600s # 6 hours options: logging: CLOUD_LOGGING_ONLY