diff --git a/infra/cifuzz/build_fuzzers.py b/infra/cifuzz/build_fuzzers.py index 10a9d3790..efb7c548b 100644 --- a/infra/cifuzz/build_fuzzers.py +++ b/infra/cifuzz/build_fuzzers.py @@ -72,6 +72,7 @@ class Builder: # pylint: disable=too-many-instance-attributes return False self.image_repo_path = result.image_repo_path self.repo_manager = result.repo_manager + logging.info('repo_dir: %s.', self.repo_manager.repo_dir) self.host_repo_path = self.repo_manager.repo_dir return True diff --git a/infra/cifuzz/build_fuzzers_test.py b/infra/cifuzz/build_fuzzers_test.py index 0b885d524..ea52b2ca2 100644 --- a/infra/cifuzz/build_fuzzers_test.py +++ b/infra/cifuzz/build_fuzzers_test.py @@ -30,6 +30,7 @@ OSS_FUZZ_DIR = os.path.dirname(INFRA_DIR) import build_fuzzers import config_utils import continuous_integration +import repo_manager import test_helpers # NOTE: This integration test relies on @@ -169,6 +170,30 @@ class BuildFuzzersIntegrationTest(unittest.TestCase): build_integration_path=build_integration_path, git_url=git_url, commit_sha='HEAD', + is_github=True, + base_commit='HEAD^1') + self.assertTrue(build_fuzzers.build_fuzzers(config)) + self.assertTrue( + os.path.exists(os.path.join(self.out_dir, EXAMPLE_BUILD_FUZZER))) + + def test_external_generic_project(self): + """Tests building fuzzers from an external project not on Github.""" + project_name = 'cifuzz-external-example' + build_integration_path = 'fuzzer-build-integration' + git_url = 'https://github.com/jonathanmetzman/cifuzz-external-example.git' + # This test is dependant on the state of + # github.com/jonathanmetzman/cifuzz-external-example. + manager = repo_manager.clone_repo_and_get_manager( + 'https://github.com/jonathanmetzman/cifuzz-external-example', + self.tmp_dir_obj.name) + project_src_path = manager.repo_dir + config = create_config(project_name=project_name, + project_repo_name=project_name, + workspace=self.workspace, + build_integration_path=build_integration_path, + git_url=git_url, + commit_sha='HEAD', + project_src_path=project_src_path, base_commit='HEAD^1') self.assertTrue(build_fuzzers.build_fuzzers(config)) self.assertTrue( diff --git a/infra/cifuzz/config_utils.py b/infra/cifuzz/config_utils.py index ad2cd36c6..941feb7d4 100644 --- a/infra/cifuzz/config_utils.py +++ b/infra/cifuzz/config_utils.py @@ -88,6 +88,7 @@ class BaseConfig: EXTERNAL_GITHUB = 0 # Non-OSS-Fuzz on GitHub actions. INTERNAL_GITHUB = 1 # OSS-Fuzz on GitHub actions. INTERNAL_GENERIC_CI = 2 # OSS-Fuzz on any CI. + EXTERNAL_GENERIC_CI = 3 # Non-OSS-Fuzz on any CI. def __init__(self): self.workspace = os.getenv('GITHUB_WORKSPACE') @@ -112,7 +113,10 @@ class BaseConfig: def platform(self): """Returns the platform CIFuzz is runnning on.""" if not self.is_internal: + if not self.is_github: + return self.Platform.EXTERNAL_GENERIC_CI return self.Platform.EXTERNAL_GITHUB + if self.is_github: return self.Platform.INTERNAL_GITHUB return self.Platform.INTERNAL_GENERIC_CI diff --git a/infra/cifuzz/continuous_integration.py b/infra/cifuzz/continuous_integration.py index b2e8af28e..c90aca0b1 100644 --- a/infra/cifuzz/continuous_integration.py +++ b/infra/cifuzz/continuous_integration.py @@ -69,6 +69,11 @@ class BaseCi: def get_ci(config): """Determines what kind of CI is being used and returns the object representing that system.""" + + if config.platform == config.Platform.EXTERNAL_GENERIC_CI: + # Non-OSS-Fuzz projects must bring their own source and their own build + # integration (which is relative to that source). + return ExternalGeneric(config) if config.platform == config.Platform.EXTERNAL_GITHUB: # Non-OSS-Fuzz projects must bring their own source and their own build # integration (which is relative to that source). @@ -132,7 +137,9 @@ class InternalGithub(GithubCiMixin, BaseCi): if not inferred_url or not image_repo_path: logging.error('Could not detect repo from project %s.', self.config.project_name) - return BuildPreparationResult(False, None, None) + return BuildPreparationResult(success=False, + image_repo_path=None, + repo_manager=None) git_workspace = os.path.join(self.config.workspace, 'storage') os.makedirs(git_workspace, exist_ok=True) @@ -147,7 +154,9 @@ class InternalGithub(GithubCiMixin, BaseCi): checkout_specified_commit(manager, self.config.pr_ref, self.config.commit_sha) - return BuildPreparationResult(True, image_repo_path, manager) + return BuildPreparationResult(success=True, + image_repo_path=image_repo_path, + repo_manager=manager) class InternalGeneric(BaseCi): @@ -167,10 +176,14 @@ class InternalGeneric(BaseCi): if not image_repo_path: logging.error('Could not detect repo from project %s.', self.config.project_name) - return BuildPreparationResult(False, None, None) + return BuildPreparationResult(success=False, + image_repo_path=None, + repo_manager=None) manager = repo_manager.RepoManager(self.config.project_src_path) - return BuildPreparationResult(True, image_repo_path, manager) + return BuildPreparationResult(success=True, + image_repo_path=image_repo_path, + repo_manager=manager) def get_diff_base(self): return 'origin...' @@ -191,6 +204,31 @@ def build_external_project_docker_image(project_name, project_src, return helper.docker_build(command) +class ExternalGeneric(BaseCi): + """CI implementation for generic CI for external (non-OSS-Fuzz) projects.""" + + def get_diff_base(self): + return 'origin...' + + def prepare_for_fuzzer_build(self): + logging.info('ExternalGeneric: preparing for fuzzer build.') + manager = repo_manager.RepoManager(self.config.project_src_path) + build_integration_abs_path = os.path.join( + manager.repo_dir, self.config.build_integration_path) + if not build_external_project_docker_image( + self.config.project_name, manager.repo_dir, build_integration_abs_path): + logging.error('Failed to build external project: %s.', + self.config.project_name) + return BuildPreparationResult(success=False, + image_repo_path=None, + repo_manager=None) + + image_repo_path = os.path.join('/src', self.config.project_repo_name) + return BuildPreparationResult(success=True, + image_repo_path=image_repo_path, + repo_manager=manager) + + class ExternalGithub(GithubCiMixin, BaseCi): """Class representing CI for a non-OSS-Fuzz project on Github Actions.""" @@ -212,12 +250,16 @@ class ExternalGithub(GithubCiMixin, BaseCi): checkout_specified_commit(manager, self.config.pr_ref, self.config.commit_sha) - build_integration_path = os.path.join(manager.repo_dir, - self.config.build_integration_path) + build_integration_abs_path = os.path.join( + manager.repo_dir, self.config.build_integration_path) if not build_external_project_docker_image( - self.config.project_name, manager.repo_dir, build_integration_path): + self.config.project_name, manager.repo_dir, build_integration_abs_path): logging.error('Failed to build external project.') - return BuildPreparationResult(False, None, None) + return BuildPreparationResult(success=False, + image_repo_path=None, + repo_manager=None) image_repo_path = os.path.join('/src', self.config.project_repo_name) - return BuildPreparationResult(True, image_repo_path, manager) + return BuildPreparationResult(success=True, + image_repo_path=image_repo_path, + repo_manager=manager)