From e3ccf897f86d9337872c5a819c017617b43d3f4c Mon Sep 17 00:00:00 2001 From: DavidKorczynski Date: Thu, 7 Nov 2024 01:40:57 +0000 Subject: [PATCH] ofg: prepare use of cached images (#12693) Ref: https://github.com/google/oss-fuzz-gen/pull/696 --------- Signed-off-by: David Korczynski Co-authored-by: Oliver Chang --- infra/build/functions/build_lib.py | 18 +++++++++++--- infra/build/functions/build_project.py | 33 +++++++++++++++----------- 2 files changed, 34 insertions(+), 17 deletions(-) diff --git a/infra/build/functions/build_lib.py b/infra/build/functions/build_lib.py index a5766cf02..58183ef66 100644 --- a/infra/build/functions/build_lib.py +++ b/infra/build/functions/build_lib.py @@ -387,7 +387,8 @@ def get_docker_build_step(image_names, directory, use_buildkit_cache=False, src_root='oss-fuzz', - architecture='x86_64'): + architecture='x86_64', + cache_image=''): """Returns the docker build step.""" assert len(image_names) >= 1 directory = os.path.join(src_root, directory) @@ -404,6 +405,9 @@ def get_docker_build_step(image_names, _make_image_name_architecture_specific(image_name, architecture) for image_name in image_names ] + if cache_image: + args.extend(['--build-arg', f'CACHE_IMAGE={cache_image}']) + for image_name in image_names: args.extend(['--tag', image_name]) @@ -437,7 +441,8 @@ def get_project_image_steps( # pylint: disable=too-many-arguments language, config, architectures=None, - experiment=False): + experiment=False, + cache_image=None): """Returns GCB steps to build OSS-Fuzz project image.""" if architectures is None: architectures = [] @@ -453,9 +458,16 @@ def get_project_image_steps( # pylint: disable=too-many-arguments if config.test_image_suffix: steps.extend(get_pull_test_images_steps(config.test_image_suffix)) src_root = 'oss-fuzz' if not experiment else '.' + + steps.append({ + 'name': 'ubuntu', + 'args': ['bash', '-c', f'cat {src_root}/projects/{name}/Dockerfile'], + }) + docker_build_step = get_docker_build_step([image], os.path.join('projects', name), - src_root=src_root) + src_root=src_root, + cache_image=cache_image) steps.append(docker_build_step) srcmap_step_id = get_srcmap_step_id() steps.extend([{ diff --git a/infra/build/functions/build_project.py b/infra/build/functions/build_project.py index 44c00a87a..3878bd086 100755 --- a/infra/build/functions/build_project.py +++ b/infra/build/functions/build_project.py @@ -185,13 +185,12 @@ class Project: # pylint: disable=too-many-instance-attributes @property def image(self): """Returns the docker image for the project.""" - if self.cached_sanitizer: - return self.cached_image(self.cached_sanitizer) - return f'gcr.io/{build_lib.IMAGE_PROJECT}/{self.name}' - def cached_image(self, sanitizer): - return _CACHED_IMAGE.format(name=self.real_name, sanitizer=sanitizer) + @property + def cached_image(self): + return _CACHED_IMAGE.format(name=self.real_name, + sanitizer=self.cached_sanitizer) def get_last_step_id(steps): @@ -354,17 +353,23 @@ def get_build_steps_for_project(project, return [] timestamp = get_datetime_now().strftime('%Y%m%d%H%M') + + # If we use caching, then we need to use the right name. We assume that + # there is only a single sanitizer. if use_caching: - # Use cached built image. - build_steps = [] + project.cached_sanitizer = project.sanitizers[0] + cache_image = project.cached_image else: - build_steps = build_lib.get_project_image_steps( - project.name, - project.image, - project.fuzzing_language, - config=config, - architectures=project.architectures, - experiment=config.experiment) + cache_image = None + + build_steps = build_lib.get_project_image_steps( + project.name, + project.image, + project.fuzzing_language, + config=config, + architectures=project.architectures, + experiment=config.experiment, + cache_image=cache_image) # Sort engines to make AFL first to test if libFuzzer has an advantage in # finding bugs first since it is generally built first.