ofg: prepare use of cached images (#12693)

Ref: https://github.com/google/oss-fuzz-gen/pull/696

---------

Signed-off-by: David Korczynski <david@adalogics.com>
Co-authored-by: Oliver Chang <ochang@google.com>
This commit is contained in:
DavidKorczynski 2024-11-07 01:40:57 +00:00 committed by GitHub
parent edfa6730fa
commit e3ccf897f8
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 34 additions and 17 deletions

View File

@ -387,7 +387,8 @@ def get_docker_build_step(image_names,
directory,
use_buildkit_cache=False,
src_root='oss-fuzz',
architecture='x86_64'):
architecture='x86_64',
cache_image=''):
"""Returns the docker build step."""
assert len(image_names) >= 1
directory = os.path.join(src_root, directory)
@ -404,6 +405,9 @@ def get_docker_build_step(image_names,
_make_image_name_architecture_specific(image_name, architecture)
for image_name in image_names
]
if cache_image:
args.extend(['--build-arg', f'CACHE_IMAGE={cache_image}'])
for image_name in image_names:
args.extend(['--tag', image_name])
@ -437,7 +441,8 @@ def get_project_image_steps( # pylint: disable=too-many-arguments
language,
config,
architectures=None,
experiment=False):
experiment=False,
cache_image=None):
"""Returns GCB steps to build OSS-Fuzz project image."""
if architectures is None:
architectures = []
@ -453,9 +458,16 @@ def get_project_image_steps( # pylint: disable=too-many-arguments
if config.test_image_suffix:
steps.extend(get_pull_test_images_steps(config.test_image_suffix))
src_root = 'oss-fuzz' if not experiment else '.'
steps.append({
'name': 'ubuntu',
'args': ['bash', '-c', f'cat {src_root}/projects/{name}/Dockerfile'],
})
docker_build_step = get_docker_build_step([image],
os.path.join('projects', name),
src_root=src_root)
src_root=src_root,
cache_image=cache_image)
steps.append(docker_build_step)
srcmap_step_id = get_srcmap_step_id()
steps.extend([{

View File

@ -185,13 +185,12 @@ class Project: # pylint: disable=too-many-instance-attributes
@property
def image(self):
"""Returns the docker image for the project."""
if self.cached_sanitizer:
return self.cached_image(self.cached_sanitizer)
return f'gcr.io/{build_lib.IMAGE_PROJECT}/{self.name}'
def cached_image(self, sanitizer):
return _CACHED_IMAGE.format(name=self.real_name, sanitizer=sanitizer)
@property
def cached_image(self):
return _CACHED_IMAGE.format(name=self.real_name,
sanitizer=self.cached_sanitizer)
def get_last_step_id(steps):
@ -354,17 +353,23 @@ def get_build_steps_for_project(project,
return []
timestamp = get_datetime_now().strftime('%Y%m%d%H%M')
# If we use caching, then we need to use the right name. We assume that
# there is only a single sanitizer.
if use_caching:
# Use cached built image.
build_steps = []
project.cached_sanitizer = project.sanitizers[0]
cache_image = project.cached_image
else:
build_steps = build_lib.get_project_image_steps(
project.name,
project.image,
project.fuzzing_language,
config=config,
architectures=project.architectures,
experiment=config.experiment)
cache_image = None
build_steps = build_lib.get_project_image_steps(
project.name,
project.image,
project.fuzzing_language,
config=config,
architectures=project.architectures,
experiment=config.experiment,
cache_image=cache_image)
# Sort engines to make AFL first to test if libFuzzer has an advantage in
# finding bugs first since it is generally built first.