ofg: prepare use of cached images (#12693)

Ref: https://github.com/google/oss-fuzz-gen/pull/696

---------

Signed-off-by: David Korczynski <david@adalogics.com>
Co-authored-by: Oliver Chang <ochang@google.com>
This commit is contained in:
DavidKorczynski 2024-11-07 01:40:57 +00:00 committed by GitHub
parent edfa6730fa
commit e3ccf897f8
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 34 additions and 17 deletions

View File

@ -387,7 +387,8 @@ def get_docker_build_step(image_names,
directory, directory,
use_buildkit_cache=False, use_buildkit_cache=False,
src_root='oss-fuzz', src_root='oss-fuzz',
architecture='x86_64'): architecture='x86_64',
cache_image=''):
"""Returns the docker build step.""" """Returns the docker build step."""
assert len(image_names) >= 1 assert len(image_names) >= 1
directory = os.path.join(src_root, directory) directory = os.path.join(src_root, directory)
@ -404,6 +405,9 @@ def get_docker_build_step(image_names,
_make_image_name_architecture_specific(image_name, architecture) _make_image_name_architecture_specific(image_name, architecture)
for image_name in image_names for image_name in image_names
] ]
if cache_image:
args.extend(['--build-arg', f'CACHE_IMAGE={cache_image}'])
for image_name in image_names: for image_name in image_names:
args.extend(['--tag', image_name]) args.extend(['--tag', image_name])
@ -437,7 +441,8 @@ def get_project_image_steps( # pylint: disable=too-many-arguments
language, language,
config, config,
architectures=None, architectures=None,
experiment=False): experiment=False,
cache_image=None):
"""Returns GCB steps to build OSS-Fuzz project image.""" """Returns GCB steps to build OSS-Fuzz project image."""
if architectures is None: if architectures is None:
architectures = [] architectures = []
@ -453,9 +458,16 @@ def get_project_image_steps( # pylint: disable=too-many-arguments
if config.test_image_suffix: if config.test_image_suffix:
steps.extend(get_pull_test_images_steps(config.test_image_suffix)) steps.extend(get_pull_test_images_steps(config.test_image_suffix))
src_root = 'oss-fuzz' if not experiment else '.' src_root = 'oss-fuzz' if not experiment else '.'
steps.append({
'name': 'ubuntu',
'args': ['bash', '-c', f'cat {src_root}/projects/{name}/Dockerfile'],
})
docker_build_step = get_docker_build_step([image], docker_build_step = get_docker_build_step([image],
os.path.join('projects', name), os.path.join('projects', name),
src_root=src_root) src_root=src_root,
cache_image=cache_image)
steps.append(docker_build_step) steps.append(docker_build_step)
srcmap_step_id = get_srcmap_step_id() srcmap_step_id = get_srcmap_step_id()
steps.extend([{ steps.extend([{

View File

@ -185,13 +185,12 @@ class Project: # pylint: disable=too-many-instance-attributes
@property @property
def image(self): def image(self):
"""Returns the docker image for the project.""" """Returns the docker image for the project."""
if self.cached_sanitizer:
return self.cached_image(self.cached_sanitizer)
return f'gcr.io/{build_lib.IMAGE_PROJECT}/{self.name}' return f'gcr.io/{build_lib.IMAGE_PROJECT}/{self.name}'
def cached_image(self, sanitizer): @property
return _CACHED_IMAGE.format(name=self.real_name, sanitizer=sanitizer) def cached_image(self):
return _CACHED_IMAGE.format(name=self.real_name,
sanitizer=self.cached_sanitizer)
def get_last_step_id(steps): def get_last_step_id(steps):
@ -354,17 +353,23 @@ def get_build_steps_for_project(project,
return [] return []
timestamp = get_datetime_now().strftime('%Y%m%d%H%M') timestamp = get_datetime_now().strftime('%Y%m%d%H%M')
# If we use caching, then we need to use the right name. We assume that
# there is only a single sanitizer.
if use_caching: if use_caching:
# Use cached built image. project.cached_sanitizer = project.sanitizers[0]
build_steps = [] cache_image = project.cached_image
else: else:
build_steps = build_lib.get_project_image_steps( cache_image = None
project.name,
project.image, build_steps = build_lib.get_project_image_steps(
project.fuzzing_language, project.name,
config=config, project.image,
architectures=project.architectures, project.fuzzing_language,
experiment=config.experiment) config=config,
architectures=project.architectures,
experiment=config.experiment,
cache_image=cache_image)
# Sort engines to make AFL first to test if libFuzzer has an advantage in # Sort engines to make AFL first to test if libFuzzer has an advantage in
# finding bugs first since it is generally built first. # finding bugs first since it is generally built first.