# Copyright 2019 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Module to build a image from a specific commit, branch or pull request This module is allows each of the OSS Fuzz projects fuzzers to be built from a specific point in time. This feature can be used for implementations like continuious integration fuzzing and bisection to find errors """ import bisect import os import collections import logging import re import shutil import time import helper import repo_manager import utils BuildData = collections.namedtuple( 'BuildData', ['project_name', 'engine', 'sanitizer', 'architecture']) _GIT_DIR_MARKER = 'gitdir: ' _IMAGE_BUILD_TRIES = 3 _IMAGE_BUILD_RETRY_SLEEP = 30.0 class BaseBuilderRepo: """Repo of base-builder images.""" def __init__(self): self.timestamps = [] self.digests = [] def add_digest(self, timestamp, digest): """Add a digest.""" self.timestamps.append(timestamp) self.digests.append(digest) def find_digest(self, timestamp): """Find the latest image before the given timestamp.""" index = bisect.bisect_right(self.timestamps, timestamp) if index > 0: return self.digests[index - 1] raise ValueError('Failed to find suitable base-builder.') def _replace_gitdir(src_dir, file_path): """Replace gitdir with a relative path.""" with open(file_path) as handle: lines = handle.readlines() new_lines = [] for line in lines: if line.startswith(_GIT_DIR_MARKER): absolute_path = line[len(_GIT_DIR_MARKER):].strip() if not os.path.isabs(absolute_path): # Already relative. return current_dir = os.path.dirname(file_path) # Rebase to /src rather than the host src dir. base_dir = current_dir.replace(src_dir, '/src') relative_path = os.path.relpath(absolute_path, base_dir) logging.info('Replacing absolute submodule gitdir from %s to %s', absolute_path, relative_path) line = _GIT_DIR_MARKER + relative_path new_lines.append(line) with open(file_path, 'w') as handle: handle.write(''.join(new_lines)) def _make_gitdirs_relative(src_dir): """Make gitdirs relative.""" for root_dir, _, files in os.walk(src_dir): for filename in files: if filename != '.git': continue file_path = os.path.join(root_dir, filename) _replace_gitdir(src_dir, file_path) def _replace_base_builder_digest(dockerfile_path, digest): """Replace the base-builder digest in a Dockerfile.""" with open(dockerfile_path) as handle: lines = handle.readlines() new_lines = [] for line in lines: if line.strip().startswith('FROM'): line = 'FROM gcr.io/oss-fuzz-base/base-builder@' + digest + '\n' new_lines.append(line) with open(dockerfile_path, 'w') as handle: handle.write(''.join(new_lines)) def copy_src_from_docker(project_name, host_dir): """Copy /src from docker to the host.""" # Copy /src to host. image_name = 'gcr.io/oss-fuzz/' + project_name src_dir = os.path.join(host_dir, 'src') if os.path.exists(src_dir): shutil.rmtree(src_dir, ignore_errors=True) docker_args = [ '-v', host_dir + ':/out', image_name, 'cp', '-r', '-p', '/src', '/out', ] helper.docker_run(docker_args) # Submodules can have gitdir entries which point to absolute paths. Make them # relative, as otherwise we can't do operations on the checkout on the host. _make_gitdirs_relative(src_dir) return src_dir def _build_image_with_retries(project_name): """Build image with retries.""" for _ in range(_IMAGE_BUILD_TRIES): result = helper.build_image_impl(project_name) if result: return result time.sleep(_IMAGE_BUILD_RETRY_SLEEP) return result def build_fuzzers_from_commit(commit, build_repo_manager, host_src_path, build_data, base_builder_repo=None): """Builds a OSS-Fuzz fuzzer at a specific commit SHA. Args: commit: The commit SHA to build the fuzzers at. build_repo_manager: The OSS-Fuzz project's repo manager to be built at. build_data: A struct containing project build information. base_builder_repo: A BaseBuilderRepo. Returns: 0 on successful build or error code on failure. """ oss_fuzz_repo_manager = repo_manager.BaseRepoManager(helper.OSS_FUZZ_DIR) num_retry = 1 def cleanup(): # Re-copy /src for a clean checkout every time. copy_src_from_docker(build_data.project_name, os.path.dirname(host_src_path)) for i in range(num_retry + 1): build_repo_manager.checkout_commit(commit, clean=False) result = helper.build_fuzzers_impl(project_name=build_data.project_name, clean=True, engine=build_data.engine, sanitizer=build_data.sanitizer, architecture=build_data.architecture, env_to_add=None, source_path=host_src_path, mount_location='/src') if result == 0 or i == num_retry: break # Retry with an OSS-Fuzz builder container that's closer to the project # commit date. commit_date = build_repo_manager.commit_date(commit) projects_dir = os.path.join('projects', build_data.project_name) # Find first change in the projects/ directory before the project # commit date. oss_fuzz_commit, _, _ = oss_fuzz_repo_manager.git([ 'log', '--before=' + commit_date.isoformat(), '-n1', '--format=%H', projects_dir ], check_result=True) oss_fuzz_commit = oss_fuzz_commit.strip() if not oss_fuzz_commit: logging.warning('No suitable earlier OSS-Fuzz commit found.') break logging.info('Build failed. Retrying on earlier OSS-Fuzz commit %s.', oss_fuzz_commit) # Check out projects/ dir to the commit that was found. oss_fuzz_repo_manager.git(['checkout', oss_fuzz_commit, projects_dir], check_result=True) # Also use the closest base-builder we can find. if base_builder_repo: base_builder_digest = base_builder_repo.find_digest(commit_date) logging.info('Using base-builder with digest %s.', base_builder_digest) _replace_base_builder_digest(os.path.join(projects_dir, 'Dockerfile'), base_builder_digest) # Rebuild image and re-copy src dir since things in /src could have changed. if not _build_image_with_retries(build_data.project_name): raise RuntimeError('Failed to rebuild image.') cleanup() cleanup() return result == 0 def detect_main_repo(project_name, repo_name=None, commit=None): """Checks a docker image for the main repo of an OSS-Fuzz project. Note: The default is to use the repo name to detect the main repo. Args: project_name: The name of the oss-fuzz project. repo_name: The name of the main repo in an OSS-Fuzz project. commit: A commit SHA that is associated with the main repo. src_dir: The location of the projects source on the docker image. Returns: The repo's origin, the repo's path. """ if not repo_name and not commit: logging.error( 'Error: can not detect main repo without a repo_name or a commit.') return None, None if repo_name and commit: logging.info( 'Both repo name and commit specific. Using repo name for detection.') # Change to oss-fuzz main directory so helper.py runs correctly. utils.chdir_to_root() if not _build_image_with_retries(project_name): logging.error('Error: building %s image failed.', project_name) return None, None docker_image_name = 'gcr.io/oss-fuzz/' + project_name command_to_run = [ 'docker', 'run', '--rm', '-t', docker_image_name, 'python3', os.path.join('/opt', 'cifuzz', 'detect_repo.py') ] if repo_name: command_to_run.extend(['--repo_name', repo_name]) else: command_to_run.extend(['--example_commit', commit]) out, _, _ = utils.execute(command_to_run) match = re.search(r'\bDetected repo: ([^ ]+) ([^ ]+)', out.rstrip()) if match and match.group(1) and match.group(2): return match.group(1), match.group(2) logging.error('Failed to detect repo:\n%s', out) return None, None