diff --git a/infra/build/functions/build_lib.py b/infra/build/functions/build_lib.py index 684ebe97a..6536933f2 100644 --- a/infra/build/functions/build_lib.py +++ b/infra/build/functions/build_lib.py @@ -22,6 +22,9 @@ import re import six.moves.urllib.parse as urlparse import sys import time +import subprocess +import tempfile +import json from googleapiclient.discovery import build as cloud_build import googleapiclient.discovery @@ -95,6 +98,8 @@ DOCKER_TOOL_IMAGE = 'gcr.io/cloud-builders/docker' _ARM64 = 'aarch64' +OSS_FUZZ_ROOT = os.path.abspath(os.path.join(__file__, '..', '..', '..', '..')) + def get_targets_list_filename(sanitizer): """Returns target list filename.""" @@ -423,7 +428,8 @@ def get_project_image_steps( # pylint: disable=too-many-arguments image, language, config, - architectures=None): + architectures=None, + experiment=False): """Returns GCB steps to build OSS-Fuzz project image.""" if architectures is None: architectures = [] @@ -433,8 +439,10 @@ def get_project_image_steps( # pylint: disable=too-many-arguments steps = [clone_step] if config.test_image_suffix: steps.extend(get_pull_test_images_steps(config.test_image_suffix)) + src_root = 'oss-fuzz' if not experiment else '.' docker_build_step = get_docker_build_step([image], - os.path.join('projects', name)) + os.path.join('projects', name), + src_root=src_root) steps.append(docker_build_step) srcmap_step_id = get_srcmap_step_id() steps.extend([{ @@ -532,7 +540,8 @@ def run_build( # pylint: disable=too-many-arguments timeout, body_overrides=None, tags=None, - use_build_pool=True): + use_build_pool=True, + experiment=False): """Runs the build.""" build_body = get_build_body(steps, @@ -540,6 +549,20 @@ def run_build( # pylint: disable=too-many-arguments body_overrides, tags, use_build_pool=use_build_pool) + if experiment: + with tempfile.NamedTemporaryFile(suffix='build.json') as config_file: + config_file.write(bytes(json.dumps(build_body), 'utf-8')) + config_file.seek(0) + subprocess.run([ + 'gcloud', + 'builds', + 'submit', + '--project=oss-fuzz', + f'--config={config_file.name}', + ], + cwd=OSS_FUZZ_ROOT) + + return 'NO-ID' # Doesn't matter, this is just printed to the user. cloudbuild = cloud_build('cloudbuild', 'v1', diff --git a/infra/build/functions/build_project.py b/infra/build/functions/build_project.py index 1fc87ab24..80a4b8e86 100755 --- a/infra/build/functions/build_project.py +++ b/infra/build/functions/build_project.py @@ -40,6 +40,7 @@ import build_lib FUZZING_BUILD_TYPE = 'fuzzing' GCB_LOGS_BUCKET = 'oss-fuzz-gcb-logs' +GCB_EXPERIMENT_LOGS_BUCKET = 'oss-fuzz-gcb-experiment-logs' DEFAULT_ARCHITECTURES = ['x86_64'] DEFAULT_ENGINES = ['libfuzzer', 'afl', 'honggfuzz', 'centipede'] @@ -55,10 +56,12 @@ PROJECTS_DIR = os.path.abspath( os.path.pardir, 'projects')) DEFAULT_OSS_FUZZ_REPO = 'https://github.com/google/oss-fuzz.git' -Config = collections.namedtuple( - 'Config', - ['testing', 'test_image_suffix', 'repo', 'branch', 'parallel', 'upload'], - defaults=(False, None, DEFAULT_OSS_FUZZ_REPO, None, False, True)) +Config = collections.namedtuple('Config', [ + 'testing', 'test_image_suffix', 'repo', 'branch', 'parallel', 'upload', + 'experiment' +], + defaults=(False, None, DEFAULT_OSS_FUZZ_REPO, + None, False, True, False)) WORKDIR_REGEX = re.compile(r'\s*WORKDIR\s*([^\s]+)') @@ -291,7 +294,8 @@ def get_build_steps( # pylint: disable=too-many-locals, too-many-statements, to project.image, project.fuzzing_language, config=config, - architectures=project.architectures) + architectures=project.architectures, + experiment=config.experiment) # Sort engines to make AFL first to test if libFuzzer has an advantage in # finding bugs first since it is generally built first. @@ -463,7 +467,8 @@ def run_build(oss_fuzz_project, credentials, build_type, cloud_project='oss-fuzz', - extra_tags=None): + extra_tags=None, + experiment=False): """Run the build for given steps on cloud build. |build_steps| are the steps to run. |credentials| are are used to authenticate to GCB and build in |cloud_project|. |oss_fuzz_project| and |build_type| are used to tag the build @@ -473,8 +478,9 @@ def run_build(oss_fuzz_project, tags = [oss_fuzz_project + '-' + build_type, build_type, oss_fuzz_project] tags.extend(extra_tags) timeout = build_lib.BUILD_TIMEOUT + bucket = GCB_LOGS_BUCKET if not experiment else GCB_EXPERIMENT_LOGS_BUCKET body_overrides = { - 'logsBucket': GCB_LOGS_BUCKET, + 'logsBucket': bucket, 'queueTtl': str(QUEUE_TTL_SECONDS) + 's', } return build_lib.run_build(build_steps, @@ -482,7 +488,8 @@ def run_build(oss_fuzz_project, cloud_project, timeout, body_overrides=body_overrides, - tags=tags) + tags=tags, + experiment=experiment) def get_args(description): @@ -508,16 +515,23 @@ def get_args(description): required=False, default=False, help='Do builds in parallel.') + parser.add_argument('--experiment', + action='store_true', + required=False, + default=False, + help='Configuration for experiments.') return parser.parse_args() def create_config_from_commandline(args): """Create a Config object from parsed command line |args|.""" + upload = not args.experiment return Config(testing=args.testing, test_image_suffix=args.test_image_suffix, branch=args.branch, parallel=args.parallel, - upload=True) + upload=upload, + experiment=args.experiment) def build_script_main(script_description, get_build_steps_func, build_type): @@ -547,7 +561,11 @@ def build_script_main(script_description, get_build_steps_func, build_type): error = True continue - run_build(project_name, steps, credentials, build_type) + run_build(project_name, + steps, + credentials, + build_type, + experiment=args.experiment) return 0 if not error else 1