oss-fuzz/infra/gcb/build_project.py

383 lines
11 KiB
Python
Executable File

#!/usr/bin/python2
"""Starts project build on Google Cloud Builder.
Usage: build_project.py <project_dir>
"""
import base64
import collections
import datetime
import json
import os
import re
import subprocess
import sys
import time
import urllib
import yaml
from oauth2client.client import GoogleCredentials
from oauth2client.service_account import ServiceAccountCredentials
from googleapiclient.discovery import build
BUILD_TIMEOUT = 10 * 60 * 60
FUZZING_BUILD_TAG = 'fuzzing'
GCB_LOGS_BUCKET = 'oss-fuzz-gcb-logs'
CONFIGURATIONS = {
'sanitizer-address': ['SANITIZER=address'],
'sanitizer-memory': ['SANITIZER=memory'],
'sanitizer-undefined': ['SANITIZER=undefined'],
'engine-libfuzzer': ['FUZZING_ENGINE=libfuzzer'],
'engine-afl': ['FUZZING_ENGINE=afl'],
'engine-honggfuzz': ['FUZZING_ENGINE=honggfuzz'],
'engine-none': ['FUZZING_ENGINE=none'],
}
EngineInfo = collections.namedtuple('EngineInfo',
['upload_bucket', 'supported_sanitizers'])
ENGINE_INFO = {
'libfuzzer':
EngineInfo(
upload_bucket='clusterfuzz-builds',
supported_sanitizers=['address', 'memory', 'undefined']),
'afl':
EngineInfo(
upload_bucket='clusterfuzz-builds-afl',
supported_sanitizers=['address']),
'honggfuzz':
EngineInfo(
upload_bucket='clusterfuzz-builds-honggfuzz',
supported_sanitizers=['address', 'memory', 'undefined']),
'none':
EngineInfo(
upload_bucket='clusterfuzz-builds-no-engine',
supported_sanitizers=['address']),
}
DEFAULT_ENGINES = ['libfuzzer', 'afl', 'honggfuzz']
DEFAULT_SANITIZERS = ['address', 'undefined']
TARGETS_LIST_BASENAME = 'targets.list'
UPLOAD_URL_FORMAT = '/{0}/{1}/{2}'
def usage():
sys.stderr.write('Usage: ' + sys.argv[0] + ' <project_dir>\n')
exit(1)
def load_project_yaml(project_dir):
project_name = os.path.basename(project_dir)
project_yaml_path = os.path.join(project_dir, 'project.yaml')
with open(project_yaml_path) as f:
project_yaml = yaml.safe_load(f)
project_yaml.setdefault('disabled', False)
project_yaml.setdefault('name', project_name)
project_yaml.setdefault('image', 'gcr.io/oss-fuzz/' + project_name)
project_yaml.setdefault('sanitizers', DEFAULT_SANITIZERS)
project_yaml.setdefault('fuzzing_engines', DEFAULT_ENGINES)
project_yaml.setdefault('run_tests', True)
project_yaml.setdefault('coverage_extra_args', '')
project_yaml.setdefault('labels', {})
return project_yaml
def get_signed_url(path, method='PUT', content_type=''):
timestamp = int(time.time() + BUILD_TIMEOUT)
blob = '{0}\n\n{1}\n{2}\n{3}'.format(method, content_type, timestamp, path)
creds = ServiceAccountCredentials.from_json_keyfile_name(
os.environ['GOOGLE_APPLICATION_CREDENTIALS'])
client_id = creds.service_account_email
signature = base64.b64encode(creds.sign_blob(blob)[1])
values = {
'GoogleAccessId': client_id,
'Expires': timestamp,
'Signature': signature,
}
return ('https://storage.googleapis.com{0}?'.format(path) +
urllib.urlencode(values))
def is_supported_configuration(fuzzing_engine, sanitizer):
return sanitizer in ENGINE_INFO[fuzzing_engine].supported_sanitizers
def get_sanitizers(project_yaml):
sanitizers = project_yaml['sanitizers']
assert isinstance(sanitizers, list)
processed_sanitizers = []
for sanitizer in sanitizers:
if isinstance(sanitizer, basestring):
processed_sanitizers.append(sanitizer)
elif isinstance(sanitizer, dict):
for key in sanitizer.iterkeys():
processed_sanitizers.append(key)
return processed_sanitizers
def workdir_from_dockerfile(dockerfile):
"""Parse WORKDIR from the Dockerfile."""
WORKDIR_REGEX = re.compile(r'\s*WORKDIR\s*([^\s]+)')
with open(dockerfile) as f:
lines = f.readlines()
for line in lines:
match = re.match(WORKDIR_REGEX, line)
if match:
# We need to escape '$' since they're used for subsitutions in Container
# Builer builds.
return match.group(1).replace('$', '$$')
return None
def get_build_steps(project_dir):
project_yaml = load_project_yaml(project_dir)
dockerfile_path = os.path.join(project_dir, 'Dockerfile')
name = project_yaml['name']
image = project_yaml['image']
run_tests = project_yaml['run_tests']
ts = datetime.datetime.now().strftime('%Y%m%d%H%M')
build_steps = [
{
'args': [
'clone',
'https://github.com/google/oss-fuzz.git',
],
'name': 'gcr.io/cloud-builders/git',
},
{
'name': 'gcr.io/cloud-builders/docker',
'args': [
'build',
'-t',
image,
'.',
],
'dir': 'oss-fuzz/projects/' + name,
},
{
'name':
image,
'args': [
'bash', '-c',
'srcmap > /workspace/srcmap.json && cat /workspace/srcmap.json'
],
'env': ['OSSFUZZ_REVISION=$REVISION_ID'],
},
{
'name': 'gcr.io/oss-fuzz-base/msan-builder',
'args': [
'bash',
'-c',
'cp -r /msan /workspace',
],
},
]
for fuzzing_engine in project_yaml['fuzzing_engines']:
for sanitizer in get_sanitizers(project_yaml):
if not is_supported_configuration(fuzzing_engine, sanitizer):
continue
env = CONFIGURATIONS['engine-' + fuzzing_engine][:]
env.extend(CONFIGURATIONS['sanitizer-' + sanitizer])
out = '/workspace/out/' + sanitizer
stamped_name = name + '-' + sanitizer + '-' + ts
zip_file = stamped_name + '.zip'
stamped_srcmap_file = stamped_name + '.srcmap.json'
bucket = ENGINE_INFO[fuzzing_engine].upload_bucket
upload_url = get_signed_url(UPLOAD_URL_FORMAT.format(bucket, name,
zip_file))
srcmap_url = get_signed_url(UPLOAD_URL_FORMAT.format(bucket, name,
stamped_srcmap_file))
targets_list_filename = get_targets_list_filename(sanitizer)
targets_list_url = get_signed_url(
get_targets_list_url(bucket, name, sanitizer))
env.append('OUT=' + out)
env.append('MSAN_LIBS_PATH=/workspace/msan')
workdir = workdir_from_dockerfile(dockerfile_path)
if not workdir:
workdir = '/src'
build_steps.append(
# compile
{
'name':
image,
'env':
env,
'args': [
'bash',
'-c',
# Remove /out to break loudly when a build script incorrectly uses
# /out instead of $OUT.
# `cd /src && cd {workdir}` (where {workdir} is parsed from the
# Dockerfile). Container Builder overrides our workdir so we need to add
# this step to set it back.
'rm -r /out && cd /src && cd {1} && mkdir -p {0} && compile'
.format(out, workdir),
],
})
if run_tests:
build_steps.append(
# test binaries
{
'name': 'gcr.io/oss-fuzz-base/base-runner',
'env': env,
'args': ['bash', '-c', 'test_all'],
})
if sanitizer == 'memory':
# Patch dynamic libraries to use instrumented ones.
build_steps.append({
'name':
'gcr.io/oss-fuzz-base/msan-builder',
'args': [
'bash',
'-c',
# TODO(ochang): Replace with just patch_build.py once permission in
# image is fixed.
'python /usr/local/bin/patch_build.py {0}'.format(out),
],
})
if project_yaml['labels']:
# write target labels
build_steps.append({
'name': image,
'env': env,
'args': [
'/usr/local/bin/write_labels.py',
json.dumps(project_yaml['labels']),
out,
],
})
build_steps.extend([
# generate targets list
{
'name':
'gcr.io/oss-fuzz-base/base-runner',
'env': env,
'args': [
'bash',
'-c',
'targets_list > /workspace/{0}'.format(targets_list_filename),
],
},
# zip binaries
{
'name':
image,
'args': [
'bash', '-c', 'cd {0} && zip -r {1} *'.format(out, zip_file)
],
},
# upload srcmap
{
'name': 'gcr.io/oss-fuzz-base/uploader',
'args': [
'/workspace/srcmap.json',
srcmap_url,
],
},
# upload binaries
{
'name': 'gcr.io/oss-fuzz-base/uploader',
'args': [
os.path.join(out, zip_file),
upload_url,
],
},
# upload targets list
{
'name':
'gcr.io/oss-fuzz-base/uploader',
'args': [
'/workspace/{0}'.format(targets_list_filename),
targets_list_url,
],
},
# cleanup
{
'name': image,
'args': [
'bash',
'-c',
'rm -r ' + out,
],
},
])
return build_steps, image
def get_logs_url(build_id):
URL_FORMAT = ('https://console.developers.google.com/logs/viewer?'
'resource=build%2Fbuild_id%2F{0}&project=oss-fuzz')
return URL_FORMAT.format(build_id)
def get_targets_list_filename(sanitizer):
return TARGETS_LIST_BASENAME + '.' + sanitizer
def get_targets_list_url(bucket, project, sanitizer):
filename = get_targets_list_filename(sanitizer)
url = UPLOAD_URL_FORMAT.format(bucket, project, filename)
return url
def run_build(build_steps, image, tag):
options = {}
if 'GCB_OPTIONS' in os.environ:
options = yaml.safe_load(os.environ['GCB_OPTIONS'])
build_body = {
'steps': build_steps,
'timeout': str(BUILD_TIMEOUT) + 's',
'options': options,
'logsBucket': GCB_LOGS_BUCKET,
'images': [ image ],
'tags': [ tag ],
}
credentials = GoogleCredentials.get_application_default()
cloudbuild = build('cloudbuild', 'v1', credentials=credentials)
build_info = cloudbuild.projects().builds().create(
projectId='oss-fuzz', body=build_body).execute()
build_id = build_info['metadata']['build']['id']
print >> sys.stderr, 'Logs:', get_logs_url(build_id)
print build_id
def main():
if len(sys.argv) != 2:
usage()
project_dir = sys.argv[1].rstrip(os.path.sep)
steps, image = get_build_steps(project_dir)
run_build(steps, image, FUZZING_BUILD_TAG)
if __name__ == '__main__':
main()