From f84b87d8e26614cb355dbe16b2b170b26849fcb3 Mon Sep 17 00:00:00 2001 From: Oliver Chang Date: Mon, 14 Nov 2016 14:47:45 -0800 Subject: [PATCH] Add build setup/status script. --- infra/auto-setup/build_status.py | 101 ++++++++++++++++++ infra/auto-setup/jenkins_config/base_job.xml | 46 ++++++++ infra/auto-setup/requirements.txt | 2 + infra/auto-setup/sync.py | 88 +++++++++++++++ .../auto-setup/templates/status_template.html | 20 ++++ 5 files changed, 257 insertions(+) create mode 100755 infra/auto-setup/build_status.py create mode 100644 infra/auto-setup/jenkins_config/base_job.xml create mode 100644 infra/auto-setup/requirements.txt create mode 100755 infra/auto-setup/sync.py create mode 100644 infra/auto-setup/templates/status_template.html diff --git a/infra/auto-setup/build_status.py b/infra/auto-setup/build_status.py new file mode 100755 index 000000000..0d60b0260 --- /dev/null +++ b/infra/auto-setup/build_status.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python + +import codecs +import datetime +import os +import subprocess + +import jenkins +import jinja2 +from jinja2 import Environment, FileSystemLoader + +JENKINS_SERVER = ('localhost', 8080) +LOGS_BUCKET = 'oss-fuzz-build-logs' + +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) + + +class Result(object): + """Result.""" + + def __init__(self, name, output): + self.name = name + self.output = output + + +def get_build_results(server): + """Return successes, failures.""" + successes = [] + failures = [] + + for job in server.get_jobs(1): + try: + name = job['fullname'] + if not name.startswith('targets/'): + continue + + print name + library = name[len('targets/'):] + + info = server.get_job_info(name) + last_build_number = info['lastCompletedBuild']['number'] + last_failed_builder_number = info['lastFailedBuild']['number'] + + if last_build_number == last_failed_builder_number: + failures.append(Result( + library, + server.get_build_console_output(name, last_build_number))) + else: + successes.append(Result( + library, + server.get_build_console_output(name, last_build_number))) + except Exception: + pass + + return successes, failures + + +def upload_status(successes, failures): + """Upload main status page.""" + env = Environment(loader=FileSystemLoader(os.path.join(SCRIPT_DIR, + 'templates'))) + with open('status.html', 'w') as f: + f.write( + env.get_template('status_template.html').render( + failures=failures, successes=successes, + last_updated=datetime.datetime.utcnow().ctime())) + + subprocess.check_output(['gsutil', 'cp', 'status.html', 'gs://' + + LOGS_BUCKET]) + + +def upload_build_logs(successes, failures): + """Upload individual build logs.""" + for result in failures + successes: + with codecs.open('latest.txt', 'w', encoding='utf-8') as f: + f.write(result.output) + + subprocess.check_output(['gsutil', 'cp', 'latest.txt', + 'gs://%s/build_logs/%s/' % + (LOGS_BUCKET, result.name)]) + + +def main(): + jenkins_login = get_jenkins_login() + server = jenkins.Jenkins('http://%s:%d' % JENKINS_SERVER, + username=jenkins_login[0], password=jenkins_login[1]) + + successes, failures = get_build_results(server) + upload_status(successes, failures) + upload_build_logs(successes, failures) + + +def get_jenkins_login(): + """Returns (username, password) for jenkins.""" + username = os.getenv('JENKINS_USER') + password = os.getenv('JENKINS_PASS') + return username, password + + +if __name__ == '__main__': + main() diff --git a/infra/auto-setup/jenkins_config/base_job.xml b/infra/auto-setup/jenkins_config/base_job.xml new file mode 100644 index 000000000..1c35aad4f --- /dev/null +++ b/infra/auto-setup/jenkins_config/base_job.xml @@ -0,0 +1,46 @@ + + + + + false + + + + + + + infra/base-images + + SUCCESS + 0 + BLUE + true + + + + H/15 * * * * + false + + + + + + + 2 + + + https://github.com/google/oss-fuzz.git + + + + + */master + + + false + + + + targets/library/Jenkinsfile + + diff --git a/infra/auto-setup/requirements.txt b/infra/auto-setup/requirements.txt new file mode 100644 index 000000000..96ada2654 --- /dev/null +++ b/infra/auto-setup/requirements.txt @@ -0,0 +1,2 @@ +python-jenkins +jinja2 diff --git a/infra/auto-setup/sync.py b/infra/auto-setup/sync.py new file mode 100755 index 000000000..937aabbd1 --- /dev/null +++ b/infra/auto-setup/sync.py @@ -0,0 +1,88 @@ +#!/usr/bin/env python +"""Script to sync CF and Jenkins jobs.""" + +import json +import os +import sys +import urllib2 +import xml.etree.ElementTree as ET + +import jenkins + +BUILD_BUCKET = 'clusterfuzz-builds' +JENKINS_SERVER = ('localhost', 8080) + +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) + + +def main(): + # Connect to jenkins server. + jenkins_login = get_jenkins_login() + server = jenkins.Jenkins('http://%s:%d' % JENKINS_SERVER, + username=jenkins_login[0], password=jenkins_login[1]) + + for library in get_libraries(): + print 'syncing configs for', library + try: + # Create/update jenkins build job. + sync_jenkins_job(server, library) + + except Exception as e: + print >>sys.stderr, 'Failed to setup job with exception', e + + +def get_libraries(): + """Return list of libraries for oss-fuzz.""" + OSSFUZZ_TREE_URL = ('https://api.github.com/repos/google/oss-fuzz/' + 'git/trees/master') + tree = json.loads(urllib2.urlopen(OSSFUZZ_TREE_URL).read()) + libraries = [] + + targets_url = None + + for item in tree['tree']: + if item['path'] == 'targets': + targets_url = item['url'] + break + + if not targets_url: + print >>sys.stderr, 'No libraries found.' + return [] + + tree = json.loads(urllib2.urlopen(targets_url).read()) + for item in tree['tree']: + if item['type'] != 'tree': + continue + + libraries.append(item['path']) + + return libraries + + +def get_jenkins_login(): + """Returns (username, password) for jenkins.""" + username = os.getenv('JENKINS_USER') + password = os.getenv('JENKINS_PASS') + + return username, password + + +def sync_jenkins_job(server, library): + """Sync the config with jenkins.""" + job_name = 'targets/' + library + if server.job_exists(job_name): + # Job already set up. + # TODO(ochang): Also update jobs if the definition is different. + return + + job_definition = ET.parse(os.path.join(SCRIPT_DIR, 'jenkins_config', + 'base_job.xml')) + jenkinsfile_location = job_definition.findall('.//definition/scriptPath')[0] + jenkinsfile_location.text = 'targets/%s/Jenkinsfile' % library + + server.create_job(job_name, ET.tostring(job_definition.getroot())) + server.build_job(job_name) + + +if __name__ == '__main__': + main() diff --git a/infra/auto-setup/templates/status_template.html b/infra/auto-setup/templates/status_template.html new file mode 100644 index 000000000..07e2a087e --- /dev/null +++ b/infra/auto-setup/templates/status_template.html @@ -0,0 +1,20 @@ + + + +OSS Fuzz Build Status + + +

Failing builds

+ +

Healthy builds

+ +

Last updated {{ last_updated }} (UTC)

+