Add build setup/status script.

infra/auto-setup/build_status.py

@@ -0,0 +1,101 @@
+#!/usr/bin/env python
+
+import codecs
+import datetime
+import os
+import subprocess
+
+import jenkins
+import jinja2
+from jinja2 import Environment, FileSystemLoader
+
+JENKINS_SERVER = ('localhost', 8080)
+LOGS_BUCKET = 'oss-fuzz-build-logs'
+
+SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
+
+
+class Result(object):
+  """Result."""
+
+  def __init__(self, name, output):
+    self.name = name
+    self.output = output
+
+
+def get_build_results(server):
+  """Return successes, failures."""
+  successes = []
+  failures = []
+
+  for job in server.get_jobs(1):
+    try:
+      name = job['fullname']
+      if not name.startswith('targets/'):
+        continue
+
+      print name
+      library = name[len('targets/'):]
+
+      info = server.get_job_info(name)
+      last_build_number = info['lastCompletedBuild']['number']
+      last_failed_builder_number = info['lastFailedBuild']['number']
+
+      if last_build_number == last_failed_builder_number:
+        failures.append(Result(
+            library,
+            server.get_build_console_output(name, last_build_number)))
+      else:
+        successes.append(Result(
+            library,
+            server.get_build_console_output(name, last_build_number)))
+    except Exception:
+      pass
+
+  return successes, failures
+
+
+def upload_status(successes, failures):
+  """Upload main status page."""
+  env = Environment(loader=FileSystemLoader(os.path.join(SCRIPT_DIR,
+                                                         'templates')))
+  with open('status.html', 'w') as f:
+    f.write(
+      env.get_template('status_template.html').render(
+          failures=failures, successes=successes,
+          last_updated=datetime.datetime.utcnow().ctime()))
+
+  subprocess.check_output(['gsutil', 'cp', 'status.html', 'gs://' +
+                           LOGS_BUCKET])
+
+
+def upload_build_logs(successes, failures):
+  """Upload individual build logs."""
+  for result in failures + successes:
+    with codecs.open('latest.txt', 'w', encoding='utf-8') as f:
+      f.write(result.output)
+
+    subprocess.check_output(['gsutil', 'cp', 'latest.txt',
+                             'gs://%s/build_logs/%s/' %
+                             (LOGS_BUCKET, result.name)])
+
+
+def main():
+  jenkins_login = get_jenkins_login()
+  server = jenkins.Jenkins('http://%s:%d' % JENKINS_SERVER,
+                           username=jenkins_login[0], password=jenkins_login[1])
+
+  successes, failures = get_build_results(server)
+  upload_status(successes, failures)
+  upload_build_logs(successes, failures)
+
+
+def get_jenkins_login():
+  """Returns (username, password) for jenkins."""
+  username = os.getenv('JENKINS_USER')
+  password = os.getenv('JENKINS_PASS')
+  return username, password
+
+
+if __name__ == '__main__':
+  main()

infra/auto-setup/jenkins_config/base_job.xml

@@ -0,0 +1,46 @@
+<?xml version='1.0' encoding='UTF-8'?>
+<flow-definition plugin="workflow-job@2.7">
+  <actions/>
+  <description></description>
+  <keepDependencies>false</keepDependencies>
+  <properties>
+    <org.jenkinsci.plugins.workflow.job.properties.DisableConcurrentBuildsJobProperty/>
+    <org.jenkinsci.plugins.workflow.job.properties.PipelineTriggersJobProperty>
+      <triggers>
+        <jenkins.triggers.ReverseBuildTrigger>
+          <spec></spec>
+          <upstreamProjects>infra/base-images</upstreamProjects>
+          <threshold>
+            <name>SUCCESS</name>
+            <ordinal>0</ordinal>
+            <color>BLUE</color>
+            <completeBuild>true</completeBuild>
+          </threshold>
+        </jenkins.triggers.ReverseBuildTrigger>
+        <hudson.triggers.SCMTrigger>
+          <spec>H/15 * * * *</spec>
+          <ignorePostCommitHooks>false</ignorePostCommitHooks>
+        </hudson.triggers.SCMTrigger>
+      </triggers>
+    </org.jenkinsci.plugins.workflow.job.properties.PipelineTriggersJobProperty>
+  </properties>
+  <definition class="org.jenkinsci.plugins.workflow.cps.CpsScmFlowDefinition" plugin="workflow-cps@2.18">
+    <scm class="hudson.plugins.git.GitSCM" plugin="git@3.0.0">
+      <configVersion>2</configVersion>
+      <userRemoteConfigs>
+        <hudson.plugins.git.UserRemoteConfig>
+          <url>https://github.com/google/oss-fuzz.git</url>
+        </hudson.plugins.git.UserRemoteConfig>
+      </userRemoteConfigs>
+      <branches>
+        <hudson.plugins.git.BranchSpec>
+          <name>*/master</name>
+        </hudson.plugins.git.BranchSpec>
+      </branches>
+      <doGenerateSubmoduleConfigurations>false</doGenerateSubmoduleConfigurations>
+      <submoduleCfg class="list"/>
+      <extensions/>
+    </scm>
+    <scriptPath>targets/library/Jenkinsfile</scriptPath>
+  </definition>
+</flow-definition>

infra/auto-setup/requirements.txt

@@ -0,0 +1,2 @@
+python-jenkins
+jinja2

infra/auto-setup/sync.py

@@ -0,0 +1,88 @@
+#!/usr/bin/env python
+"""Script to sync CF and Jenkins jobs.""" 
+
+import json
+import os
+import sys
+import urllib2
+import xml.etree.ElementTree as ET
+
+import jenkins
+
+BUILD_BUCKET = 'clusterfuzz-builds'
+JENKINS_SERVER = ('localhost', 8080)
+
+SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
+
+
+def main():
+  # Connect to jenkins server.
+  jenkins_login = get_jenkins_login()
+  server = jenkins.Jenkins('http://%s:%d' % JENKINS_SERVER,
+                           username=jenkins_login[0], password=jenkins_login[1])
+
+  for library in get_libraries():
+    print 'syncing configs for', library
+    try:
+      # Create/update jenkins build job.
+      sync_jenkins_job(server, library)
+
+    except Exception as e:
+      print >>sys.stderr, 'Failed to setup job with exception', e
+
+
+def get_libraries():
+  """Return list of libraries for oss-fuzz."""
+  OSSFUZZ_TREE_URL = ('https://api.github.com/repos/google/oss-fuzz/'
+                      'git/trees/master')
+  tree = json.loads(urllib2.urlopen(OSSFUZZ_TREE_URL).read())
+  libraries = []
+
+  targets_url = None
+
+  for item in tree['tree']:
+    if item['path'] == 'targets':
+      targets_url = item['url']
+      break
+
+  if not targets_url:
+    print >>sys.stderr, 'No libraries found.'
+    return []
+
+  tree = json.loads(urllib2.urlopen(targets_url).read())
+  for item in tree['tree']:
+    if item['type'] != 'tree':
+      continue
+
+    libraries.append(item['path'])
+
+  return libraries
+
+
+def get_jenkins_login():
+  """Returns (username, password) for jenkins."""
+  username = os.getenv('JENKINS_USER')
+  password = os.getenv('JENKINS_PASS')
+
+  return username, password
+
+
+def sync_jenkins_job(server, library):
+  """Sync the config with jenkins."""
+  job_name = 'targets/' + library
+  if server.job_exists(job_name):
+    # Job already set up.
+    # TODO(ochang): Also update jobs if the definition is different.
+    return
+
+  job_definition = ET.parse(os.path.join(SCRIPT_DIR, 'jenkins_config',
+                                         'base_job.xml'))
+  jenkinsfile_location = job_definition.findall('.//definition/scriptPath')[0]
+  jenkinsfile_location.text = 'targets/%s/Jenkinsfile' % library
+
+  server.create_job(job_name, ET.tostring(job_definition.getroot()))
+  server.build_job(job_name)
+
+
+if __name__ == '__main__':
+  main()

infra/auto-setup/templates/status_template.html

@@ -0,0 +1,20 @@
+<!DOCTYPE html>
+<meta charset="utf-8">
+<head>
+<title>OSS Fuzz Build Status</title>
+</head>
+<body>
+<h1>Failing builds</h1>
+<ul>
+{% for failure in failures -%}
+<li><a href="/build_logs/{{ failure.name }}/latest.txt">{{ failure.name }}</a></li>
+{% endfor -%}
+</ul>
+<h1>Healthy builds</h1>
+<ul>
+{% for success in successes -%}
+<li><a href="/build_logs/{{ success.name }}/latest.txt">{{ success.name }}</a></li>
+{% endfor -%}
+</ul>
+<p>Last updated {{ last_updated }} (UTC)</p>
+</body>