[infra] gcb build status script

infra/auto-setup/build_status_gcb.py

@@ -1,51 +0,0 @@
-#!/usr/bin/env python2
-
-import os
-import sys
-
-from oauth2client.client import GoogleCredentials
-from googleapiclient.discovery import build as gcb_build
-
-def usage():
-  sys.stderr.write(
-    "Usage: " + sys.argv[0] + " <projects_dir>\n")
-  exit(1)
-
-
-def scan_project_names(projects_dir):
-  projects = []
-  for root, dirs, files in os.walk(projects_dir):
-    for f in files:
-      if f == "Dockerfile":
-        projects.append(os.path.basename(root))
-  return sorted(projects)
-
-
-def main():
-  if len(sys.argv) != 2:
-    usage()
-
-  projects_dir = sys.argv[1]
-
-  credentials = GoogleCredentials.get_application_default()
-  cloudbuild = gcb_build('cloudbuild', 'v1', credentials=credentials)
-
-
-  for project in scan_project_names(projects_dir):
-    print project
-    query_filter = ('(status="SUCCESS" OR status="FAILURE") AND ' + 
-        'images="gcr.io/clusterfuzz-external/oss-fuzz/{0}"'.format(project))
-    response = cloudbuild.projects().builds().list(
-        projectId='clusterfuzz-external',
-        filter=query_filter).execute()
-    if not 'builds' in response:
-      continue
-
-    builds = sorted(response['builds'], key=lambda b: b['startTime'])
-    last_build = builds[-1]
-    print last_build['startTime'], last_build['status']
-
-
-if __name__ == "__main__":
-  main()
-

infra/gcb/build_status.py

@@ -0,0 +1,92 @@
+#!/usr/bin/env python2
+
+import datetime
+import os
+import sys
+import jinja2
+import json
+
+from oauth2client.client import GoogleCredentials
+from googleapiclient.discovery import build as gcb_build
+from google.cloud import storage
+from jinja2 import Environment, FileSystemLoader
+
+
+LOGS_BUCKET = 'oss-fuzz-gcb-logs'
+SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
+
+
+def usage():
+  sys.stderr.write(
+    "Usage: " + sys.argv[0] + " <projects_dir>\n")
+  exit(1)
+
+
+def scan_project_names(projects_dir):
+  projects = []
+  for root, dirs, files in os.walk(projects_dir):
+    for f in files:
+      if f == "Dockerfile":
+        projects.append(os.path.basename(root))
+  return sorted(projects)
+
+
+def upload_status(successes, failures, unstable):
+  """Upload main status page."""
+  env = Environment(loader=FileSystemLoader(os.path.join(SCRIPT_DIR,
+                                                         'templates')))
+  data = {
+      'projects': failures + successes + unstable,
+      'failures': failures,
+      'successes': successes,
+      'unstable': unstable,
+      'last_updated': datetime.datetime.utcnow().ctime()
+  }
+
+  storage_client = storage.Client()
+  bucket = storage_client.get_bucket(LOGS_BUCKET)
+
+  bucket.blob('status_template.html').upload_from_string(
+          env.get_template('status_template.html').render(data),
+          content_type='text/html')
+
+  bucket.blob('status.json').upload_from_string(
+          json.dumps(data),
+          content_type='text/html')
+
+
+def main():
+  if len(sys.argv) != 2:
+    usage()
+
+  projects_dir = sys.argv[1]
+
+  credentials = GoogleCredentials.get_application_default()
+  cloudbuild = gcb_build('cloudbuild', 'v1', credentials=credentials)
+
+
+  successes = []
+  failures = []
+  for project in scan_project_names(projects_dir):
+    print project
+    query_filter = ('(status="SUCCESS" OR status="FAILURE") AND ' + 
+        'images="gcr.io/clusterfuzz-external/oss-fuzz/{0}"'.format(project))
+    response = cloudbuild.projects().builds().list(
+        projectId='clusterfuzz-external',
+        filter=query_filter).execute()
+    if not 'builds' in response:
+      continue
+
+    builds = response['builds']
+    last_build = builds[0]
+    print last_build['startTime'], last_build['status'], last_build['id']
+    if last_build['status'] == 'SUCCESS':
+        successes.append(project)
+    else:
+        failures.append(project)
+
+  upload_status(successes, failures, [])
+
+if __name__ == "__main__":
+  main()
+

infra/gcb/requirements.txt

@@ -2,3 +2,5 @@ google-api-python-client
 PyYAML
 google-cloud-pubsub
 google-cloud-logging
+google-cloud-storage
+jinja2

infra/gcb/templates/status_template.html

@@ -0,0 +1,26 @@
+<!DOCTYPE html>
+<meta charset="utf-8">
+<head>
+<title>OSS Fuzz Build Status</title>
+</head>
+<body>
+<h1>Failing builds</h1>
+<ul>
+{% for failure in failures -%}
+<li><a href="/build_logs/{{failure}}/latest.txt">{{failure}}</a></li>
+{% endfor -%}
+</ul>
+<h1>Unstable builds</h1>
+<ul>
+{% for project in unstable -%}
+<li><a href="/build_logs/{{project}}/latest.txt">{{project}}</a></li>
+{% endfor -%}
+</ul>
+<h1>Healthy builds</h1>
+<ul>
+{% for success in successes -%}
+<li><a href="/build_logs/{{success}}/latest.txt">{{success}}</a></li>
+{% endfor -%}
+</ul>
+<p>Last updated {{ last_updated }} (UTC)</p>
+</body>