Adding msan builder to gcp (#4234)

* Adding msan builder to gcp

* Formatting changes

* Refactoring and reducing redundancy

* Moving msan builder entry point into base_images

infra/build/functions/base_images.py

@@ -21,37 +21,56 @@ import google.auth
 from googleapiclient.discovery import build
 
 import build_base_images
+import build_msan_libs
 
 BASE_PROJECT = 'oss-fuzz-base'
 
 
 # pylint: disable=no-member
-def base_builder(event, context):
-  """Cloud function to build base images."""
-  del event, context
+def run_build(steps, images):
+  """Execute the retrieved build steps in gcp."""
   credentials, _ = google.auth.default()
-  tag_prefix = f'gcr.io/{BASE_PROJECT}/'
   build_body = {
-      'steps':
-          build_base_images.get_steps(build_base_images.BASE_IMAGES,
-                                      tag_prefix),
-      'timeout':
-          str(4 * 3600) + 's',
+      'steps': steps,
+      'timeout': str(6 * 3600) + 's',
       'options': {
           'machineType': 'N1_HIGHCPU_32'
       },
-      'images': [
-          tag_prefix + base_image
-          for base_image in build_base_images.BASE_IMAGES
-      ],
+      'images': images
   }
   cloudbuild = build('cloudbuild',
                      'v1',
                      credentials=credentials,
                      cache_discovery=False)
-  build_info = cloudbuild.projects().builds().create(projectId=BASE_PROJECT,
+  build_info = cloudbuild.projects().builds().create(project_id=BASE_PROJECT,
                                                      body=build_body).execute()
   build_id = build_info['metadata']['build']['id']
   logging.info('Build ID: %s', build_id)
   logging.info('Logs: %s',
                build_base_images.get_logs_url(build_id, BASE_PROJECT))
+
+
+def base_builder(event, context):
+  """Cloud function to build base images."""
+  del event, context
+
+  tag_prefix = f'gcr.io/{BASE_PROJECT}/'
+  steps = build_base_images.get_steps(build_base_images.BASE_IMAGES, tag_prefix)
+  images = [
+      tag_prefix + base_image for base_image in build_base_images.BASE_IMAGES
+  ]
+
+  run_build(steps, images)
+
+
+def base_msan_builder(event, context):
+  """Cloud function to build base images."""
+  del event, context
+  image = f'gcr.io/{BASE_PROJECT}/msan-libs-builder'
+  steps = build_msan_libs.get_steps(image)
+  images = [
+      f'gcr.io/{BASE_PROJECT}/base-sanitizer-libs-builder',
+      image,
+  ]
+
+  run_build(steps, images)

infra/build/functions/build_msan_libs.py

@@ -0,0 +1 @@
+../../gcb/build_msan_libs.py

infra/build/functions/deploy.sh

@@ -125,6 +125,11 @@ deploy_cloud_function base-image-build \
 					  $BASE_IMAGE_JOB_TOPIC \
 					  $PROJECT_ID
 
+deploy_cloud_function base-msan-build \
+					  build_msan \
+					  $BASE_IMAGE_JOB_TOPIC \
+					  $PROJECT_ID
+
 deploy_cloud_function request-build \
 					  build_project \
 					  $BUILD_JOB_TOPIC \

infra/build/functions/main.py

@@ -45,3 +45,8 @@ def coverage_build(event, context):
 def builds_status(event, context):
   """Entry point for builds status cloud function."""
   update_build_status.update_status(event, context)
+
+
+def build_msan(event, context):
+  """Entry point for base msan builder."""
+  base_images.base_msan_builder(event, context)

infra/gcb/build_msan_libs.py

@@ -1,13 +1,29 @@
+# Copyright 2020 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+################################################################################
 #!/usr/bin/python2
 """Build base images on Google Cloud Builder.
 
 Usage: build_base_images.py
 """
+from __future__ import print_function
 
 import datetime
 import os
-import yaml
 import sys
+import yaml
 
 from oauth2client.client import GoogleCredentials
 from googleapiclient.discovery import build
@@ -15,18 +31,16 @@ from googleapiclient.discovery import build
 import build_base_images
 
 
-def main():
-  options = {}
-  if 'GCB_OPTIONS' in os.environ:
-    options = yaml.safe_load(os.environ['GCB_OPTIONS'])
+def get_steps(image):
+  """Get build steps for msan-libs-builder."""
+
+  timestamp = datetime.datetime.utcnow().strftime('%Y%m%d%H%M')
+  upload_name = 'msan-libs-' + timestamp + '.zip'
 
-  image = 'gcr.io/oss-fuzz-base/msan-libs-builder'
   steps = build_base_images.get_steps([
       'base-sanitizer-libs-builder',
       'msan-libs-builder',
   ])
-  ts = datetime.datetime.utcnow().strftime('%Y%m%d%H%M')
-  upload_name = 'msan-libs-' + ts + '.zip'
 
   steps.extend([{
       'name': image,
@@ -45,6 +59,18 @@ def main():
       ],
   }])
 
+  return steps
+
+
+# pylint: disable=no-member
+def main():
+  """Build msan libs."""
+  options = {}
+  if 'GCB_OPTIONS' in os.environ:
+    options = yaml.safe_load(os.environ['GCB_OPTIONS'])
+
+  image = 'gcr.io/oss-fuzz-base/msan-libs-builder'
+  steps = get_steps(image)
   build_body = {
       'steps': steps,
       'timeout': str(6 * 3600) + 's',
@@ -54,15 +80,14 @@ def main():
           image,
       ],
   }
-
   credentials = GoogleCredentials.get_application_default()
   cloudbuild = build('cloudbuild', 'v1', credentials=credentials)
   build_info = cloudbuild.projects().builds().create(projectId='oss-fuzz-base',
                                                      body=build_body).execute()
   build_id = build_info['metadata']['build']['id']
 
-  print >> sys.stderr, 'Logs:', build_base_images.get_logs_url(build_id)
-  print build_id
+  print('Logs:', build_base_images.get_logs_url(build_id), file=sys.stderr)
+  print(build_id)
 
 
 if __name__ == '__main__':