Base image builder (#4080)

* Base image builder

* Forgot to run formatter

* Fixing lint issues and removing unused imports

* Adding missing newline

* Refactoring code

* Fixing linting errors and changing name of cloud function entry

* Adding license header to build_base_images

* Changed logging and print_function

Co-authored-by: Kabeer Seth <kabeerseth@google.com>

infra/build/functions/base_image/build_base_images.py

@@ -0,0 +1 @@
+../../../gcb/build_base_images.py

infra/build/functions/base_image/deploy.sh

@@ -0,0 +1,55 @@
+# Copyright 2020 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+################################################################################
+
+JOB_TOPIC=schedule-base-image-build
+SCHEDULER_JOB=base-image-scheduler
+JOB_SCHEDULE="0 3 * * *"
+MESSAGE="Start base image build"
+ENTRY_POINT=base_builder
+
+if [ "$1" ]; then
+	PROJECT_ID=$1
+else
+	echo -e "\n Usage ./deploy.sh my-project-name"; exit;
+fi
+
+# Checking if the given pubsub topic exists
+if ! gcloud pubsub topics describe $JOB_TOPIC --project $PROJECT_ID ;
+	then
+		gcloud pubsub topics create $JOB_TOPIC \
+		--project $PROJECT_ID
+fi
+# Checking if the given scheduler job exists
+if gcloud scheduler jobs describe $SCHEDULER_JOB --project $PROJECT_ID ;
+	then
+		gcloud scheduler jobs update pubsub $SCHEDULER_JOB \
+			--schedule "$JOB_SCHEDULE" \
+			--topic $JOB_TOPIC \
+			--message-body "$MESSAGE" \
+			--project $PROJECT_ID
+	else
+		gcloud scheduler jobs create pubsub $SCHEDULER_JOB \
+			--schedule "$JOB_SCHEDULE" \
+			--topic $JOB_TOPIC \
+			--message-body "$MESSAGE" \
+			--project $PROJECT_ID
+fi
+
+gcloud functions deploy base-image-build \
+	--entry-point $ENTRY_POINT \
+	--trigger-topic $JOB_TOPIC \
+	--runtime python37 \
+	--project $PROJECT_ID

infra/build/functions/base_image/main.py

@@ -0,0 +1,54 @@
+# Copyright 2020 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+################################################################################
+"""Cloud function to build base images on Google Cloud Builder."""
+
+import logging
+
+import google.auth
+from googleapiclient.discovery import build
+
+import build_base_images
+
+
+# pylint: disable=no-member
+def base_builder(event, context):
+  """Cloud function to build base images."""
+  del event, context
+  credentials, project_id = google.auth.default()
+  tag_prefix = f'gcr.io/{project_id}/'
+  build_body = {
+      'steps':
+          build_base_images.get_steps(build_base_images.BASE_IMAGES,
+                                      tag_prefix),
+      'timeout':
+          str(4 * 3600) + 's',
+      'options': {
+          'machineType': 'N1_HIGHCPU_32'
+      },
+      'images': [
+          tag_prefix + base_image
+          for base_image in build_base_images.BASE_IMAGES
+      ],
+  }
+  cloudbuild = build('cloudbuild',
+                     'v1',
+                     credentials=credentials,
+                     cache_discovery=False)
+  build_info = cloudbuild.projects().builds().create(projectId=project_id,
+                                                     body=build_body).execute()
+  build_id = build_info['metadata']['build']['id']
+  logging.info('Build ID: %s', build_id)
+  logging.info('Logs: %s', build_base_images.get_logs_url(build_id, project_id))

infra/build/functions/base_image/requirements.txt

@@ -0,0 +1 @@
+../requirements.txt

infra/build/functions/requirements.txt

@@ -17,6 +17,9 @@
 PyYaml==5.1
 PyGithub==1.51
 grpcio==1.29.0
+google-auth==1.18.0
 google-cloud-ndb==1.3.0
 google-cloud-scheduler==1.3.0
-google-api-core==1.21.0
+google-api-core==1.21.0
+google-api-python-client==1.9.3
+oauth2client==4.1.3

infra/gcb/build_base_images.py

@@ -1,15 +1,31 @@
+# Copyright 2020 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+################################################################################
 #!/usr/bin/python2
 """Build base images on Google Cloud Builder.
 
 Usage: build_base_images.py
 """
+from __future__ import print_function
 
 import os
 import sys
 import yaml
 
-from oauth2client.client import GoogleCredentials
 from googleapiclient.discovery import build
+from oauth2client.client import GoogleCredentials
 
 BASE_IMAGES = [
     'base-image',
@@ -23,7 +39,8 @@ BASE_IMAGES = [
 TAG_PREFIX = 'gcr.io/oss-fuzz-base/'
 
 
-def get_steps(images):
+def get_steps(images, tag_prefix=TAG_PREFIX):
+  """Returns build steps for given images."""
   steps = [{
       'args': [
           'clone',
@@ -37,7 +54,7 @@ def get_steps(images):
         'args': [
             'build',
             '-t',
-            TAG_PREFIX + base_image,
+            tag_prefix + base_image,
             '.',
         ],
         'dir': 'oss-fuzz/infra/base-images/' + base_image,
@@ -47,12 +64,14 @@ def get_steps(images):
   return steps
 
 
-def get_logs_url(build_id):
-  URL_FORMAT = ('https://console.developers.google.com/logs/viewer?'
-                'resource=build%2Fbuild_id%2F{0}&project=oss-fuzz-base')
-  return URL_FORMAT.format(build_id)
+def get_logs_url(build_id, project_id='oss-fuzz-base'):
+  """Returns url that displays the build logs."""
+  url_format = ('https://console.developers.google.com/logs/viewer?'
+                'resource=build%2Fbuild_id%2F{0}&project={1}')
+  return url_format.format(build_id, project_id)
 
 
+# pylint: disable=no-member, missing-function-docstring
 def main():
   options = {}
   if 'GCB_OPTIONS' in os.environ:
@@ -67,12 +86,12 @@ def main():
 
   credentials = GoogleCredentials.get_application_default()
   cloudbuild = build('cloudbuild', 'v1', credentials=credentials)
-  build_info = cloudbuild.projects().builds().create(
-      projectId='oss-fuzz-base', body=build_body).execute()
+  build_info = cloudbuild.projects().builds().create(projectId='oss-fuzz-base',
+                                                     body=build_body).execute()
   build_id = build_info['metadata']['build']['id']
 
-  print >> sys.stderr, 'Logs:', get_logs_url(build_id)
-  print build_id
+  print('Logs:', get_logs_url(build_id), file=sys.stderr)
+  print(build_id)
 
 
 if __name__ == '__main__':