diff --git a/infra/build/functions/base_image/build_base_images.py b/infra/build/functions/base_image/build_base_images.py new file mode 120000 index 000000000..7541a4f9a --- /dev/null +++ b/infra/build/functions/base_image/build_base_images.py @@ -0,0 +1 @@ +../../../gcb/build_base_images.py \ No newline at end of file diff --git a/infra/build/functions/base_image/deploy.sh b/infra/build/functions/base_image/deploy.sh new file mode 100755 index 000000000..ab982e682 --- /dev/null +++ b/infra/build/functions/base_image/deploy.sh @@ -0,0 +1,55 @@ +# Copyright 2020 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +################################################################################ + +JOB_TOPIC=schedule-base-image-build +SCHEDULER_JOB=base-image-scheduler +JOB_SCHEDULE="0 3 * * *" +MESSAGE="Start base image build" +ENTRY_POINT=base_builder + +if [ "$1" ]; then + PROJECT_ID=$1 +else + echo -e "\n Usage ./deploy.sh my-project-name"; exit; +fi + +# Checking if the given pubsub topic exists +if ! gcloud pubsub topics describe $JOB_TOPIC --project $PROJECT_ID ; + then + gcloud pubsub topics create $JOB_TOPIC \ + --project $PROJECT_ID +fi +# Checking if the given scheduler job exists +if gcloud scheduler jobs describe $SCHEDULER_JOB --project $PROJECT_ID ; + then + gcloud scheduler jobs update pubsub $SCHEDULER_JOB \ + --schedule "$JOB_SCHEDULE" \ + --topic $JOB_TOPIC \ + --message-body "$MESSAGE" \ + --project $PROJECT_ID + else + gcloud scheduler jobs create pubsub $SCHEDULER_JOB \ + --schedule "$JOB_SCHEDULE" \ + --topic $JOB_TOPIC \ + --message-body "$MESSAGE" \ + --project $PROJECT_ID +fi + +gcloud functions deploy base-image-build \ + --entry-point $ENTRY_POINT \ + --trigger-topic $JOB_TOPIC \ + --runtime python37 \ + --project $PROJECT_ID diff --git a/infra/build/functions/base_image/main.py b/infra/build/functions/base_image/main.py new file mode 100644 index 000000000..f3f132240 --- /dev/null +++ b/infra/build/functions/base_image/main.py @@ -0,0 +1,54 @@ +# Copyright 2020 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +################################################################################ +"""Cloud function to build base images on Google Cloud Builder.""" + +import logging + +import google.auth +from googleapiclient.discovery import build + +import build_base_images + + +# pylint: disable=no-member +def base_builder(event, context): + """Cloud function to build base images.""" + del event, context + credentials, project_id = google.auth.default() + tag_prefix = f'gcr.io/{project_id}/' + build_body = { + 'steps': + build_base_images.get_steps(build_base_images.BASE_IMAGES, + tag_prefix), + 'timeout': + str(4 * 3600) + 's', + 'options': { + 'machineType': 'N1_HIGHCPU_32' + }, + 'images': [ + tag_prefix + base_image + for base_image in build_base_images.BASE_IMAGES + ], + } + cloudbuild = build('cloudbuild', + 'v1', + credentials=credentials, + cache_discovery=False) + build_info = cloudbuild.projects().builds().create(projectId=project_id, + body=build_body).execute() + build_id = build_info['metadata']['build']['id'] + logging.info('Build ID: %s', build_id) + logging.info('Logs: %s', build_base_images.get_logs_url(build_id, project_id)) diff --git a/infra/build/functions/base_image/requirements.txt b/infra/build/functions/base_image/requirements.txt new file mode 120000 index 000000000..dc833dd4b --- /dev/null +++ b/infra/build/functions/base_image/requirements.txt @@ -0,0 +1 @@ +../requirements.txt \ No newline at end of file diff --git a/infra/build/functions/requirements.txt b/infra/build/functions/requirements.txt index 978831217..d623f8f5b 100644 --- a/infra/build/functions/requirements.txt +++ b/infra/build/functions/requirements.txt @@ -17,6 +17,9 @@ PyYaml==5.1 PyGithub==1.51 grpcio==1.29.0 +google-auth==1.18.0 google-cloud-ndb==1.3.0 google-cloud-scheduler==1.3.0 -google-api-core==1.21.0 \ No newline at end of file +google-api-core==1.21.0 +google-api-python-client==1.9.3 +oauth2client==4.1.3 diff --git a/infra/gcb/build_base_images.py b/infra/gcb/build_base_images.py index 3c2bace2e..a950c85a1 100755 --- a/infra/gcb/build_base_images.py +++ b/infra/gcb/build_base_images.py @@ -1,15 +1,31 @@ +# Copyright 2020 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +################################################################################ #!/usr/bin/python2 """Build base images on Google Cloud Builder. Usage: build_base_images.py """ +from __future__ import print_function import os import sys import yaml -from oauth2client.client import GoogleCredentials from googleapiclient.discovery import build +from oauth2client.client import GoogleCredentials BASE_IMAGES = [ 'base-image', @@ -23,7 +39,8 @@ BASE_IMAGES = [ TAG_PREFIX = 'gcr.io/oss-fuzz-base/' -def get_steps(images): +def get_steps(images, tag_prefix=TAG_PREFIX): + """Returns build steps for given images.""" steps = [{ 'args': [ 'clone', @@ -37,7 +54,7 @@ def get_steps(images): 'args': [ 'build', '-t', - TAG_PREFIX + base_image, + tag_prefix + base_image, '.', ], 'dir': 'oss-fuzz/infra/base-images/' + base_image, @@ -47,12 +64,14 @@ def get_steps(images): return steps -def get_logs_url(build_id): - URL_FORMAT = ('https://console.developers.google.com/logs/viewer?' - 'resource=build%2Fbuild_id%2F{0}&project=oss-fuzz-base') - return URL_FORMAT.format(build_id) +def get_logs_url(build_id, project_id='oss-fuzz-base'): + """Returns url that displays the build logs.""" + url_format = ('https://console.developers.google.com/logs/viewer?' + 'resource=build%2Fbuild_id%2F{0}&project={1}') + return url_format.format(build_id, project_id) +# pylint: disable=no-member, missing-function-docstring def main(): options = {} if 'GCB_OPTIONS' in os.environ: @@ -67,12 +86,12 @@ def main(): credentials = GoogleCredentials.get_application_default() cloudbuild = build('cloudbuild', 'v1', credentials=credentials) - build_info = cloudbuild.projects().builds().create( - projectId='oss-fuzz-base', body=build_body).execute() + build_info = cloudbuild.projects().builds().create(projectId='oss-fuzz-base', + body=build_body).execute() build_id = build_info['metadata']['build']['id'] - print >> sys.stderr, 'Logs:', get_logs_url(build_id) - print build_id + print('Logs:', get_logs_url(build_id), file=sys.stderr) + print(build_id) if __name__ == '__main__':