diff --git a/infra/build/functions/base_images.py b/infra/build/functions/base_images.py index f3f132240..a56e9ffb5 100644 --- a/infra/build/functions/base_images.py +++ b/infra/build/functions/base_images.py @@ -22,13 +22,15 @@ from googleapiclient.discovery import build import build_base_images +BASE_PROJECT = 'oss-fuzz-base' + # pylint: disable=no-member def base_builder(event, context): """Cloud function to build base images.""" del event, context - credentials, project_id = google.auth.default() - tag_prefix = f'gcr.io/{project_id}/' + credentials, _ = google.auth.default() + tag_prefix = f'gcr.io/{BASE_PROJECT}/' build_body = { 'steps': build_base_images.get_steps(build_base_images.BASE_IMAGES, @@ -47,8 +49,9 @@ def base_builder(event, context): 'v1', credentials=credentials, cache_discovery=False) - build_info = cloudbuild.projects().builds().create(projectId=project_id, + build_info = cloudbuild.projects().builds().create(projectId=BASE_PROJECT, body=build_body).execute() build_id = build_info['metadata']['build']['id'] logging.info('Build ID: %s', build_id) - logging.info('Logs: %s', build_base_images.get_logs_url(build_id, project_id)) + logging.info('Logs: %s', + build_base_images.get_logs_url(build_id, BASE_PROJECT)) diff --git a/infra/build/functions/datastore_entities.py b/infra/build/functions/datastore_entities.py index 1946cb521..8683803c7 100644 --- a/infra/build/functions/datastore_entities.py +++ b/infra/build/functions/datastore_entities.py @@ -36,6 +36,6 @@ class GithubCreds(ndb.Model): # pylint: disable=too-few-public-methods class BuildsHistory(ndb.Model): """Container for build history of projects.""" - build_tag_suffix = ndb.StringProperty() + build_tag = ndb.StringProperty() project = ndb.StringProperty() build_ids = ndb.StringProperty(repeated=True) diff --git a/infra/build/functions/deploy.sh b/infra/build/functions/deploy.sh index 0c361a3d4..26f99cbb4 100755 --- a/infra/build/functions/deploy.sh +++ b/infra/build/functions/deploy.sh @@ -83,19 +83,19 @@ function deploy_cloud_function { --trigger-topic $topic \ --runtime python37 \ --project $project \ - --timeout 540 + --timeout 540 \ + --max-instances 1 } -if [ $# == 2 ]; then +if [ $# == 1 ]; then PROJECT_ID=$1 - BASE_PROJECT_ID=$2 else - echo -e "\n Usage ./deploy.sh "; exit; + echo -e "\n Usage ./deploy.sh "; exit; fi deploy_pubsub_topic $BUILD_JOB_TOPIC $PROJECT_ID deploy_pubsub_topic $SYNC_JOB_TOPIC $PROJECT_ID -deploy_pubsub_topic $BASE_IMAGE_JOB_TOPIC $BASE_PROJECT_ID +deploy_pubsub_topic $BASE_IMAGE_JOB_TOPIC $PROJECT_ID deploy_pubsub_topic $COVERAGE_BUILD_JOB_TOPIC $PROJECT_ID deploy_pubsub_topic $UPDATE_BUILD_JOB_TOPIC $PROJECT_ID @@ -106,10 +106,10 @@ deploy_scheduler $SYNC_SCHEDULER_JOB \ $PROJECT_ID deploy_scheduler $BASE_IMAGE_SCHEDULER_JOB \ - "$BASE_IMAGE SCHEDULE" \ + "$BASE_IMAGE_SCHEDULE" \ $BASE_IMAGE_JOB_TOPIC \ "$BASE_IMAGE_MESSAGE" \ - $BASE_PROJECT_ID + $PROJECT_ID deploy_scheduler $COVERAGE_BUILD_SCHEDULER_JOB \ "$COVERAGE_BUILD_SCHEDULE" \ @@ -121,7 +121,7 @@ deploy_scheduler $UPDATE_BUILD_SCHEDULER_JOB \ "$UPDATE_BUILD_JOB_SCHEDULE" \ $UPDATE_BUILD_JOB_TOPIC \ "$UPDATE_BUILD_MESSAGE" \ - $PROJECT_ID + $PROJECT_ID deploy_cloud_function sync \ @@ -132,7 +132,7 @@ deploy_cloud_function sync \ deploy_cloud_function base-image-build \ build_base_images \ $BASE_IMAGE_JOB_TOPIC \ - $BASE_PROJECT_ID + $PROJECT_ID deploy_cloud_function request-build \ build_project \ diff --git a/infra/build/functions/request_build.py b/infra/build/functions/request_build.py index c50c6e3d5..b7046c7eb 100644 --- a/infra/build/functions/request_build.py +++ b/infra/build/functions/request_build.py @@ -30,14 +30,14 @@ BASE_PROJECT = 'oss-fuzz-base' MAX_BUILD_HISTORY_LENGTH = 64 -def update_build_history(project_name, build_id, build_tag_suffix): +def update_build_history(project_name, build_id, build_tag): """Update build history of project.""" - project_key = ndb.Key(BuildsHistory, project_name + build_tag_suffix) + project_key = ndb.Key(BuildsHistory, project_name + build_tag) project = project_key.get() if not project: - project = BuildsHistory(id=project_name + '-' + build_tag_suffix, - build_tag_suffix=build_tag_suffix, + project = BuildsHistory(id=project_name + '-' + build_tag, + build_tag=build_tag, project=project_name, build_ids=[]) @@ -50,14 +50,13 @@ def update_build_history(project_name, build_id, build_tag_suffix): def get_project_data(project_name): """Retrieve project metadata from datastore.""" - with ndb.Client().context(): - query = Project.query(Project.name == project_name) - project = query.get() - if not project: - raise RuntimeError( - 'Project {0} not available in cloud datastore'.format(project_name)) - project_yaml_contents = project.project_yaml_contents - dockerfile_lines = project.dockerfile_contents.split('\n') + query = Project.query(Project.name == project_name) + project = query.get() + if not project: + raise RuntimeError( + 'Project {0} not available in cloud datastore'.format(project_name)) + project_yaml_contents = project.project_yaml_contents + dockerfile_lines = project.dockerfile_contents.split('\n') return (project_yaml_contents, dockerfile_lines) @@ -107,6 +106,8 @@ def request_build(event, context): else: raise RuntimeError('Project name missing from payload') - credentials, image_project = google.auth.default() - build_steps = get_build_steps(project_name, image_project, BASE_PROJECT) - run_build(project_name, image_project, build_steps, credentials, '-fuzzing') + with ndb.Client().context(): + credentials, image_project = google.auth.default() + build_steps = get_build_steps(project_name, image_project, BASE_PROJECT) + run_build(project_name, image_project, build_steps, credentials, + build_project.FUZZING_BUILD_TAG) diff --git a/infra/build/functions/request_build_test.py b/infra/build/functions/request_build_test.py index 57ff3fe4c..9589c09b4 100644 --- a/infra/build/functions/request_build_test.py +++ b/infra/build/functions/request_build_test.py @@ -65,9 +65,8 @@ class TestRequestBuilds(unittest.TestCase): Project(name='test-project', project_yaml_contents=project_yaml_contents, dockerfile_contents='test line').put() - - build_steps = get_build_steps('test-project', image_project, - base_images_project) + build_steps = get_build_steps('test-project', image_project, + base_images_project) self.assertEqual(build_steps, expected_build_steps) def test_get_build_steps_no_project(self): @@ -80,7 +79,7 @@ class TestRequestBuilds(unittest.TestCase): """Testing build history.""" with ndb.Client().context(): BuildsHistory(id='test-project-fuzzing', - build_tag_suffix='fuzzing', + build_tag='fuzzing', project='test-project', build_ids=[str(i) for i in range(1, 65)]).put() update_build_history('test-project', '65', '-fuzzing') diff --git a/infra/build/functions/request_coverage_build.py b/infra/build/functions/request_coverage_build.py index d874f0ead..0bc57e44e 100644 --- a/infra/build/functions/request_coverage_build.py +++ b/infra/build/functions/request_coverage_build.py @@ -43,5 +43,7 @@ def request_coverage_build(event, context): image_project, BASE_PROJECT) except SystemExit: continue + request_build.run_build(project_name, image_project, build_steps, - credentials, '-coverage') + credentials, + build_and_run_coverage.COVERAGE_BUILD_TAG) diff --git a/infra/build/functions/update_build_status.py b/infra/build/functions/update_build_status.py index 490efab9c..618a2a58e 100644 --- a/infra/build/functions/update_build_status.py +++ b/infra/build/functions/update_build_status.py @@ -58,13 +58,13 @@ def get_last_build(build_ids): return None -def update_build_status(build_tag_suffix, status_filename): +def update_build_status(build_tag, status_filename): """Update build statuses.""" statuses = {} successes = [] failures = [] for project_build in BuildsHistory.query( - BuildsHistory.build_tag_suffix == build_tag_suffix): + BuildsHistory.build_tag == build_tag): last_build = get_last_build(project_build.build_ids) if not last_build: logging.error('Failed to get last build for project %s', @@ -136,7 +136,8 @@ def update_status(event, context): status_filename='status-coverage.json') for project in Project.query(): - if project.name not in project_build_statuses or project.name not in coverage_build_statuses: + if (project.name not in project_build_statuses or + project.name not in coverage_build_statuses): continue update_build_badges(project.name, project_build_statuses[project.name], diff --git a/infra/build/request_all_builds.sh b/infra/build/request_all_builds.sh new file mode 100755 index 000000000..2c7f39be9 --- /dev/null +++ b/infra/build/request_all_builds.sh @@ -0,0 +1,24 @@ +#!/bin/bash -ex +# Copyright 2020 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +################################################################################ + +for project in ../../projects/*; do + if [[ ! -f $project/Dockerfile ]]; then + continue + fi + + ./request_build.sh $(basename $project) +done diff --git a/infra/build/request_build.sh b/infra/build/request_build.sh new file mode 100755 index 000000000..58e1de4c7 --- /dev/null +++ b/infra/build/request_build.sh @@ -0,0 +1,17 @@ +#!/bin/bash -ex +# Copyright 2020 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +################################################################################ +gcloud pubsub topics publish request-build --message "$1" --project oss-fuzz