Rooba
/
oss-fuzz
mirror of https://github.com/google/oss-fuzz.git
1
1
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Many build infra fixes. (#4209) 

- Remove base image project argument from deploy.sh. Deploy all
  functions to the main oss-fuzz image.
- Reduce max instances of functions to 1 to avoid rate limiting issues.
- Fix missing ndb context initialization in request_build.py
- Fix incorrect tags in BuildHistory entities. "-" was doubled.
- Fix base build deployment with incorrect schedule variable.
- Add scripts for requesting builds locally.
This commit is contained in:
Oliver Chang 2020-07-29 12:39:49 +10:00 committed by GitHub
parent e1920a468d
commit 4e2a7c07e5
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 84 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
infra/build/functions/base_images.py
View File

@ -22,13 +22,15 @@ from googleapiclient.discovery import build
import build_base_images
BASE_PROJECT = 'oss-fuzz-base'
# pylint: disable=no-member
def base_builder(event, context):
"""Cloud function to build base images."""
del event, context
credentials, project_id = google.auth.default()
tag_prefix = f'gcr.io/{project_id}/'
credentials, _ = google.auth.default()
tag_prefix = f'gcr.io/{BASE_PROJECT}/'
build_body = {
'steps':
build_base_images.get_steps(build_base_images.BASE_IMAGES,
@ -47,8 +49,9 @@ def base_builder(event, context):
'v1',
credentials=credentials,
cache_discovery=False)
build_info = cloudbuild.projects().builds().create(projectId=project_id,
build_info = cloudbuild.projects().builds().create(projectId=BASE_PROJECT,
body=build_body).execute()
build_id = build_info['metadata']['build']['id']
logging.info('Build ID: %s', build_id)
logging.info('Logs: %s', build_base_images.get_logs_url(build_id, project_id))
logging.info('Logs: %s',
build_base_images.get_logs_url(build_id, BASE_PROJECT))

2
infra/build/functions/datastore_entities.py
View File

@ -36,6 +36,6 @@ class GithubCreds(ndb.Model):
# pylint: disable=too-few-public-methods
class BuildsHistory(ndb.Model):
"""Container for build history of projects."""
build_tag_suffix = ndb.StringProperty()
build_tag = ndb.StringProperty()
project = ndb.StringProperty()
build_ids = ndb.StringProperty(repeated=True)

18
infra/build/functions/deploy.sh
View File

@ -83,19 +83,19 @@ function deploy_cloud_function {
--trigger-topic $topic \
--runtime python37 \
--project $project \
--timeout 540
--timeout 540 \
--max-instances 1
}
if [ $# == 2 ]; then
if [ $# == 1 ]; then
PROJECT_ID=$1
BASE_PROJECT_ID=$2
else
echo -e "\n Usage ./deploy.sh <project-name> <base-project-name>"; exit;
echo -e "\n Usage ./deploy.sh <project-name>"; exit;
fi
deploy_pubsub_topic $BUILD_JOB_TOPIC $PROJECT_ID
deploy_pubsub_topic $SYNC_JOB_TOPIC $PROJECT_ID
deploy_pubsub_topic $BASE_IMAGE_JOB_TOPIC $BASE_PROJECT_ID
deploy_pubsub_topic $BASE_IMAGE_JOB_TOPIC $PROJECT_ID
deploy_pubsub_topic $COVERAGE_BUILD_JOB_TOPIC $PROJECT_ID
deploy_pubsub_topic $UPDATE_BUILD_JOB_TOPIC $PROJECT_ID
@ -106,10 +106,10 @@ deploy_scheduler $SYNC_SCHEDULER_JOB \
$PROJECT_ID
deploy_scheduler $BASE_IMAGE_SCHEDULER_JOB \
"$BASE_IMAGE SCHEDULE" \
"$BASE_IMAGE_SCHEDULE" \
$BASE_IMAGE_JOB_TOPIC \
"$BASE_IMAGE_MESSAGE" \
$BASE_PROJECT_ID
$PROJECT_ID
deploy_scheduler $COVERAGE_BUILD_SCHEDULER_JOB \
"$COVERAGE_BUILD_SCHEDULE" \
@ -121,7 +121,7 @@ deploy_scheduler $UPDATE_BUILD_SCHEDULER_JOB \
"$UPDATE_BUILD_JOB_SCHEDULE" \
$UPDATE_BUILD_JOB_TOPIC \
"$UPDATE_BUILD_MESSAGE" \
$PROJECT_ID
$PROJECT_ID
deploy_cloud_function sync \
@ -132,7 +132,7 @@ deploy_cloud_function sync \
deploy_cloud_function base-image-build \
build_base_images \
$BASE_IMAGE_JOB_TOPIC \
$BASE_PROJECT_ID
$PROJECT_ID
deploy_cloud_function request-build \
build_project \

31
infra/build/functions/request_build.py
View File

@ -30,14 +30,14 @@ BASE_PROJECT = 'oss-fuzz-base'
MAX_BUILD_HISTORY_LENGTH = 64
def update_build_history(project_name, build_id, build_tag_suffix):
def update_build_history(project_name, build_id, build_tag):
"""Update build history of project."""
project_key = ndb.Key(BuildsHistory, project_name + build_tag_suffix)
project_key = ndb.Key(BuildsHistory, project_name + build_tag)
project = project_key.get()
if not project:
project = BuildsHistory(id=project_name + '-' + build_tag_suffix,
build_tag_suffix=build_tag_suffix,
project = BuildsHistory(id=project_name + '-' + build_tag,
build_tag=build_tag,
project=project_name,
build_ids=[])
@ -50,14 +50,13 @@ def update_build_history(project_name, build_id, build_tag_suffix):
def get_project_data(project_name):
"""Retrieve project metadata from datastore."""
with ndb.Client().context():
query = Project.query(Project.name == project_name)
project = query.get()
if not project:
raise RuntimeError(
'Project {0} not available in cloud datastore'.format(project_name))
project_yaml_contents = project.project_yaml_contents
dockerfile_lines = project.dockerfile_contents.split('\n')
query = Project.query(Project.name == project_name)
project = query.get()
if not project:
raise RuntimeError(
'Project {0} not available in cloud datastore'.format(project_name))
project_yaml_contents = project.project_yaml_contents
dockerfile_lines = project.dockerfile_contents.split('\n')
return (project_yaml_contents, dockerfile_lines)
@ -107,6 +106,8 @@ def request_build(event, context):
else:
raise RuntimeError('Project name missing from payload')
credentials, image_project = google.auth.default()
build_steps = get_build_steps(project_name, image_project, BASE_PROJECT)
run_build(project_name, image_project, build_steps, credentials, '-fuzzing')
with ndb.Client().context():
credentials, image_project = google.auth.default()
build_steps = get_build_steps(project_name, image_project, BASE_PROJECT)
run_build(project_name, image_project, build_steps, credentials,
build_project.FUZZING_BUILD_TAG)

7
infra/build/functions/request_build_test.py
View File

@ -65,9 +65,8 @@ class TestRequestBuilds(unittest.TestCase):
Project(name='test-project',
project_yaml_contents=project_yaml_contents,
dockerfile_contents='test line').put()
build_steps = get_build_steps('test-project', image_project,
base_images_project)
build_steps = get_build_steps('test-project', image_project,
base_images_project)
self.assertEqual(build_steps, expected_build_steps)
def test_get_build_steps_no_project(self):
@ -80,7 +79,7 @@ class TestRequestBuilds(unittest.TestCase):
"""Testing build history."""
with ndb.Client().context():
BuildsHistory(id='test-project-fuzzing',
build_tag_suffix='fuzzing',
build_tag='fuzzing',
project='test-project',
build_ids=[str(i) for i in range(1, 65)]).put()
update_build_history('test-project', '65', '-fuzzing')

4
infra/build/functions/request_coverage_build.py
View File

@ -43,5 +43,7 @@ def request_coverage_build(event, context):
image_project, BASE_PROJECT)
except SystemExit:
continue
request_build.run_build(project_name, image_project, build_steps,
credentials, '-coverage')
credentials,
build_and_run_coverage.COVERAGE_BUILD_TAG)

7
infra/build/functions/update_build_status.py
View File

@ -58,13 +58,13 @@ def get_last_build(build_ids):
return None
def update_build_status(build_tag_suffix, status_filename):
def update_build_status(build_tag, status_filename):
"""Update build statuses."""
statuses = {}
successes = []
failures = []
for project_build in BuildsHistory.query(
BuildsHistory.build_tag_suffix == build_tag_suffix):
BuildsHistory.build_tag == build_tag):
last_build = get_last_build(project_build.build_ids)
if not last_build:
logging.error('Failed to get last build for project %s',
@ -136,7 +136,8 @@ def update_status(event, context):
status_filename='status-coverage.json')
for project in Project.query():
if project.name not in project_build_statuses or project.name not in coverage_build_statuses:
if (project.name not in project_build_statuses or
project.name not in coverage_build_statuses):
continue
update_build_badges(project.name, project_build_statuses[project.name],

24
infra/build/request_all_builds.sh Executable file
View File

@ -0,0 +1,24 @@
#!/bin/bash -ex
# Copyright 2020 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
################################################################################
for project in ../../projects/*; do
if [[ ! -f $project/Dockerfile ]]; then
continue
fi
./request_build.sh $(basename $project)
done

17
infra/build/request_build.sh Executable file
View File

@ -0,0 +1,17 @@
#!/bin/bash -ex
# Copyright 2020 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
################################################################################
gcloud pubsub topics publish request-build --message "$1" --project oss-fuzz