From d06de9716a928c3d6fee0139c948eaa919d13d54 Mon Sep 17 00:00:00 2001 From: Oliver Chang Date: Fri, 10 Mar 2017 15:29:30 -0800 Subject: [PATCH] cloudbuild setup (#448) --- .../jenkins_config/base_job_gcb.xml | 45 ++++++++ infra/auto-setup/sync_gcb.py | 100 ++++++++++++++++++ infra/jenkins-cluster/Dockerfile | 17 --- infra/jenkins-cluster/server/Dockerfile | 43 +++----- 4 files changed, 158 insertions(+), 47 deletions(-) create mode 100644 infra/auto-setup/jenkins_config/base_job_gcb.xml create mode 100755 infra/auto-setup/sync_gcb.py delete mode 100644 infra/jenkins-cluster/Dockerfile diff --git a/infra/auto-setup/jenkins_config/base_job_gcb.xml b/infra/auto-setup/jenkins_config/base_job_gcb.xml new file mode 100644 index 000000000..50ef65a49 --- /dev/null +++ b/infra/auto-setup/jenkins_config/base_job_gcb.xml @@ -0,0 +1,45 @@ + + + + + false + + + 2 + + + https://github.com/google/oss-fuzz.git + + + + + */master + + + false + + + + oss-fuzz + + + + true + false + false + false + + false + + + virtualenv ENV +. ENV/bin/activate + +cd $WORKSPACE/oss-fuzz/infra/gcb +pip install -r requirements.txt +python build.py $WORKSPACE/oss-fuzz/$JOB_NAME + + + + + diff --git a/infra/auto-setup/sync_gcb.py b/infra/auto-setup/sync_gcb.py new file mode 100755 index 000000000..617e407b6 --- /dev/null +++ b/infra/auto-setup/sync_gcb.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python +"""Script to sync CF and Jenkins jobs.""" + +import json +import os +import re +import sys +import urllib2 +import yaml +import xml.etree.ElementTree as ET + +import jenkins + +JENKINS_SERVER = ('localhost', 8080) + +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +OSSFUZZ_DIR = os.path.dirname(os.path.dirname(SCRIPT_DIR)) + +VALID_PROJECT_NAME = re.compile(r'^[a-zA-Z0-9_-]+$') + + +def main(): + # Connect to jenkins server. + jenkins_login = get_jenkins_login() + server = jenkins.Jenkins('http://%s:%d' % JENKINS_SERVER, + username=jenkins_login[0], password=jenkins_login[1]) + + for project in get_projects(): + print 'syncing configs for', project + try: + # Create/update jenkins build job. + sync_jenkins_job(server, project) + + except Exception as e: + print >>sys.stderr, 'Failed to setup job with exception', e + + +def _has_dockerfile(project_dir): + """Whether or not the project has a Dockerfile.""" + if os.path.exists(os.path.join(project_dir, 'Dockerfile')): + return True + + project_yaml_path = os.path.join(project_dir, 'project.yaml') + if not os.path.exists(project_yaml_path): + return False + + with open(project_yaml_path) as f: + project_info = yaml.safe_load(f) + + return 'dockerfile' in project_info + + +def get_projects(): + """Return list of projects for oss-fuzz.""" + projects = [] + projects_dir = os.path.join(OSSFUZZ_DIR, 'projects') + for name in os.listdir(projects_dir): + full_path = os.path.join(projects_dir, name) + if not os.path.isdir(full_path) or not _has_dockerfile(full_path): + continue + + if not VALID_PROJECT_NAME.match(name): + print >>sys.stderr, 'Invalid project name:', name + continue + + projects.append(name) + + if not projects: + print >>sys.stderr, 'No projects found.' + + return projects + + +def get_jenkins_login(): + """Returns (username, password) for jenkins.""" + username = os.getenv('JENKINS_USER') + password = os.getenv('JENKINS_PASS') + + return username, password + + +def sync_jenkins_job(server, project): + """Sync the config with jenkins.""" + project_yaml = os.path.join(OSSFUZZ_DIR, 'projects', project, 'project.yaml') + with open(project_yaml, 'r') as f: + project_json_string = json.dumps(json.dumps(yaml.safe_load(f))) + + job_name = 'projects/' + project + with open(os.path.join(SCRIPT_DIR, 'jenkins_config', 'base_job_gcb.xml')) as f: + job_config_xml = f.read() + + if server.job_exists(job_name): + server.reconfig_job(job_name, job_config_xml) + else: + server.create_job(job_name, job_config_xml) + server.build_job(job_name) + + +if __name__ == '__main__': + main() diff --git a/infra/jenkins-cluster/Dockerfile b/infra/jenkins-cluster/Dockerfile deleted file mode 100644 index e58745c1b..000000000 --- a/infra/jenkins-cluster/Dockerfile +++ /dev/null @@ -1,17 +0,0 @@ -FROM jenkinsci/jenkins -USER root - -RUN mkdir /var/secrets -RUN apt-get -y update && apt-get -y upgrade -RUN apt-get -y install python-dev virtualenv python-pip build-essential - -WORKDIR / -RUN wget https://dl.google.com/dl/cloudsdk/release/google-cloud-sdk.zip -RUN unzip google-cloud-sdk.zip - -RUN /google-cloud-sdk/install.sh --usage-reporting=false --bash-completion=false --disable-installation-options -RUN /google-cloud-sdk/bin/gcloud -q components install alpha beta -RUN /google-cloud-sdk/bin/gcloud -q components update - -RUN chown -R jenkins /google-cloud-sdk -ENV PATH=$PATH:/google-cloud-sdk/bin diff --git a/infra/jenkins-cluster/server/Dockerfile b/infra/jenkins-cluster/server/Dockerfile index 024119e79..e58745c1b 100644 --- a/infra/jenkins-cluster/server/Dockerfile +++ b/infra/jenkins-cluster/server/Dockerfile @@ -1,34 +1,17 @@ -# Copyright 2016 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -################################################################################ - -FROM jenkins -MAINTAINER mike.aizatsky@gmail.com +FROM jenkinsci/jenkins USER root -ENV DEBIAN_FRONTEND noninteractive -RUN apt-get update && apt-get upgrade -y +RUN mkdir /var/secrets +RUN apt-get -y update && apt-get -y upgrade +RUN apt-get -y install python-dev virtualenv python-pip build-essential -# should not be newer than container's. -ENV DOCKER_ENGINE_VERSION="1.11.2-0~jessie" +WORKDIR / +RUN wget https://dl.google.com/dl/cloudsdk/release/google-cloud-sdk.zip +RUN unzip google-cloud-sdk.zip -# Install docker -# /var/run/docker.sock will be mounted to a host. -RUN apt-get install -y apt-transport-https ca-certificates -RUN apt-key adv --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys 58118E89F3A912897C070ADBF76221572C52609D -RUN echo "deb https://apt.dockerproject.org/repo debian-jessie main" > /etc/apt/sources.list.d/docker.list -RUN apt-get update -RUN apt-cache policy docker-engine -RUN apt-get install -y docker-engine=$DOCKER_ENGINE_VERSION +RUN /google-cloud-sdk/install.sh --usage-reporting=false --bash-completion=false --disable-installation-options +RUN /google-cloud-sdk/bin/gcloud -q components install alpha beta +RUN /google-cloud-sdk/bin/gcloud -q components update + +RUN chown -R jenkins /google-cloud-sdk +ENV PATH=$PATH:/google-cloud-sdk/bin