From 8e9c92798798d11048cd0376642d7d06924248f7 Mon Sep 17 00:00:00 2001 From: Catena cyber <35799796+catenacyber@users.noreply.github.com> Date: Thu, 13 Jan 2022 20:27:53 +0100 Subject: [PATCH] [clusterfuzzlite] Support gitlab (#7073) Related: https://github.com/google/clusterfuzzlite/issues/55 --- infra/cifuzz/docker.py | 4 +- infra/cifuzz/filestore/gitlab/__init__.py | 131 ++++++++++++++++++++++ infra/cifuzz/filestore_utils.py | 2 + infra/cifuzz/platform_config/gitlab.py | 73 ++++++++++++ 4 files changed, 209 insertions(+), 1 deletion(-) create mode 100644 infra/cifuzz/filestore/gitlab/__init__.py create mode 100644 infra/cifuzz/platform_config/gitlab.py diff --git a/infra/cifuzz/docker.py b/infra/cifuzz/docker.py index f957b4bf3..30a08724e 100644 --- a/infra/cifuzz/docker.py +++ b/infra/cifuzz/docker.py @@ -21,6 +21,7 @@ sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) import constants import utils +import environment BASE_BUILDER_TAG = 'gcr.io/oss-fuzz-base/base-builder' PROJECT_TAG_PREFIX = 'gcr.io/oss-fuzz/' @@ -79,7 +80,8 @@ def get_base_docker_run_args(workspace, 'OUT': workspace.out } docker_args += get_docker_env_vars(env_mapping) - docker_container = utils.get_container_name() + docker_container = environment.get('CFL_CONTAINER_ID', + utils.get_container_name()) logging.info('Docker container: %s.', docker_container) if docker_container and not docker_in_docker: # Don't map specific volumes if in a docker container, it breaks when diff --git a/infra/cifuzz/filestore/gitlab/__init__.py b/infra/cifuzz/filestore/gitlab/__init__.py new file mode 100644 index 000000000..299215c0e --- /dev/null +++ b/infra/cifuzz/filestore/gitlab/__init__.py @@ -0,0 +1,131 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""GitLab filestore implementation.""" +import logging + +import json +import os +import shutil +import tempfile + +import filestore +import http_utils + +# pylint: disable=no-self-use,unused-argument + + +class GitlabFilestore(filestore.BaseFilestore): + """Implementation of BaseFilestore using GitLab. + Needs a cache to upload and download builds. + Needs a git repository for corpus and coverage. + """ + + BUILD_PREFIX = 'build-' + CORPUS_PREFIX = 'corpus-' + COVERAGE_PREFIX = 'coverage-' + CRASHES_PREFIX = 'crashes-' + + def __init__(self, config): + super().__init__(config) + self.artifacts_dir = self.config.platform_conf.artifacts_dir + self.cache_dir = self.config.platform_conf.cache_dir + if self.config.git_store_repo: + self.git_filestore = filestore.git.GitFilestore(config, None) + else: + self.git_filestore = None + + def upload_crashes(self, name, directory): + """GitLab artifacts implementation of upload_crashes.""" + # Upload crashes as job artifacts. + if os.listdir(directory): + dest_dir_artifacts = os.path.join(self.config.project_src_path, + self.artifacts_dir, + self.CRASHES_PREFIX + name) + logging.info('Uploading artifacts to %s.', dest_dir_artifacts) + shutil.copytree(directory, dest_dir_artifacts) + + def upload_corpus(self, name, directory, replace=False): + """GitLab artifacts implementation of upload_corpus.""" + # Use the git filestore if any. + if self.git_filestore: + self.git_filestore.upload_corpus(name, directory, replace) + return + # Fall back to cache. + dest_dir_cache = os.path.join(self.config.project_src_path, self.cache_dir, + self.CORPUS_PREFIX + name) + logging.info('Copying from %s to cache %s.', directory, dest_dir_cache) + shutil.copytree(directory, dest_dir_cache, dirs_exist_ok=True) + + def upload_build(self, name, directory): + """GitLab artifacts implementation of upload_build.""" + # Puts build into the cache. + dest_dir_cache = os.path.join(self.config.project_src_path, self.cache_dir, + self.BUILD_PREFIX + name) + logging.info('Copying from %s to cache %s.', directory, dest_dir_cache) + shutil.copytree(directory, dest_dir_cache, dirs_exist_ok=True) + + def upload_coverage(self, name, directory): + """GitLab artifacts implementation of upload_coverage.""" + # Use the git filestore. + if self.git_filestore: + self.git_filestore.upload_coverage(name, directory) + return + # Fall back to cache. + dest_dir_cache = os.path.join(self.config.project_src_path, self.cache_dir, + self.COVERAGE_PREFIX + name) + logging.info('Copying from %s to cache %s.', directory, dest_dir_cache) + shutil.copytree(directory, dest_dir_cache, dirs_exist_ok=True) + # And also updates coverage reports as artifacts + # as it should not be too big. + dest_dir_artifacts = os.path.join(self.config.project_src_path, + self.artifacts_dir, + self.COVERAGE_PREFIX + name) + logging.info('Uploading artifacts to %s.', dest_dir_artifacts) + shutil.copytree(directory, dest_dir_artifacts) + + def _copy_from_cache(self, src_dir_cache, dst_directory): + if not os.path.exists(src_dir_cache): + logging.info('Cache %s does not exist.', src_dir_cache) + return False + logging.info('Copying %s from cache to %s.', src_dir_cache, dst_directory) + shutil.copytree(src_dir_cache, dst_directory, dirs_exist_ok=True) + return True + + def download_corpus(self, name, dst_directory): + """GitLab artifacts implementation of download_corpus.""" + # Use the git filestore if any. + if self.git_filestore: + self.git_filestore.download_corpus(name, dst_directory) + return + # Fall back to cache. + src_dir_cache = os.path.join(self.config.project_src_path, self.cache_dir, + self.CORPUS_PREFIX + name) + self._copy_from_cache(src_dir_cache, dst_directory) + + def download_build(self, name, dst_directory): + """GitLab artifacts implementation of download_build.""" + # Gets build from the cache. + src_dir_cache = os.path.join(self.config.project_src_path, self.cache_dir, + self.BUILD_PREFIX + name) + return self._copy_from_cache(src_dir_cache, dst_directory) + + def download_coverage(self, name, dst_directory): + """GitLab artifacts implementation of download_coverage.""" + # Use the git filestore if any. + if self.git_filestore: + return self.git_filestore.download_coverage(name, dst_directory) + # Fall back to cache. + src_dir_cache = os.path.join(self.config.project_src_path, self.cache_dir, + self.COVERAGE_PREFIX + name) + return self._copy_from_cache(src_dir_cache, dst_directory) diff --git a/infra/cifuzz/filestore_utils.py b/infra/cifuzz/filestore_utils.py index 5e083c006..e41994d0f 100644 --- a/infra/cifuzz/filestore_utils.py +++ b/infra/cifuzz/filestore_utils.py @@ -17,12 +17,14 @@ import filestore.git import filestore.github_actions import filestore.gsutil import filestore.no_filestore +import filestore.gitlab FILESTORE_MAPPING = { 'gsutil': filestore.gsutil.GSUtilFilestore, 'github-actions': filestore.github_actions.GithubActionsFilestore, 'git': filestore.git.GitFilestore, 'no_filestore': filestore.no_filestore.NoFilestore, + 'gitlab': filestore.gitlab.GitlabFilestore, } diff --git a/infra/cifuzz/platform_config/gitlab.py b/infra/cifuzz/platform_config/gitlab.py new file mode 100644 index 000000000..7392fc784 --- /dev/null +++ b/infra/cifuzz/platform_config/gitlab.py @@ -0,0 +1,73 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Module for getting the configuration CIFuzz needs to run on GitLab.""" +import logging +import os + +import environment +import platform_config + + +class PlatformConfig(platform_config.BasePlatformConfig): + """CI environment for GitLab.""" + + @property + def workspace(self): + """Returns the workspace.""" + return os.path.join(os.getenv('CI_BUILDS_DIR'), os.getenv('CI_JOB_ID')) + + @property + def git_sha(self): + """Returns the Git SHA to checkout and fuzz.""" + return os.getenv('CI_COMMIT_SHA') + + @property + def project_src_path(self): + """Returns the directory with the source of the project""" + return os.getenv('CI_PROJECT_DIR') + + @property + def token(self): + """Returns the job token""" + return os.getenv('CI_JOB_TOKEN') + + @property + def project_repo_name(self): + """Returns the project's name""" + return os.getenv('CI_PROJECT_NAME') + + @property + def base_commit(self): + """Returns the previous commit sha for commit-fuzzing""" + base_commit = None + if os.getenv('CI_PIPELINE_SOURCE') == 'push': + base_commit = os.getenv('CI_COMMIT_BEFORE_SHA') + logging.debug('base_commit: %s.', base_commit) + return base_commit + + @property + def base_ref(self): + """Returns the base commit sha for a merge request""" + # Could also be CI_MERGE_REQUEST_TARGET_BRANCH_NAME. + return os.getenv('CI_MERGE_REQUEST_DIFF_BASE_SHA') + + @property + def artifacts_dir(self): + """Gitlab: returns the directory to put artifacts""" + return environment.get('CFL_ARTIFACTS_DIR', 'artifacts') + + @property + def cache_dir(self): + """Gitlab: returns the directory to use as cache""" + return environment.get('CFL_CACHE_DIR', 'cache')