[clusterfuzzlite] Support gitlab (#7073)

Related: https://github.com/google/clusterfuzzlite/issues/55

infra/cifuzz/docker.py

@@ -21,6 +21,7 @@ sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
 import constants
 import utils
+import environment
 
 BASE_BUILDER_TAG = 'gcr.io/oss-fuzz-base/base-builder'
 PROJECT_TAG_PREFIX = 'gcr.io/oss-fuzz/'
@@ -79,7 +80,8 @@ def get_base_docker_run_args(workspace,
       'OUT': workspace.out
   }
   docker_args += get_docker_env_vars(env_mapping)
-  docker_container = utils.get_container_name()
+  docker_container = environment.get('CFL_CONTAINER_ID',
+                                     utils.get_container_name())
   logging.info('Docker container: %s.', docker_container)
   if docker_container and not docker_in_docker:
     # Don't map specific volumes if in a docker container, it breaks when

infra/cifuzz/filestore/gitlab/__init__.py

@@ -0,0 +1,131 @@
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""GitLab filestore implementation."""
+import logging
+
+import json
+import os
+import shutil
+import tempfile
+
+import filestore
+import http_utils
+
+# pylint: disable=no-self-use,unused-argument
+
+
+class GitlabFilestore(filestore.BaseFilestore):
+  """Implementation of BaseFilestore using GitLab.
+  Needs a cache to upload and download builds.
+  Needs a git repository for corpus and coverage.
+  """
+
+  BUILD_PREFIX = 'build-'
+  CORPUS_PREFIX = 'corpus-'
+  COVERAGE_PREFIX = 'coverage-'
+  CRASHES_PREFIX = 'crashes-'
+
+  def __init__(self, config):
+    super().__init__(config)
+    self.artifacts_dir = self.config.platform_conf.artifacts_dir
+    self.cache_dir = self.config.platform_conf.cache_dir
+    if self.config.git_store_repo:
+      self.git_filestore = filestore.git.GitFilestore(config, None)
+    else:
+      self.git_filestore = None
+
+  def upload_crashes(self, name, directory):
+    """GitLab artifacts implementation of upload_crashes."""
+    # Upload crashes as job artifacts.
+    if os.listdir(directory):
+      dest_dir_artifacts = os.path.join(self.config.project_src_path,
+                                        self.artifacts_dir,
+                                        self.CRASHES_PREFIX + name)
+      logging.info('Uploading artifacts to %s.', dest_dir_artifacts)
+      shutil.copytree(directory, dest_dir_artifacts)
+
+  def upload_corpus(self, name, directory, replace=False):
+    """GitLab artifacts implementation of upload_corpus."""
+    # Use the git filestore if any.
+    if self.git_filestore:
+      self.git_filestore.upload_corpus(name, directory, replace)
+      return
+    # Fall back to cache.
+    dest_dir_cache = os.path.join(self.config.project_src_path, self.cache_dir,
+                                  self.CORPUS_PREFIX + name)
+    logging.info('Copying from %s to cache %s.', directory, dest_dir_cache)
+    shutil.copytree(directory, dest_dir_cache, dirs_exist_ok=True)
+
+  def upload_build(self, name, directory):
+    """GitLab artifacts implementation of upload_build."""
+    # Puts build into the cache.
+    dest_dir_cache = os.path.join(self.config.project_src_path, self.cache_dir,
+                                  self.BUILD_PREFIX + name)
+    logging.info('Copying from %s to cache %s.', directory, dest_dir_cache)
+    shutil.copytree(directory, dest_dir_cache, dirs_exist_ok=True)
+
+  def upload_coverage(self, name, directory):
+    """GitLab artifacts implementation of upload_coverage."""
+    # Use the git filestore.
+    if self.git_filestore:
+      self.git_filestore.upload_coverage(name, directory)
+      return
+    # Fall back to cache.
+    dest_dir_cache = os.path.join(self.config.project_src_path, self.cache_dir,
+                                  self.COVERAGE_PREFIX + name)
+    logging.info('Copying from %s to cache %s.', directory, dest_dir_cache)
+    shutil.copytree(directory, dest_dir_cache, dirs_exist_ok=True)
+    # And also updates coverage reports as artifacts
+    # as it should not be too big.
+    dest_dir_artifacts = os.path.join(self.config.project_src_path,
+                                      self.artifacts_dir,
+                                      self.COVERAGE_PREFIX + name)
+    logging.info('Uploading artifacts to %s.', dest_dir_artifacts)
+    shutil.copytree(directory, dest_dir_artifacts)
+
+  def _copy_from_cache(self, src_dir_cache, dst_directory):
+    if not os.path.exists(src_dir_cache):
+      logging.info('Cache %s does not exist.', src_dir_cache)
+      return False
+    logging.info('Copying %s from cache to %s.', src_dir_cache, dst_directory)
+    shutil.copytree(src_dir_cache, dst_directory, dirs_exist_ok=True)
+    return True
+
+  def download_corpus(self, name, dst_directory):
+    """GitLab artifacts implementation of download_corpus."""
+    # Use the git filestore if any.
+    if self.git_filestore:
+      self.git_filestore.download_corpus(name, dst_directory)
+      return
+    # Fall back to cache.
+    src_dir_cache = os.path.join(self.config.project_src_path, self.cache_dir,
+                                 self.CORPUS_PREFIX + name)
+    self._copy_from_cache(src_dir_cache, dst_directory)
+
+  def download_build(self, name, dst_directory):
+    """GitLab artifacts implementation of download_build."""
+    # Gets build from the cache.
+    src_dir_cache = os.path.join(self.config.project_src_path, self.cache_dir,
+                                 self.BUILD_PREFIX + name)
+    return self._copy_from_cache(src_dir_cache, dst_directory)
+
+  def download_coverage(self, name, dst_directory):
+    """GitLab artifacts implementation of download_coverage."""
+    # Use the git filestore if any.
+    if self.git_filestore:
+      return self.git_filestore.download_coverage(name, dst_directory)
+    # Fall back to cache.
+    src_dir_cache = os.path.join(self.config.project_src_path, self.cache_dir,
+                                 self.COVERAGE_PREFIX + name)
+    return self._copy_from_cache(src_dir_cache, dst_directory)

infra/cifuzz/filestore_utils.py

@@ -17,12 +17,14 @@ import filestore.git
 import filestore.github_actions
 import filestore.gsutil
 import filestore.no_filestore
+import filestore.gitlab
 
 FILESTORE_MAPPING = {
     'gsutil': filestore.gsutil.GSUtilFilestore,
     'github-actions': filestore.github_actions.GithubActionsFilestore,
     'git': filestore.git.GitFilestore,
     'no_filestore': filestore.no_filestore.NoFilestore,
+    'gitlab': filestore.gitlab.GitlabFilestore,
 }
 
 

infra/cifuzz/platform_config/gitlab.py

@@ -0,0 +1,73 @@
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Module for getting the configuration CIFuzz needs to run on GitLab."""
+import logging
+import os
+
+import environment
+import platform_config
+
+
+class PlatformConfig(platform_config.BasePlatformConfig):
+  """CI environment for GitLab."""
+
+  @property
+  def workspace(self):
+    """Returns the workspace."""
+    return os.path.join(os.getenv('CI_BUILDS_DIR'), os.getenv('CI_JOB_ID'))
+
+  @property
+  def git_sha(self):
+    """Returns the Git SHA to checkout and fuzz."""
+    return os.getenv('CI_COMMIT_SHA')
+
+  @property
+  def project_src_path(self):
+    """Returns the directory with the source of the project"""
+    return os.getenv('CI_PROJECT_DIR')
+
+  @property
+  def token(self):
+    """Returns the job token"""
+    return os.getenv('CI_JOB_TOKEN')
+
+  @property
+  def project_repo_name(self):
+    """Returns the project's name"""
+    return os.getenv('CI_PROJECT_NAME')
+
+  @property
+  def base_commit(self):
+    """Returns the previous commit sha for commit-fuzzing"""
+    base_commit = None
+    if os.getenv('CI_PIPELINE_SOURCE') == 'push':
+      base_commit = os.getenv('CI_COMMIT_BEFORE_SHA')
+    logging.debug('base_commit: %s.', base_commit)
+    return base_commit
+
+  @property
+  def base_ref(self):
+    """Returns the base commit sha for a merge request"""
+    # Could also be CI_MERGE_REQUEST_TARGET_BRANCH_NAME.
+    return os.getenv('CI_MERGE_REQUEST_DIFF_BASE_SHA')
+
+  @property
+  def artifacts_dir(self):
+    """Gitlab: returns the directory to put artifacts"""
+    return environment.get('CFL_ARTIFACTS_DIR', 'artifacts')
+
+  @property
+  def cache_dir(self):
+    """Gitlab: returns the directory to use as cache"""
+    return environment.get('CFL_CACHE_DIR', 'cache')