[ClusterFuzzLite] Support local runs (#6987)

README.md

@@ -15,6 +15,8 @@ community.
 In cooperation with the [Core Infrastructure Initiative] and the [OpenSSF],
 OSS-Fuzz aims to make common open source software more secure and stable by
 combining modern fuzzing techniques with scalable, distributed execution.
+Projects that do not qualify for OSS-Fuzz (e.g. closed source) can run their own
+instances of [ClusterFuzz] or [ClusterFuzzLite].
 
 [Core Infrastructure Initiative]: https://www.coreinfrastructure.org/
 [OpenSSF]: https://www.openssf.org/
@@ -28,6 +30,7 @@ execution environment and reporting tool.
 [Honggfuzz]: https://github.com/google/honggfuzz
 [Sanitizers]: https://github.com/google/sanitizers
 [ClusterFuzz]: https://github.com/google/clusterfuzz
+[ClusterFuzzLite]: https://google.github.io/clusterfuzzlite/
 
 Currently, OSS-Fuzz supports C/C++, Rust, Go, Python and Java/JVM code. Other languages
 supported by [LLVM] may work too. OSS-Fuzz supports fuzzing x86_64 and i386

docs/getting-started/continuous_integration.md

@@ -11,7 +11,9 @@ permalink: /getting-started/continuous-integration/
 OSS-Fuzz offers **CIFuzz**, a GitHub action/CI job that runs your fuzz targets
 on pull requests. This works similarly to running unit tests in CI. CIFuzz helps
 you find and fix bugs before they make it into your codebase.
-Currently, CIFuzz only supports projects hosted on GitHub.
+Currently, CIFuzz primarily supports projects hosted on GitHub.
+Non-OSS-Fuzz users can use CIFuzz with additional features through
+[ClusterFuzzLite](https://google.github.io/clusterfuzzlite/).
 
 ## How it works
 

docs/index.md

@@ -24,6 +24,8 @@ community.
 In cooperation with the [Core Infrastructure Initiative] and the [OpenSSF],
 OSS-Fuzz aims to make common open source software more secure and stable by
 combining modern fuzzing techniques with scalable, distributed execution.
+Projects that do not qualify for OSS-Fuzz (e.g. closed source) can run their own
+instances of [ClusterFuzz] or [ClusterFuzzLite].
 
 [Core Infrastructure Initiative]: https://www.coreinfrastructure.org/
 [OpenSSF]: https://www.openssf.org/
@@ -37,6 +39,7 @@ execution environment and reporting tool.
 [Honggfuzz]: https://github.com/google/honggfuzz
 [Sanitizers]: https://github.com/google/sanitizers
 [ClusterFuzz]: https://github.com/google/clusterfuzz
+[ClusterFuzzLite]: https://google.github.io/clusterfuzzlite/
 
 Currently, OSS-Fuzz supports C/C++, Rust, Go, Python and Java/JVM code. Other
 languages supported by [LLVM] may work too. OSS-Fuzz supports fuzzing x86_64

infra/cifuzz/clusterfuzz_deployment.py

@@ -20,7 +20,6 @@ import urllib.request
 
 import config_utils
 import continuous_integration
-import filestore
 import filestore_utils
 import http_utils
 import get_coverage
@@ -159,9 +158,9 @@ class ClusterFuzzLite(BaseClusterFuzzDeployment):
     try:
       self.filestore.upload_corpus(name, corpus_dir, replace=replace)
       logging.info('Done uploading corpus.')
-    except Exception as error:  # pylint: disable=broad-except
+    except Exception as err:  # pylint: disable=broad-except
       logging.error('Failed to upload corpus for target: %s. Error: %s.',
-                    target_name, error)
+                    target_name, err)
 
   def upload_build(self, commit):
     """Upload the build produced by CIFuzz as the latest build."""
@@ -171,9 +170,9 @@ class ClusterFuzzLite(BaseClusterFuzzDeployment):
       result = self.filestore.upload_build(build_name, self.workspace.out)
       logging.info('Done uploading latest build.')
       return result
-    except Exception as error:  # pylint: disable=broad-except
+    except Exception as err:  # pylint: disable=broad-except
       logging.error('Failed to upload latest build: %s. Error: %s',
-                    self.workspace.out, error)
+                    self.workspace.out, err)
 
   def upload_crashes(self):
     """Uploads crashes."""
@@ -193,8 +192,8 @@ class ClusterFuzzLite(BaseClusterFuzzDeployment):
       try:
         self.filestore.upload_crashes(crash_target, artifact_dir)
         logging.info('Done uploading crashes.')
-      except Exception as error:  # pylint: disable=broad-except
-        logging.error('Failed to upload crashes. Error: %s', error)
+      except Exception as err:  # pylint: disable=broad-except
+        logging.error('Failed to upload crashes. Error: %s', err)
 
   def upload_coverage(self):
     """Uploads the coverage report to the filestore."""
@@ -211,8 +210,8 @@ class ClusterFuzzLite(BaseClusterFuzzDeployment):
         return None
       return get_coverage.FilesystemCoverage(
           repo_path, self.workspace.clusterfuzz_coverage)
-    except (get_coverage.CoverageError, filestore.FilestoreError):
-      logging.error('Could not get coverage.')
+    except Exception as err:  # pylint: disable=broad-except
+      logging.error('Could not get coverage: %s.', err)
       return None
 
 

infra/cifuzz/filestore/filesystem/__init__.py

@@ -0,0 +1,107 @@
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Filestore implementation using a filesystem directory."""
+import logging
+import os
+import shutil
+import subprocess
+import sys
+
+from distutils import dir_util
+
+# pylint: disable=wrong-import-position,import-error
+sys.path.append(
+    os.path.join(os.path.dirname(os.path.abspath(__file__)), os.pardir,
+                 os.pardir, os.pardir))
+import filestore
+
+
+def recursive_list_dir(directory):
+  """Returns list of all files in |directory|, including those in
+  subdirectories."""
+  files = []
+  for root, _, filenames in os.walk(directory):
+    for filename in filenames:
+      files.append(os.path.join(root, filename))
+  return files
+
+
+class FilesystemFilestore(filestore.BaseFilestore):
+  """Filesystem implementation using a filesystem directory."""
+  BUILD_DIR = 'build'
+  CRASHES_DIR = 'crashes'
+  CORPUS_DIR = 'corpus'
+  COVERAGE_DIR = 'coverage'
+
+  def __init__(self, config):
+    super().__init__(config)
+    self._filestore_root_dir = self.config.platform_conf.filestore_root_dir
+
+  def _get_filestore_path(self, name, prefix_dir):
+    """Returns the filesystem path in the filestore for |name| and
+    |prefix_dir|."""
+    return os.path.join(self._filestore_root_dir, prefix_dir, name)
+
+  def _upload_directory(self, name, directory, prefix, delete=False):
+    filestore_path = self._get_filestore_path(name, prefix)
+    if os.path.exists(filestore_path):
+      initial_files = set(recursive_list_dir(filestore_path))
+    else:
+      initial_files = set()
+
+    # Make directory and any parents.
+    os.makedirs(filestore_path, exist_ok=True)
+    copied_files = set(dir_util.copy_tree(directory, filestore_path))
+    if not delete:
+      return True
+
+    files_to_delete = initial_files - copied_files
+    for file_path in files_to_delete:
+      os.remove(file_path)
+    return True
+
+  def _download_directory(self, name, dst_directory, prefix):
+    filestore_path = self._get_filestore_path(name, prefix)
+    return dir_util.copy_tree(filestore_path, dst_directory)
+
+  def upload_crashes(self, name, directory):
+    """Uploads the crashes at |directory| to |name|."""
+    return self._upload_directory(name, directory, self.CRASHES_DIR)
+
+  def upload_corpus(self, name, directory, replace=False):
+    """Uploads the crashes at |directory| to |name|."""
+    return self._upload_directory(name,
+                                  directory,
+                                  self.CORPUS_DIR,
+                                  delete=replace)
+
+  def upload_build(self, name, directory):
+    """Uploads the build located at |directory| to |name|."""
+    return self._upload_directory(name, directory, self.BUILD_DIR)
+
+  def upload_coverage(self, name, directory):
+    """Uploads the coverage report at |directory| to |name|."""
+    return self._upload_directory(name, directory, self.COVERAGE_DIR)
+
+  def download_corpus(self, name, dst_directory):
+    """Downloads the corpus located at |name| to |dst_directory|."""
+    return self._download_directory(name, dst_directory, self.CORPUS_DIR)
+
+  def download_build(self, name, dst_directory):
+    """Downloads the build with |name| to |dst_directory|."""
+    return self._download_directory(name, dst_directory, self.BUILD_DIR)
+
+  def download_coverage(self, name, dst_directory):
+    """Downloads the latest project coverage report."""
+    return self._download_directory(name, dst_directory, self.COVERAGE_DIR)

infra/cifuzz/filestore_utils.py

@@ -13,6 +13,7 @@
 # limitations under the License.
 """External filestore interface. Cannot be depended on by filestore code."""
 import filestore
+import filestore.filesystem
 import filestore.git
 import filestore.github_actions
 import filestore.gsutil
@@ -20,9 +21,11 @@ import filestore.no_filestore
 import filestore.gitlab
 
 FILESTORE_MAPPING = {
+    'filesystem': filestore.filesystem.FilesystemFilestore,
     'gsutil': filestore.gsutil.GSUtilFilestore,
     'github-actions': filestore.github_actions.GithubActionsFilestore,
     'git': filestore.git.GitFilestore,
+    # TODO(metzman): Change to "no-filestore"
     'no_filestore': filestore.no_filestore.NoFilestore,
     'gitlab': filestore.gitlab.GitlabFilestore,
 }
@@ -40,5 +43,6 @@ def get_filestore(config):
 
   filestore_cls = FILESTORE_MAPPING.get(config.filestore)
   if filestore_cls is None:
-    raise filestore.FilestoreError('Filestore doesn\'t exist.')
+    raise filestore.FilestoreError(
+        f'Filestore: {config.filestore} doesn\'t exist.')
   return filestore_cls(config)

infra/cifuzz/platform_config/standalone.py

@@ -0,0 +1,33 @@
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Module for getting the configuration CIFuzz needs to run standalone."""
+import os
+
+import platform_config
+
+# pylint: disable=too-few-public-methods
+
+
+class PlatformConfig(platform_config.BasePlatformConfig):
+  """CI environment for Standalone."""
+
+  @property
+  def filestore(self):
+    """Returns the filestore used to store persistent data."""
+    return os.environ.get('FILESTORE', 'filesystem')
+
+  @property
+  def filestore_root_dir(self):
+    """Returns the filestore used to store persistent data."""
+    return os.environ['FILESTORE_ROOT_DIR']