diff --git a/README.md b/README.md index 6b2c3c78b..df56e47a2 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,8 @@ community. In cooperation with the [Core Infrastructure Initiative] and the [OpenSSF], OSS-Fuzz aims to make common open source software more secure and stable by combining modern fuzzing techniques with scalable, distributed execution. +Projects that do not qualify for OSS-Fuzz (e.g. closed source) can run their own +instances of [ClusterFuzz] or [ClusterFuzzLite]. [Core Infrastructure Initiative]: https://www.coreinfrastructure.org/ [OpenSSF]: https://www.openssf.org/ @@ -28,6 +30,7 @@ execution environment and reporting tool. [Honggfuzz]: https://github.com/google/honggfuzz [Sanitizers]: https://github.com/google/sanitizers [ClusterFuzz]: https://github.com/google/clusterfuzz +[ClusterFuzzLite]: https://google.github.io/clusterfuzzlite/ Currently, OSS-Fuzz supports C/C++, Rust, Go, Python and Java/JVM code. Other languages supported by [LLVM] may work too. OSS-Fuzz supports fuzzing x86_64 and i386 diff --git a/docs/getting-started/continuous_integration.md b/docs/getting-started/continuous_integration.md index 6ec2632d4..ee9c9f59a 100644 --- a/docs/getting-started/continuous_integration.md +++ b/docs/getting-started/continuous_integration.md @@ -11,7 +11,9 @@ permalink: /getting-started/continuous-integration/ OSS-Fuzz offers **CIFuzz**, a GitHub action/CI job that runs your fuzz targets on pull requests. This works similarly to running unit tests in CI. CIFuzz helps you find and fix bugs before they make it into your codebase. -Currently, CIFuzz only supports projects hosted on GitHub. +Currently, CIFuzz primarily supports projects hosted on GitHub. +Non-OSS-Fuzz users can use CIFuzz with additional features through +[ClusterFuzzLite](https://google.github.io/clusterfuzzlite/). ## How it works diff --git a/docs/index.md b/docs/index.md index 989836cf6..f804e2357 100644 --- a/docs/index.md +++ b/docs/index.md @@ -24,6 +24,8 @@ community. In cooperation with the [Core Infrastructure Initiative] and the [OpenSSF], OSS-Fuzz aims to make common open source software more secure and stable by combining modern fuzzing techniques with scalable, distributed execution. +Projects that do not qualify for OSS-Fuzz (e.g. closed source) can run their own +instances of [ClusterFuzz] or [ClusterFuzzLite]. [Core Infrastructure Initiative]: https://www.coreinfrastructure.org/ [OpenSSF]: https://www.openssf.org/ @@ -37,6 +39,7 @@ execution environment and reporting tool. [Honggfuzz]: https://github.com/google/honggfuzz [Sanitizers]: https://github.com/google/sanitizers [ClusterFuzz]: https://github.com/google/clusterfuzz +[ClusterFuzzLite]: https://google.github.io/clusterfuzzlite/ Currently, OSS-Fuzz supports C/C++, Rust, Go, Python and Java/JVM code. Other languages supported by [LLVM] may work too. OSS-Fuzz supports fuzzing x86_64 diff --git a/infra/cifuzz/clusterfuzz_deployment.py b/infra/cifuzz/clusterfuzz_deployment.py index 74ad48676..e43908c89 100644 --- a/infra/cifuzz/clusterfuzz_deployment.py +++ b/infra/cifuzz/clusterfuzz_deployment.py @@ -20,7 +20,6 @@ import urllib.request import config_utils import continuous_integration -import filestore import filestore_utils import http_utils import get_coverage @@ -159,9 +158,9 @@ class ClusterFuzzLite(BaseClusterFuzzDeployment): try: self.filestore.upload_corpus(name, corpus_dir, replace=replace) logging.info('Done uploading corpus.') - except Exception as error: # pylint: disable=broad-except + except Exception as err: # pylint: disable=broad-except logging.error('Failed to upload corpus for target: %s. Error: %s.', - target_name, error) + target_name, err) def upload_build(self, commit): """Upload the build produced by CIFuzz as the latest build.""" @@ -171,9 +170,9 @@ class ClusterFuzzLite(BaseClusterFuzzDeployment): result = self.filestore.upload_build(build_name, self.workspace.out) logging.info('Done uploading latest build.') return result - except Exception as error: # pylint: disable=broad-except + except Exception as err: # pylint: disable=broad-except logging.error('Failed to upload latest build: %s. Error: %s', - self.workspace.out, error) + self.workspace.out, err) def upload_crashes(self): """Uploads crashes.""" @@ -193,8 +192,8 @@ class ClusterFuzzLite(BaseClusterFuzzDeployment): try: self.filestore.upload_crashes(crash_target, artifact_dir) logging.info('Done uploading crashes.') - except Exception as error: # pylint: disable=broad-except - logging.error('Failed to upload crashes. Error: %s', error) + except Exception as err: # pylint: disable=broad-except + logging.error('Failed to upload crashes. Error: %s', err) def upload_coverage(self): """Uploads the coverage report to the filestore.""" @@ -211,8 +210,8 @@ class ClusterFuzzLite(BaseClusterFuzzDeployment): return None return get_coverage.FilesystemCoverage( repo_path, self.workspace.clusterfuzz_coverage) - except (get_coverage.CoverageError, filestore.FilestoreError): - logging.error('Could not get coverage.') + except Exception as err: # pylint: disable=broad-except + logging.error('Could not get coverage: %s.', err) return None diff --git a/infra/cifuzz/filestore/filesystem/__init__.py b/infra/cifuzz/filestore/filesystem/__init__.py new file mode 100644 index 000000000..7ddea1326 --- /dev/null +++ b/infra/cifuzz/filestore/filesystem/__init__.py @@ -0,0 +1,107 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Filestore implementation using a filesystem directory.""" +import logging +import os +import shutil +import subprocess +import sys + +from distutils import dir_util + +# pylint: disable=wrong-import-position,import-error +sys.path.append( + os.path.join(os.path.dirname(os.path.abspath(__file__)), os.pardir, + os.pardir, os.pardir)) +import filestore + + +def recursive_list_dir(directory): + """Returns list of all files in |directory|, including those in + subdirectories.""" + files = [] + for root, _, filenames in os.walk(directory): + for filename in filenames: + files.append(os.path.join(root, filename)) + return files + + +class FilesystemFilestore(filestore.BaseFilestore): + """Filesystem implementation using a filesystem directory.""" + BUILD_DIR = 'build' + CRASHES_DIR = 'crashes' + CORPUS_DIR = 'corpus' + COVERAGE_DIR = 'coverage' + + def __init__(self, config): + super().__init__(config) + self._filestore_root_dir = self.config.platform_conf.filestore_root_dir + + def _get_filestore_path(self, name, prefix_dir): + """Returns the filesystem path in the filestore for |name| and + |prefix_dir|.""" + return os.path.join(self._filestore_root_dir, prefix_dir, name) + + def _upload_directory(self, name, directory, prefix, delete=False): + filestore_path = self._get_filestore_path(name, prefix) + if os.path.exists(filestore_path): + initial_files = set(recursive_list_dir(filestore_path)) + else: + initial_files = set() + + # Make directory and any parents. + os.makedirs(filestore_path, exist_ok=True) + copied_files = set(dir_util.copy_tree(directory, filestore_path)) + if not delete: + return True + + files_to_delete = initial_files - copied_files + for file_path in files_to_delete: + os.remove(file_path) + return True + + def _download_directory(self, name, dst_directory, prefix): + filestore_path = self._get_filestore_path(name, prefix) + return dir_util.copy_tree(filestore_path, dst_directory) + + def upload_crashes(self, name, directory): + """Uploads the crashes at |directory| to |name|.""" + return self._upload_directory(name, directory, self.CRASHES_DIR) + + def upload_corpus(self, name, directory, replace=False): + """Uploads the crashes at |directory| to |name|.""" + return self._upload_directory(name, + directory, + self.CORPUS_DIR, + delete=replace) + + def upload_build(self, name, directory): + """Uploads the build located at |directory| to |name|.""" + return self._upload_directory(name, directory, self.BUILD_DIR) + + def upload_coverage(self, name, directory): + """Uploads the coverage report at |directory| to |name|.""" + return self._upload_directory(name, directory, self.COVERAGE_DIR) + + def download_corpus(self, name, dst_directory): + """Downloads the corpus located at |name| to |dst_directory|.""" + return self._download_directory(name, dst_directory, self.CORPUS_DIR) + + def download_build(self, name, dst_directory): + """Downloads the build with |name| to |dst_directory|.""" + return self._download_directory(name, dst_directory, self.BUILD_DIR) + + def download_coverage(self, name, dst_directory): + """Downloads the latest project coverage report.""" + return self._download_directory(name, dst_directory, self.COVERAGE_DIR) diff --git a/infra/cifuzz/filestore_utils.py b/infra/cifuzz/filestore_utils.py index e41994d0f..acb018969 100644 --- a/infra/cifuzz/filestore_utils.py +++ b/infra/cifuzz/filestore_utils.py @@ -13,6 +13,7 @@ # limitations under the License. """External filestore interface. Cannot be depended on by filestore code.""" import filestore +import filestore.filesystem import filestore.git import filestore.github_actions import filestore.gsutil @@ -20,9 +21,11 @@ import filestore.no_filestore import filestore.gitlab FILESTORE_MAPPING = { + 'filesystem': filestore.filesystem.FilesystemFilestore, 'gsutil': filestore.gsutil.GSUtilFilestore, 'github-actions': filestore.github_actions.GithubActionsFilestore, 'git': filestore.git.GitFilestore, + # TODO(metzman): Change to "no-filestore" 'no_filestore': filestore.no_filestore.NoFilestore, 'gitlab': filestore.gitlab.GitlabFilestore, } @@ -40,5 +43,6 @@ def get_filestore(config): filestore_cls = FILESTORE_MAPPING.get(config.filestore) if filestore_cls is None: - raise filestore.FilestoreError('Filestore doesn\'t exist.') + raise filestore.FilestoreError( + f'Filestore: {config.filestore} doesn\'t exist.') return filestore_cls(config) diff --git a/infra/cifuzz/platform_config/standalone.py b/infra/cifuzz/platform_config/standalone.py new file mode 100644 index 000000000..1975dfb1b --- /dev/null +++ b/infra/cifuzz/platform_config/standalone.py @@ -0,0 +1,33 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Module for getting the configuration CIFuzz needs to run standalone.""" +import os + +import platform_config + +# pylint: disable=too-few-public-methods + + +class PlatformConfig(platform_config.BasePlatformConfig): + """CI environment for Standalone.""" + + @property + def filestore(self): + """Returns the filestore used to store persistent data.""" + return os.environ.get('FILESTORE', 'filesystem') + + @property + def filestore_root_dir(self): + """Returns the filestore used to store persistent data.""" + return os.environ['FILESTORE_ROOT_DIR']