From 14582175d07c8a66adf3f9087a464612f2a3f630 Mon Sep 17 00:00:00 2001 From: Leo Neat Date: Wed, 15 Jan 2020 13:30:57 -0800 Subject: [PATCH] [infra] Scripts for building fuzzers with CIFuzz (#3207) --- infra/base-images/base-builder/detect_repo.py | 81 ++++++++---- .../base-builder/detect_repo_test.py | 64 ++++++++-- infra/bisector.py | 116 ++++++++---------- infra/bisector_test.py | 49 ++++++-- infra/build_specified_commit.py | 106 +++++++++------- infra/build_specified_commit_test.py | 74 ++++++++--- infra/cifuzz.py | 88 +++++++++++++ infra/helper.py | 22 ++-- 8 files changed, 409 insertions(+), 191 deletions(-) create mode 100644 infra/cifuzz.py diff --git a/infra/base-images/base-builder/detect_repo.py b/infra/base-images/base-builder/detect_repo.py index d7974ca5e..f57947e7e 100644 --- a/infra/base-images/base-builder/detect_repo.py +++ b/infra/base-images/base-builder/detect_repo.py @@ -16,9 +16,11 @@ inside of an OSS-Fuzz project. Example Usage: - python detect_repo.py --src_dir /src --example_commit b534f03eecd8a109db2b085ab24d419b6486de97 + python detect_repo.py --src_dir /src --example_commit + b534f03eecd8a109db2b085ab24d419b6486de97 -Prints the location of the git remote repo as well as the repos name seperated by a space. +Prints the location of the git remote repo as well as the repo's name +seperated by a space. https://github.com/VirusTotal/yara.git yara @@ -29,26 +31,36 @@ import subprocess def main(): - """Function to get a git repos information based on its commit.""" + """Function to get a git repo's url and name referenced by OSS-Fuzz + Dockerfile. + + Raises: + ValueError when a commit or a ref is not provided. + """ parser = argparse.ArgumentParser( - description='Finds a specific git repo in an oss-fuzz projects docker file.' - ) + description= + 'Finds a specific git repo in an oss-fuzz project\'s docker file.') parser.add_argument( '--src_dir', - help='The location of the oss-fuzz projects source directory', - required=True) - parser.add_argument( - '--example_commit', - help='A commit SHA refrencing the projects main repo', + help='The location of an oss-fuzz project\'s source directory.', required=True) + parser.add_argument('--repo_name', help='The name of the git repo.') + parser.add_argument('--example_commit', + help='A commit SHA referencing the project\'s main repo.') + args = parser.parse_args() + if not args.repo_name and not args.example_commit: + raise ValueError( + 'Requires an example commit or a repo name to find repo location.') for single_dir in os.listdir(args.src_dir): full_path = os.path.join(args.src_dir, single_dir) if not os.path.isdir(full_path): continue - if check_for_commit( - os.path.join(args.src_dir, full_path), args.example_commit): - print('Detected repo: %s %s' % (get_repo(full_path), single_dir.rstrip())) + if args.example_commit and check_for_commit(full_path, args.example_commit): + print('Detected repo:', get_repo(full_path), single_dir) + return + if args.repo_name and check_for_repo_name(full_path, args.repo_name): + print('Detected repo:', get_repo(full_path), single_dir) return print('No git repos with specific commit: %s found in %s' % (args.example_commit, args.src_dir)) @@ -61,26 +73,41 @@ def get_repo(repo_path): repo_path: The directory on the image where the git repo exists. Returns: - The repo location or None + The repo location or None. """ - output, return_code = execute( - ['git', 'config', '--get', 'remote.origin.url'], - location=repo_path, - check_result=True) + output, return_code = execute(['git', 'config', '--get', 'remote.origin.url'], + location=repo_path, + check_result=True) if return_code == 0 and output: return output.rstrip() return None +def check_for_repo_name(repo_path, repo_name): + """Check to see if the repo_name matches the remote repository repo name. + + Args: + repo_path: The directory of the git repo. + repo_name: The name of the target git repo. + """ + if not os.path.exists(os.path.join(repo_path, '.git')): + return False + + out, _ = execute(['git', 'config', '--get', 'remote.origin.url'], + location=repo_path) + out = out.split('/')[-1].replace('.git', '').rstrip() + return out == repo_name + + def check_for_commit(repo_path, commit): """Checks a directory for a specific commit. Args: - repo_path: The name of the directory to test for the commit - commit: The commit SHA to check for + repo_path: The name of the directory to test for the commit. + commit: The commit SHA to check for. Returns: - True if directory contains that commit + True if directory contains that commit. """ # Check if valid git repo. @@ -93,7 +120,7 @@ def check_for_commit(repo_path, commit): # Check if commit is in history. _, return_code = execute(['git', 'cat-file', '-e', commit], - location=repo_path) + location=repo_path) return return_code == 0 @@ -101,15 +128,15 @@ def execute(command, location, check_result=False): """Runs a shell command in the specified directory location. Args: - command: The command as a list to be run - location: The directory the command is run in - check_result: Should an exception be thrown on failed command + command: The command as a list to be run. + location: The directory the command is run in. + check_result: Should an exception be thrown on failed command. Returns: - The stdout of the command, the error code + The stdout of the command, the error code. Raises: - RuntimeError: running a command resulted in an error + RuntimeError: running a command resulted in an error. """ process = subprocess.Popen(command, stdout=subprocess.PIPE, cwd=location) output, err = process.communicate() diff --git a/infra/base-images/base-builder/detect_repo_test.py b/infra/base-images/base-builder/detect_repo_test.py index cac0f8821..e9029b7ff 100644 --- a/infra/base-images/base-builder/detect_repo_test.py +++ b/infra/base-images/base-builder/detect_repo_test.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. """Test the functionality of the detect_repo module. -The will consist of the following functional test +This will consist of the following functional test: 1. Determine if a OSS-Fuzz projects main repo can be accurately deduce from example commits. """ @@ -23,11 +23,14 @@ import tempfile import unittest import detect_repo + # Appending to path for access to repo_manager module. +# pylint: disable=wrong-import-position sys.path.append( - os.path.dirname( - os.path.dirname(os.path.dirname(os.path.realpath(__file__))))) + os.path.dirname(os.path.dirname(os.path.dirname( + os.path.abspath(__file__))))) import repo_manager +# pylint: enable=wrong-import-position class DetectRepoTest(unittest.TestCase): @@ -62,25 +65,66 @@ class DetectRepoTest(unittest.TestCase): self.check_commit_with_repo(None, None, 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', tmp_dir) - def check_commit_with_repo(self, repo_origin, repo_name, commit, tmp_dir): + def test_infer_main_repo_from_name(self): + """Tests that the main project repo can be inferred from a repo name.""" + + with tempfile.TemporaryDirectory() as tmp_dir: + # Construct example repos to check for name. + repo_manager.RepoManager('https://github.com/curl/curl.git', tmp_dir) + repo_manager.RepoManager('https://github.com/ntop/nDPI.git', tmp_dir) + repo_manager.RepoManager('https://github.com/libarchive/libarchive.git', + tmp_dir) + self.check_ref_with_repo('https://github.com/curl/curl.git', 'curl', + tmp_dir) + self.check_ref_with_repo('https://github.com/ntop/nDPI.git', 'nDPI', + tmp_dir) + self.check_ref_with_repo('https://github.com/libarchive/libarchive.git', + 'libarchive', tmp_dir) + + def check_ref_with_repo(self, repo_origin, repo_name, tmp_dir): """Checks the detect repo's main method for a specific set of inputs. + Args: + repo_origin: URL of the git repo. + repo_name: The name of the directory it is cloned to. + tmp_dir: The location of the directory of git repos to be searched. + """ + command = [ + 'python3', 'detect_repo.py', '--src_dir', tmp_dir, '--repo_name', + repo_name + ] + out, _ = detect_repo.execute(command, + location=os.path.dirname( + os.path.realpath(__file__))) + match = re.search(r'\bDetected repo: ([^ ]+) ([^ ]+)', out.rstrip()) + if match and match.group(1) and match.group(2): + self.assertEqual(match.group(1), repo_origin) + else: + self.assertIsNone(repo_origin) + + def check_commit_with_repo(self, repo_origin, repo_name, commit, tmp_dir): + """Checks the detect repos main method for a specific set of inputs. + Args: - repo_origin: The location of where the git repo is stored - repo_name: The name of the directory it is cloned to - commit: The commit that should be used to look up the repo - tmp_dir: The location of the directory of git repos to be searched + repo_origin: URL of the git repo. + repo_name: The name of the directory it is cloned to. + commit: The commit that should be used to look up the repo. + tmp_dir: The location of the directory of git repos to be searched. """ command = [ 'python3', 'detect_repo.py', '--src_dir', tmp_dir, '--example_commit', commit ] - out, _ = detect_repo.execute( - command, location=os.path.dirname(os.path.realpath(__file__))) + out, _ = detect_repo.execute(command, + location=os.path.dirname( + os.path.abspath(__file__))) match = re.search(r'\bDetected repo: ([^ ]+) ([^ ]+)', out.rstrip()) if match and match.group(1) and match.group(2): self.assertEqual(match.group(1), repo_origin) self.assertEqual(match.group(2), repo_name) + else: + self.assertIsNone(repo_origin) + self.assertIsNone(repo_name) if __name__ == '__main__': diff --git a/infra/bisector.py b/infra/bisector.py index afb5a4cef..557b92beb 100644 --- a/infra/bisector.py +++ b/infra/bisector.py @@ -18,7 +18,7 @@ where the bug was introduced. It also looks for where the bug was fixed. This is done with the following steps: - NOTE: NEEDS TO BE RUN FROM THE OSS-Fuzz HOME directory + NOTE: Needs to be run from root of the OSS-Fuzz source checkout. Typical usage example: python3 infra/bisector.py @@ -31,7 +31,6 @@ This is done with the following steps: """ import argparse -from dataclasses import dataclass import os import tempfile @@ -40,22 +39,6 @@ import helper import repo_manager -@dataclass -class BuildData(): - """List of data requried for bisection of errors in OSS-Fuzz projects. - - Attributes: - project_name: The name of the OSS-Fuzz project that is being checked - engine: The fuzzing engine to be used - sanitizer: The sanitizer to be used - architecture: CPU architecture to build the fuzzer for - """ - project_name: str - engine: str - sanitizer: str - architecture: str - - def main(): """Finds the commit SHA where an error was initally introduced.""" oss_fuzz_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) @@ -65,29 +48,34 @@ def main(): parser = argparse.ArgumentParser( description='git bisection for finding introduction of bugs') - parser.add_argument( - '--project_name', - help='The name of the project where the bug occured', - required=True) - parser.add_argument( - '--commit_new', - help='The newest commit SHA to be bisected', - required=True) - parser.add_argument( - '--commit_old', - help='The oldest commit SHA to be bisected', - required=True) - parser.add_argument( - '--fuzz_target', help='the name of the fuzzer to be built', required=True) - parser.add_argument('--testcase', help='path to test case', required=True) - parser.add_argument( - '--engine', help='the default is "libfuzzer"', default='libfuzzer') - parser.add_argument( - '--sanitizer', default='address', help='the default is "address"') + parser.add_argument('--project_name', + help='The name of the project where the bug occurred.', + required=True) + parser.add_argument('--commit_new', + help='The newest commit SHA to be bisected.', + required=True) + parser.add_argument('--commit_old', + help='The oldest commit SHA to be bisected.', + required=True) + parser.add_argument('--fuzz_target', + help='The name of the fuzzer to be built.', + required=True) + parser.add_argument('--testcase', + help='The path to test case.', + required=True) + parser.add_argument('--engine', + help='The default is "libfuzzer".', + default='libfuzzer') + parser.add_argument('--sanitizer', + default='address', + help='The default is "address".') parser.add_argument('--architecture', default='x86_64') args = parser.parse_args() - build_data = BuildData(args.project_name, args.engine, args.sanitizer, - args.architecture) + build_data = build_specified_commit.BuildData() + build_data.project_name = args.project_name + build_data.engine = args.engine + build_data.sanitizer = args.sanitizer + build_data.architecture = args.architecture error_sha = bisect(args.commit_old, args.commit_new, args.testcase, args.fuzz_target, build_data) if not error_sha: @@ -107,54 +95,52 @@ def bisect(commit_old, commit_new, testcase, fuzz_target, build_data): specific error from a fuzz testcase. Args: - commit_old: The oldest commit in the error regression range - commit_new: The newest commit in the error regression range + commit_old: The oldest commit in the error regression range. + commit_new: The newest commit in the error regression range. testcase: The file path of the test case that triggers the error - fuzz_target: The name of the fuzzer to be tested - build_data: a class holding all of the input parameters for bisection + fuzz_target: The name of the fuzzer to be tested. + build_data: a class holding all of the input parameters for bisection. Returns: - The commit SHA that introduced the error or None + The commit SHA that introduced the error or None. Raises: - ValueError: when a repo url can't be determine from the project + ValueError: when a repo url can't be determine from the project. """ with tempfile.TemporaryDirectory() as tmp_dir: - repo_url, repo_name = build_specified_commit.detect_main_repo_from_docker( - build_data.project_name, commit_old) + repo_url, repo_name = build_specified_commit.detect_main_repo( + build_data.project_name, commit=commit_old) if not repo_url or not repo_name: raise ValueError('Main git repo can not be determined.') - bisect_repo_manager = repo_manager.RepoManager( - repo_url, tmp_dir, repo_name=repo_name) + bisect_repo_manager = repo_manager.RepoManager(repo_url, + tmp_dir, + repo_name=repo_name) commit_list = bisect_repo_manager.get_commit_list(commit_old, commit_new) old_idx = len(commit_list) - 1 new_idx = 0 - build_specified_commit.build_fuzzer_from_commit( - build_data.project_name, commit_list[new_idx], - bisect_repo_manager.repo_dir, build_data.engine, build_data.sanitizer, - build_data.architecture, bisect_repo_manager) + + build_specified_commit.build_fuzzers_from_commit(build_data, + commit_list[new_idx], + bisect_repo_manager) expected_error_code = helper.reproduce_impl(build_data.project_name, fuzz_target, False, [], [], testcase) # Check if the error is persistent through the commit range - build_specified_commit.build_fuzzer_from_commit( - build_data.project_name, commit_list[old_idx], - bisect_repo_manager.repo_dir, build_data.engine, build_data.sanitizer, - build_data.architecture, bisect_repo_manager) - oldest_error_code = helper.reproduce_impl(build_data.project_name, - fuzz_target, False, [], [], - testcase) + build_specified_commit.build_fuzzers_from_commit(build_data, + commit_list[old_idx], + bisect_repo_manager) - if expected_error_code == oldest_error_code: + if expected_error_code == helper.reproduce_impl(build_data.project_name, + fuzz_target, False, [], [], + testcase): return commit_list[old_idx] while old_idx - new_idx > 1: curr_idx = (old_idx + new_idx) // 2 - build_specified_commit.build_fuzzer_from_commit( - build_data.project_name, commit_list[curr_idx], - bisect_repo_manager.repo_dir, build_data.engine, build_data.sanitizer, - build_data.architecture, bisect_repo_manager) + build_specified_commit.build_fuzzers_from_commit(build_data, + commit_list[curr_idx], + bisect_repo_manager) error_code = helper.reproduce_impl(build_data.project_name, fuzz_target, False, [], [], testcase) if expected_error_code == error_code: diff --git a/infra/bisector_test.py b/infra/bisector_test.py index 1644e8faa..a758d30fe 100644 --- a/infra/bisector_test.py +++ b/infra/bisector_test.py @@ -11,27 +11,31 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing perepo_managerissions and # limitations under the License. -"""Test the functionality of bisection module. -1) Test a known case where an error appears in a regression range -2) Bisect can handle incorrect inputs +"""Test the functionality of bisection module: +1) Test a known case where an error appears in a regression range. +2) Bisect can handle incorrect inputs. """ import os import unittest import bisector +import build_specified_commit # Necessary because __file__ changes with os.chdir TEST_DIR_PATH = os.path.dirname(os.path.realpath(__file__)) class TestBisect(unittest.TestCase): - """Class to test the functionality of bisection method""" + """Class to test the functionality of bisection method.""" def test_bisect_invalid_repo(self): - """Test the bisection method on a project that does not exist""" - build_data = bisector.BuildData('not-a-real-repo', 'libfuzzer', 'address', - 'x86_64') + """Test the bisection method on a project that does not exist.""" + build_data = build_specified_commit.BuildData() + build_data.project_name = 'not-a-real-repo' + build_data.engine = 'libfuzzer' + build_data.sanitizer = 'address' + build_data.architecture = 'x86_64' commit_old = 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' commit_new = 'bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb' testcase = os.path.join(TEST_DIR_PATH, 'testcases', 'usrsctp_test_data') @@ -41,7 +45,11 @@ class TestBisect(unittest.TestCase): def test_bisect_curl(self): """Test the bisect method on the curl project.""" - build_data = bisector.BuildData('curl', 'libfuzzer', 'address', 'x86_64') + build_data = build_specified_commit.BuildData() + build_data.project_name = 'curl' + build_data.engine = 'libfuzzer' + build_data.sanitizer = 'address' + build_data.architecture = 'x86_64' commit_new = 'dda418266c99ceab368d723facb52069cbb9c8d5' commit_old = 'df26f5f9c36e19cd503c0e462e9f72ad37b84c82' fuzz_target = 'curl_fuzzer_ftp' @@ -52,8 +60,11 @@ class TestBisect(unittest.TestCase): def test_bisect_libarchive(self): """Test the bisect method on libarchive.""" - build_data = bisector.BuildData('libarchive', 'libfuzzer', 'undefined', - 'x86_64') + build_data = build_specified_commit.BuildData() + build_data.project_name = 'libarchive' + build_data.engine = 'libfuzzer' + build_data.sanitizer = 'address' + build_data.architecture = 'x86_64' commit_new = '458e49358f17ec58d65ab1c45cf299baaf3c98d1' commit_old = '5bd2a9b6658a3a6efa20bb9ad75bd39a44d71da6' fuzz_target = 'libarchive_fuzzer' @@ -64,7 +75,11 @@ class TestBisect(unittest.TestCase): def test_bisect_usrsctp(self): """Test the bisect method on the usrsctp.""" - build_data = bisector.BuildData('usrsctp', 'libfuzzer', 'address', 'x86_64') + build_data = build_specified_commit.BuildData() + build_data.project_name = 'usrsctp' + build_data.engine = 'libfuzzer' + build_data.sanitizer = 'address' + build_data.architecture = 'x86_64' commit_old = '4886aaa49fb90e479226fcfc3241d74208908232' commit_new = 'c710749b1053978179a027973a3ea3bccf20ee5c' testcase = os.path.join(TEST_DIR_PATH, 'testcases', 'usrsctp_test_data') @@ -75,7 +90,11 @@ class TestBisect(unittest.TestCase): def test_bisect_usrsctp_single_error_exists(self): """Tests what happens with a single with an error.""" - build_data = bisector.BuildData('usrsctp', 'libfuzzer', 'address', 'x86_64') + build_data = build_specified_commit.BuildData() + build_data.project_name = 'usrsctp' + build_data.engine = 'libfuzzer' + build_data.sanitizer = 'address' + build_data.architecture = 'x86_64' commit_old = 'c710749b1053978179a027973a3ea3bccf20ee5c' commit_new = 'c710749b1053978179a027973a3ea3bccf20ee5c' testcase = os.path.join(TEST_DIR_PATH, 'testcases', 'usrsctp_test_data') @@ -86,7 +105,11 @@ class TestBisect(unittest.TestCase): def test_bisect_usrsctp_single_no_error_exists(self): """Tests what happens with a single with an error.""" - build_data = bisector.BuildData('usrsctp', 'libfuzzer', 'address', 'x86_64') + build_data = build_specified_commit.BuildData() + build_data.project_name = 'usrsctp' + build_data.engine = 'libfuzzer' + build_data.sanitizer = 'address' + build_data.architecture = 'x86_64' commit_old = '4886aaa49fb90e479226fcfc3241d74208908232' commit_new = '4886aaa49fb90e479226fcfc3241d74208908232' testcase = os.path.join(TEST_DIR_PATH, 'testcases', 'usrsctp_test_data') diff --git a/infra/build_specified_commit.py b/infra/build_specified_commit.py index 2452a4d91..2288cc135 100644 --- a/infra/build_specified_commit.py +++ b/infra/build_specified_commit.py @@ -22,72 +22,84 @@ import re import subprocess import helper -import repo_manager -class DockerExecutionError(Exception): - """An error that occurs when running a docker command.""" +class BuildData: + """Data required for bisection of errors in OSS-Fuzz projects. + + Attributes: + project_name: The name of the OSS-Fuzz project that is being checked. + engine: The fuzzing engine to be used. + sanitizer: The sanitizer to be used. + architecture: CPU architecture to build the fuzzer for. + """ + + # pylint: disable=too-few-public-methods + + def __init__(self): + self.project_name = '' + self.engine = 'libfuzzer' + self.sanitizer = 'address' + self.architecture = 'x86_64' -def build_fuzzer_from_commit(project_name, - commit, - local_store_path, - engine='libfuzzer', - sanitizer='address', - architecture='x86_64', - old_repo_manager=None): +def build_fuzzers_from_commit(commit, build_repo_manager, build_data): """Builds a OSS-Fuzz fuzzer at a specific commit SHA. Args: - project_name: The OSS-Fuzz project name - commit: The commit SHA to build the fuzzers at - local_store_path: The full file path of a place where a temp git repo is stored - engine: The fuzzing engine to be used - sanitizer: The fuzzing sanitizer to be used - architecture: The system architiecture to be used for fuzzing - + commit: The commit SHA to build the fuzzers at. + build_repo_manager: The OSS-Fuzz project's repo manager to be built at. + build_data: A struct containing project build information. Returns: - 0 on successful build 1 on failure + 0 on successful build or error code on failure. """ - if not old_repo_manager: - inferred_url, repo_name = detect_main_repo_from_docker(project_name, commit) - old_repo_manager = repo_manager.RepoManager( - inferred_url, local_store_path, repo_name=repo_name) - old_repo_manager.checkout_commit(commit) - return helper.build_fuzzers_impl( - project_name=project_name, - clean=True, - engine=engine, - sanitizer=sanitizer, - architecture=architecture, - env_to_add=None, - source_path=old_repo_manager.repo_dir, - mount_location=os.path.join('/src', old_repo_manager.repo_name)) + build_repo_manager.checkout_commit(commit) + return helper.build_fuzzers_impl(project_name=build_data.project_name, + clean=True, + engine=build_data.engine, + sanitizer=build_data.sanitizer, + architecture=build_data.architecture, + env_to_add=None, + source_path=build_repo_manager.repo_dir, + mount_location=os.path.join( + '/src', build_repo_manager.repo_name)) -def detect_main_repo_from_docker(project_name, example_commit, src_dir='/src'): +def detect_main_repo(project_name, repo_name=None, commit=None, src_dir='/src'): """Checks a docker image for the main repo of an OSS-Fuzz project. + Note: The default is to use the repo name to detect the main repo. + Args: - project_name: The name of the OSS-Fuzz project - example_commit: An associated commit SHA - src_dir: The location of the projects source on the docker image + project_name: The name of the oss-fuzz project. + repo_name: The name of the main repo in an OSS-Fuzz project. + commit: A commit SHA that is associated with the main repo. + src_dir: The location of the projects source on the docker image. Returns: - The repo's origin, the repo's name + The repo's origin, the repo's name. """ + # TODO: Add infra for non hardcoded '/src'. + if not repo_name and not commit: + print('Error: can not detect main repo without a repo_name or a commit.') + return None, None + if repo_name and commit: + print('Both repo name and commit specific. Using repo name for detection.') + helper.build_image_impl(project_name) docker_image_name = 'gcr.io/oss-fuzz/' + project_name command_to_run = [ - 'docker', 'run', '--rm', '-i', '-t', docker_image_name, 'python3', - os.path.join(src_dir, 'detect_repo.py'), '--src_dir', src_dir, - '--example_commit', example_commit + 'docker', 'run', '--rm', '-t', docker_image_name, 'python3', + os.path.join(src_dir, 'detect_repo.py'), '--src_dir', src_dir ] + if repo_name: + command_to_run.extend(['--repo_name', repo_name]) + else: + command_to_run.extend(['--example_commit', commit]) out, _ = execute(command_to_run) - match = re.search(r'\bDetected repo: ([^ ]+) ([^ ]+)', out.rstrip()) if match and match.group(1) and match.group(2): - return match.group(1), match.group(2).rstrip() + return match.group(1), match.group(2) return None, None @@ -95,15 +107,15 @@ def execute(command, location=None, check_result=False): """ Runs a shell command in the specified directory location. Args: - command: The command as a list to be run - location: The directory the command is run in - check_result: Should an exception be thrown on failed command + command: The command as a list to be run. + location: The directory the command is run in. + check_result: Should an exception be thrown on failed command. Returns: - The stdout of the command, the error code + The stdout of the command, the error code. Raises: - RuntimeError: running a command resulted in an error + RuntimeError: running a command resulted in an error. """ if not location: diff --git a/infra/build_specified_commit_test.py b/infra/build_specified_commit_test.py index e151fe52f..ef0eb1f3f 100644 --- a/infra/build_specified_commit_test.py +++ b/infra/build_specified_commit_test.py @@ -12,8 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. """Test the functionality of the build image from commit module. -The will consist of the following functional tests - 1. The inferance of the main repo for a specific project +The will consist of the following functional tests: + 1. The inferance of the main repo for a specific project. """ import os import tempfile @@ -21,13 +21,14 @@ import unittest import build_specified_commit import helper +import repo_manager # Necessary because __file__ changes with os.chdir TEST_DIR_PATH = os.path.dirname(os.path.realpath(__file__)) class BuildImageIntegrationTests(unittest.TestCase): - """Testing if an image can be built from different states e.g. a commit""" + """Testing if an image can be built from different states e.g. a commit.""" def test_build_fuzzers_from_commit(self): """Tests if the fuzzers can build at a proper commit. @@ -43,42 +44,79 @@ class BuildImageIntegrationTests(unittest.TestCase): old_commit = 'f79be4f2330f4b89ea2f42e1c44ca998c59a0c0f' new_commit = 'f50a39051ea8c7f10d6d8db9656658b49601caef' fuzzer = 'rules_fuzzer' - build_specified_commit.build_fuzzer_from_commit( - project_name, old_commit, tmp_dir, sanitizer='address') + + yara_repo_manager = repo_manager.RepoManager( + 'https://github.com/VirusTotal/yara.git', tmp_dir, repo_name='yara') + build_data = build_specified_commit.BuildData() + build_data.sanitizer = 'address' + build_data.architecture = 'x86_64' + build_data.engine = 'libfuzzer' + build_data.project_name = 'yara' + build_specified_commit.build_fuzzers_from_commit(old_commit, + yara_repo_manager, + build_data) old_error_code = helper.reproduce_impl(project_name, fuzzer, False, [], [], test_data) - build_specified_commit.build_fuzzer_from_commit( - project_name, new_commit, tmp_dir, sanitizer='address') + build_specified_commit.build_fuzzers_from_commit(new_commit, + yara_repo_manager, + build_data) new_error_code = helper.reproduce_impl(project_name, fuzzer, False, [], [], test_data) self.assertNotEqual(new_error_code, old_error_code) - def test_detect_main_repo(self): - """Test the detect main repo functionality of the build specific commit module.""" - repo_origin, repo_name = build_specified_commit.detect_main_repo_from_docker( - 'curl', 'bc5d22c3dede2f04870c37aec9a50474c4b888ad') + def test_detect_main_repo_from_commit(self): + """Test the detect main repo function from build specific commit module.""" + repo_origin, repo_name = build_specified_commit.detect_main_repo( + 'curl', commit='bc5d22c3dede2f04870c37aec9a50474c4b888ad') self.assertEqual(repo_origin, 'https://github.com/curl/curl.git') self.assertEqual(repo_name, 'curl') - repo_origin, repo_name = build_specified_commit.detect_main_repo_from_docker( - 'usrsctp', '4886aaa49fb90e479226fcfc3241d74208908232') + repo_origin, repo_name = build_specified_commit.detect_main_repo( + 'usrsctp', commit='4886aaa49fb90e479226fcfc3241d74208908232') self.assertEqual(repo_origin, 'https://github.com/weinrank/usrsctp') self.assertEqual(repo_name, 'usrsctp') - repo_origin, repo_name = build_specified_commit.detect_main_repo_from_docker( - 'ndpi', 'c4d476cc583a2ef1e9814134efa4fbf484564ed7') + repo_origin, repo_name = build_specified_commit.detect_main_repo( + 'ndpi', commit='c4d476cc583a2ef1e9814134efa4fbf484564ed7') self.assertEqual(repo_origin, 'https://github.com/ntop/nDPI.git') self.assertEqual(repo_name, 'ndpi') - repo_origin, repo_name = build_specified_commit.detect_main_repo_from_docker( - 'notproj', 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa') + repo_origin, repo_name = build_specified_commit.detect_main_repo( + 'notproj', commit='aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa') + self.assertIsNone(repo_origin) + self.assertIsNone(repo_name) + + def test_detect_main_repo_from_name(self): + """Test the detect main repo function from build specific commit module.""" + repo_origin, repo_name = build_specified_commit.detect_main_repo( + 'curl', repo_name='curl') + self.assertEqual(repo_origin, 'https://github.com/curl/curl.git') + self.assertEqual(repo_name, 'curl') + + repo_origin, repo_name = build_specified_commit.detect_main_repo( + 'yara', repo_name='yara') + self.assertEqual(repo_origin, 'https://github.com/VirusTotal/yara.git') + self.assertEqual(repo_name, 'yara') + + repo_origin, repo_name = build_specified_commit.detect_main_repo( + 'usrsctp', repo_name='usrsctp') + self.assertEqual(repo_origin, 'https://github.com/weinrank/usrsctp') + self.assertEqual(repo_name, 'usrsctp') + + repo_origin, repo_name = build_specified_commit.detect_main_repo( + 'ndpi', repo_name='nDPI') + self.assertEqual(repo_origin, 'https://github.com/ntop/nDPI.git') + self.assertEqual(repo_name, 'ndpi') + + repo_origin, repo_name = build_specified_commit.detect_main_repo( + 'notproj', repo_name='aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa') self.assertIsNone(repo_origin) self.assertIsNone(repo_name) if __name__ == '__main__': - # Change to oss-fuzz main directory so helper.py runs correctly + # Change to oss-fuzz main directory so helper.py runs correctly. if os.getcwd() != os.path.dirname(TEST_DIR_PATH): os.chdir(os.path.dirname(TEST_DIR_PATH)) unittest.main() diff --git a/infra/cifuzz.py b/infra/cifuzz.py new file mode 100644 index 000000000..161e9d953 --- /dev/null +++ b/infra/cifuzz.py @@ -0,0 +1,88 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Module used by CI tools in order to interact with fuzzers. +This module helps CI tools do the following: + 1. Build fuzzers. + 2. Run fuzzers. +Eventually it will be used to help CI tools determine which fuzzers to run. +""" + +import argparse +import os +import tempfile + +import build_specified_commit +import repo_manager +import helper + + +def main(): + """Connects Fuzzers with CI tools. + + Returns: + True on success False on failure. + """ + parser = argparse.ArgumentParser( + description='Help CI tools manage specific fuzzers.') + + subparsers = parser.add_subparsers(dest='command') + build_fuzzer_parser = subparsers.add_parser( + 'build_fuzzers', help='Build an OSS-Fuzz projects fuzzers.') + build_fuzzer_parser.add_argument('project_name') + build_fuzzer_parser.add_argument('repo_name') + build_fuzzer_parser.add_argument('commit_sha') + + run_fuzzer_parser = subparsers.add_parser( + 'run_fuzzers', help='Run an OSS-Fuzz projects fuzzers.') + run_fuzzer_parser.add_argument('project_name') + args = parser.parse_args() + + # Change to oss-fuzz main directory so helper.py runs correctly. + if os.getcwd() != helper.OSSFUZZ_DIR: + os.chdir(helper.OSSFUZZ_DIR) + + if args.command == 'build_fuzzers': + return build_fuzzers(args) == 0 + if args.command == 'run_fuzzer': + print('Not implemented') + return False + print('Invalid argument option, use build_fuzzers or run_fuzzer.') + return False + + +def build_fuzzers(args): + """Builds all of the fuzzers for a specific OSS-Fuzz project. + + Returns: + True on success False on failure. + """ + + # TODO: Fix return value bubble to actually handle errors. + with tempfile.TemporaryDirectory() as tmp_dir: + inferred_url, repo_name = build_specified_commit.detect_main_repo( + args.project_name, repo_name=args.repo_name) + build_repo_manager = repo_manager.RepoManager(inferred_url, + tmp_dir, + repo_name=repo_name) + build_data = build_specified_commit.BuildData() + build_data.project_name = args.project_name + build_data.sanitizer = 'address' + build_data.engine = 'libfuzzer' + build_data.architecture = 'x86_64' + return build_specified_commit.build_fuzzers_from_commit( + args.commit_sha, build_repo_manager, build_data) == 0 + + +if __name__ == '__main__': + main() diff --git a/infra/helper.py b/infra/helper.py index 46c34abbf..4d83175f1 100755 --- a/infra/helper.py +++ b/infra/helper.py @@ -342,7 +342,12 @@ def _workdir_from_dockerfile(project_name): def docker_run(run_args, print_output=True): """Call `docker run`.""" - command = ['docker', 'run', '--rm', '-i', '--privileged'] + command = ['docker', 'run', '--rm', '--privileged'] + + # Support environments with a TTY. + if sys.stdin.isatty(): + command.append('-i') + command.extend(run_args) print('Running:', _get_command_string(command)) @@ -453,9 +458,7 @@ def build_fuzzers_impl(project_name, clean, engine, sanitizer, architecture, 'bash', '-c', 'cp -r /msan /work']) env.append('MSAN_LIBS_PATH=' + '/work/msan') - command = ( - ['docker', 'run', '--rm', '-i', '--cap-add', 'SYS_PTRACE'] + - _env_to_docker_args(env)) + command = ['--cap-add', 'SYS_PTRACE'] + _env_to_docker_args(env) if source_path: workdir = _workdir_from_dockerfile(project_name) if workdir == '/src': @@ -478,13 +481,10 @@ def build_fuzzers_impl(project_name, clean, engine, sanitizer, architecture, '-t', 'gcr.io/oss-fuzz/%s' % project_name ] - print('Running:', _get_command_string(command)) - - try: - subprocess.check_call(command) - except subprocess.CalledProcessError: - print('Fuzzers build failed.', file=sys.stderr) - return 1 + result_code = docker_run(command) + if result_code: + print('Building fuzzers failed.', file=sys.stderr) + return result_code # Patch MSan builds to use instrumented shared libraries. if sanitizer == 'memory':