diff --git a/.travis.yml b/.travis.yml index 2064c60d3..332a52b38 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,12 +10,15 @@ before_install: install: - pip install -r infra/travis/requirements.txt + matrix: include: - name: "presubmit" install: - pip install -r infra/dev-requirements.txt script: ./infra/presubmit.py + - name: "tests" + script: sudo ./infra/presubmit.py test - name: "libfuzzer address x86_64" env: - TRAVIS_ENGINE=libfuzzer diff --git a/infra/bisector_test.py b/infra/bisector_test.py index 29904dd6d..aa42329eb 100644 --- a/infra/bisector_test.py +++ b/infra/bisector_test.py @@ -29,6 +29,7 @@ import test_repos TEST_DIR_PATH = os.path.dirname(os.path.realpath(__file__)) +@unittest.skip('Test is too long to be run with presubmit.') class BisectIntegrationTests(unittest.TestCase): """Class to test the functionality of bisection method.""" diff --git a/infra/cifuzz/cifuzz_test.py b/infra/cifuzz/cifuzz_test.py index 27c96d90b..3060cea7a 100644 --- a/infra/cifuzz/cifuzz_test.py +++ b/infra/cifuzz/cifuzz_test.py @@ -17,6 +17,7 @@ """ import os +import shutil import sys import tempfile import unittest @@ -28,9 +29,16 @@ import cifuzz import fuzz_target # NOTE: This integration test relies on -# https://github.com/google/oss-fuzz/tree/master/projects/example project +# https://github.com/google/oss-fuzz/tree/master/projects/example project. EXAMPLE_PROJECT = 'example' +# Location of files used for testing. +TEST_FILES_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), + 'test_files') + +# An example fuzzer that triggers an error. +EXAMPLE_FUZZER = 'do_stuff_fuzzer' + class BuildFuzzersIntegrationTest(unittest.TestCase): """Test build_fuzzers function in the utils module.""" @@ -46,7 +54,7 @@ class BuildFuzzersIntegrationTest(unittest.TestCase): 'oss-fuzz', tmp_dir, commit_sha='0b95fe1039ed7c38fea1f97078316bfc1030c523')) - self.assertTrue(os.path.exists(os.path.join(out_path, 'do_stuff_fuzzer'))) + self.assertTrue(os.path.exists(os.path.join(out_path, EXAMPLE_FUZZER))) def test_valid_pull_request(self): """Test building fuzzers with valid pull request.""" @@ -58,7 +66,7 @@ class BuildFuzzersIntegrationTest(unittest.TestCase): 'oss-fuzz', tmp_dir, pr_ref='refs/pull/1757/merge')) - self.assertTrue(os.path.exists(os.path.join(out_path, 'do_stuff_fuzzer'))) + self.assertTrue(os.path.exists(os.path.join(out_path, EXAMPLE_FUZZER))) def test_invalid_pull_request(self): """Test building fuzzers with invalid pull request.""" @@ -114,60 +122,53 @@ class BuildFuzzersIntegrationTest(unittest.TestCase): class RunFuzzersIntegrationTest(unittest.TestCase): """Test build_fuzzers function in the cifuzz module.""" + def tearDown(self): + """Remove any existing crashes and test files.""" + out_dir = os.path.join(TEST_FILES_PATH, 'out') + for out_file in os.listdir(out_dir): + out_path = os.path.join(out_dir, out_file) + if out_file == EXAMPLE_FUZZER: + continue + if os.path.isdir(out_path): + shutil.rmtree(out_path) + else: + os.remove(out_path) + def test_new_bug_found(self): """Test run_fuzzers with a valid build.""" - with tempfile.TemporaryDirectory() as tmp_dir: - out_path = os.path.join(tmp_dir, 'out') - os.mkdir(out_path) - self.assertTrue( - cifuzz.build_fuzzers( - EXAMPLE_PROJECT, - 'oss-fuzz', - tmp_dir, - commit_sha='0b95fe1039ed7c38fea1f97078316bfc1030c523')) - self.assertTrue(os.path.exists(os.path.join(out_path, 'do_stuff_fuzzer'))) - - # Setting the first return value to True, then the second to False to - # emulate a bug existing in the current PR but not on the downloaded - # OSS-Fuzz build. - with unittest.mock.patch.object(fuzz_target.FuzzTarget, - 'is_reproducible', - side_effect=[True, False]): - run_success, bug_found = cifuzz.run_fuzzers(5, tmp_dir, EXAMPLE_PROJECT) - build_dir = os.path.join(tmp_dir, 'out', 'oss_fuzz_latest') - self.assertTrue(os.path.exists(build_dir)) - self.assertNotEqual(0, len(os.listdir(build_dir))) - self.assertTrue(run_success) - self.assertTrue(bug_found) + # Setting the first return value to True, then the second to False to + # emulate a bug existing in the current PR but not on the downloaded + # OSS-Fuzz build. + with unittest.mock.patch.object(fuzz_target.FuzzTarget, + 'is_reproducible', + side_effect=[True, False]): + run_success, bug_found = cifuzz.run_fuzzers(100, TEST_FILES_PATH, + EXAMPLE_PROJECT) + build_dir = os.path.join(TEST_FILES_PATH, 'out', 'oss_fuzz_latest') + self.assertTrue(os.path.exists(build_dir)) + self.assertNotEqual(0, len(os.listdir(build_dir))) + self.assertTrue(run_success) + self.assertTrue(bug_found) def test_old_bug_found(self): """Test run_fuzzers with a bug found in OSS-Fuzz before.""" - with tempfile.TemporaryDirectory() as tmp_dir: - out_path = os.path.join(tmp_dir, 'out') - os.mkdir(out_path) - self.assertTrue( - cifuzz.build_fuzzers( - EXAMPLE_PROJECT, - 'oss-fuzz', - tmp_dir, - commit_sha='0b95fe1039ed7c38fea1f97078316bfc1030c523')) - self.assertTrue(os.path.exists(os.path.join(out_path, 'do_stuff_fuzzer'))) - with unittest.mock.patch.object(fuzz_target.FuzzTarget, - 'is_reproducible', - side_effect=[True, True]): - run_success, bug_found = cifuzz.run_fuzzers(5, tmp_dir, EXAMPLE_PROJECT) - build_dir = os.path.join(tmp_dir, 'out', 'oss_fuzz_latest') - self.assertTrue(os.path.exists(build_dir)) - self.assertNotEqual(0, len(os.listdir(build_dir))) - self.assertTrue(run_success) - self.assertFalse(bug_found) + with unittest.mock.patch.object(fuzz_target.FuzzTarget, + 'is_reproducible', + side_effect=[True, True]): + run_success, bug_found = cifuzz.run_fuzzers(100, TEST_FILES_PATH, + EXAMPLE_PROJECT) + build_dir = os.path.join(TEST_FILES_PATH, 'out', 'oss_fuzz_latest') + self.assertTrue(os.path.exists(build_dir)) + self.assertNotEqual(0, len(os.listdir(build_dir))) + self.assertTrue(run_success) + self.assertFalse(bug_found) def test_invalid_build(self): """Test run_fuzzers with an invalid build.""" with tempfile.TemporaryDirectory() as tmp_dir: out_path = os.path.join(tmp_dir, 'out') os.mkdir(out_path) - run_success, bug_found = cifuzz.run_fuzzers(5, tmp_dir, EXAMPLE_PROJECT) + run_success, bug_found = cifuzz.run_fuzzers(100, tmp_dir, EXAMPLE_PROJECT) self.assertFalse(run_success) self.assertFalse(bug_found) @@ -182,7 +183,7 @@ class RunFuzzersIntegrationTest(unittest.TestCase): def test_invalid_out_dir(self): """Tests run_fuzzers with an invalid out directory.""" - run_success, bug_found = cifuzz.run_fuzzers(5, 'not/a/valid/path', + run_success, bug_found = cifuzz.run_fuzzers(100, 'not/a/valid/path', EXAMPLE_PROJECT) self.assertFalse(run_success) self.assertFalse(bug_found) @@ -193,10 +194,9 @@ class ParseOutputUnitTest(unittest.TestCase): def test_parse_valid_output(self): """Checks that the parse fuzzer output can correctly parse output.""" - test_case_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), - 'test_files') - test_output_path = os.path.join(test_case_path, 'example_fuzzer_output.txt') - test_summary_path = os.path.join(test_case_path, 'bug_summary_example.txt') + test_output_path = os.path.join(TEST_FILES_PATH, + 'example_fuzzer_output.txt') + test_summary_path = os.path.join(TEST_FILES_PATH, 'bug_summary_example.txt') with tempfile.TemporaryDirectory() as tmp_dir: with open(test_output_path, 'r') as test_fuzz_output: cifuzz.parse_fuzzer_output(test_fuzz_output.read(), tmp_dir) diff --git a/infra/cifuzz/fuzz_target.py b/infra/cifuzz/fuzz_target.py index d49c09815..304f57cb1 100644 --- a/infra/cifuzz/fuzz_target.py +++ b/infra/cifuzz/fuzz_target.py @@ -113,7 +113,7 @@ class FuzzTarget: err_str = err.decode('ascii') test_case = self.get_test_case(err_str) if not test_case: - logging.error('No test case found in stack trace. %s.', sys.stderr) + logging.error('No test case found in stack trace: %s.', err_str) return None, None if self.check_reproducibility_and_regression(test_case): return test_case, err_str diff --git a/infra/cifuzz/fuzz_target_test.py b/infra/cifuzz/fuzz_target_test.py index d770b13bf..14f869624 100644 --- a/infra/cifuzz/fuzz_target_test.py +++ b/infra/cifuzz/fuzz_target_test.py @@ -42,32 +42,32 @@ class IsReproducibleUnitTest(unittest.TestCase): def test_with_reproducible(self): """Tests that a is_reproducible will return true if crash is detected.""" test_all_success = [(0, 0, 1)] * 10 - all_success_mock = unittest.mock.Mock() - all_success_mock.side_effect = test_all_success - utils.execute = all_success_mock - self.assertTrue( - self.test_target.is_reproducible('/fake/path/to/testcase', - '/fake/target')) - self.assertEqual(1, all_success_mock.call_count) + with unittest.mock.patch.object(utils, + 'execute', + side_effect=test_all_success) as patch: + self.assertTrue( + self.test_target.is_reproducible('/fake/path/to/testcase', + '/fake/target')) + self.assertEqual(1, patch.call_count) test_one_success = [(0, 0, 0)] * 9 + [(0, 0, 1)] - one_success_mock = unittest.mock.Mock() - one_success_mock.side_effect = test_one_success - utils.execute = one_success_mock - self.assertTrue( - self.test_target.is_reproducible('/fake/path/to/testcase', - '/fake/target')) - self.assertEqual(10, one_success_mock.call_count) + with unittest.mock.patch.object(utils, + 'execute', + side_effect=test_one_success) as patch: + self.assertTrue( + self.test_target.is_reproducible('/fake/path/to/testcase', + '/fake/target')) + self.assertEqual(10, patch.call_count) def test_with_not_reproducible(self): """Tests that a is_reproducible will return False if crash not detected.""" test_all_fail = [(0, 0, 0)] * 10 - all_fail_mock = unittest.mock.Mock() - all_fail_mock.side_effect = test_all_fail - utils.execute = all_fail_mock - self.assertFalse( - self.test_target.is_reproducible('/fake/path/to/testcase', - '/fake/target')) + with unittest.mock.patch.object(utils, 'execute', + side_effect=test_all_fail) as patch: + self.assertFalse( + self.test_target.is_reproducible('/fake/path/to/testcase', + '/fake/target')) + self.assertEqual(10, patch.call_count) class GetTestCaseUnitTest(unittest.TestCase): diff --git a/infra/cifuzz/test_files/out/do_stuff_fuzzer b/infra/cifuzz/test_files/out/do_stuff_fuzzer new file mode 100755 index 000000000..704800dda Binary files /dev/null and b/infra/cifuzz/test_files/out/do_stuff_fuzzer differ diff --git a/infra/presubmit.py b/infra/presubmit.py index 71da8fd9a..c51f66b9a 100755 --- a/infra/presubmit.py +++ b/infra/presubmit.py @@ -20,6 +20,7 @@ import argparse import os import subprocess import sys +import unittest import yaml _SRC_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) @@ -320,12 +321,25 @@ def get_changed_files(): ] +def run_tests(): + """Run all unit tests in directories that are different from HEAD.""" + changed_dirs = set() + for file in get_changed_files(): + changed_dirs.add(os.path.dirname(file)) + suite_list = [] + for change_dir in changed_dirs: + suite_list.append(unittest.TestLoader().discover(change_dir, + pattern='*_test.py')) + full_suite = unittest.TestSuite(suite_list) + unittest.TextTestRunner().run(full_suite) + + def main(): """Check changes on a branch for common issues before submitting.""" # Get program arguments. parser = argparse.ArgumentParser(description='Presubmit script for oss-fuzz.') parser.add_argument('command', - choices=['format', 'lint', 'license'], + choices=['format', 'lint', 'license', 'test'], nargs='?') args = parser.parse_args() @@ -346,8 +360,12 @@ def main(): success = check_license(changed_files) return bool_to_returncode(success) - # Otherwise, do all of them. + if args.command == 'test': + return run_tests() + + # Do all the checks (but no tests). success = do_checks(changed_files) + return bool_to_returncode(success) diff --git a/infra/repo_manager_test.py b/infra/repo_manager_test.py index 199df0181..b5a914d3a 100644 --- a/infra/repo_manager_test.py +++ b/infra/repo_manager_test.py @@ -116,9 +116,9 @@ class RepoManagerCheckoutPullRequestUnitTests(unittest.TestCase): """Tests that the git checkout pull request works.""" with tempfile.TemporaryDirectory() as tmp_dir: test_repo_manager = repo_manager.RepoManager(OSS_FUZZ_REPO, tmp_dir) - test_repo_manager.checkout_pr('refs/pull/1757/merge') + test_repo_manager.checkout_pr('refs/pull/3415/merge') self.assertEqual(test_repo_manager.get_current_commit(), - '2a2b11cc3d370db8f7bdf73046f3290a39615347') + '314c9249a54a08e764a5bbcb7333294ae7c1f9ed') def test_checkout_invalid_pull_request(self): """Tests that the git checkout invalid pull request fails.""" diff --git a/infra/utils_test.py b/infra/utils_test.py index 5898537c7..f586b4e82 100644 --- a/infra/utils_test.py +++ b/infra/utils_test.py @@ -22,6 +22,9 @@ import helper EXAMPLE_PROJECT = 'example' +TEST_OUT_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), + 'cifuzz', 'test_files', 'out') + class IsFuzzTargetLocalUnitTest(unittest.TestCase): """Test is_fuzz_target_local function in the utils module.""" @@ -37,22 +40,11 @@ class IsFuzzTargetLocalUnitTest(unittest.TestCase): def test_valid_filepath(self): """Checks is_fuzz_target_local function with a valid filepath.""" - utils.chdir_to_root() - helper.build_fuzzers_impl(EXAMPLE_PROJECT, - True, - 'libfuzzer', - 'address', - 'x86_64', [], - None, - no_cache=False, - mount_location=None) + is_local = utils.is_fuzz_target_local( - os.path.join(helper.OSSFUZZ_DIR, 'build', 'out', EXAMPLE_PROJECT, - 'do_stuff_fuzzer')) + os.path.join(TEST_OUT_DIR, 'do_stuff_fuzzer')) self.assertTrue(is_local) - is_local = utils.is_fuzz_target_local( - os.path.join(helper.OSSFUZZ_DIR, 'build', 'out', EXAMPLE_PROJECT, - 'do_stuff_fuzzer.dict')) + is_local = utils.is_fuzz_target_local(TEST_OUT_DIR) self.assertFalse(is_local) @@ -61,36 +53,15 @@ class GetFuzzTargetsUnitTest(unittest.TestCase): def test_valid_filepath(self): """Tests that fuzz targets can be retrieved once the fuzzers are built.""" - utils.chdir_to_root() - helper.build_fuzzers_impl(EXAMPLE_PROJECT, - True, - 'libfuzzer', - 'address', - 'x86_64', [], - None, - no_cache=False, - mount_location=None) - fuzz_targets = utils.get_fuzz_targets( - os.path.join(helper.OSSFUZZ_DIR, 'build', 'out', EXAMPLE_PROJECT)) - self.assertCountEqual(fuzz_targets, [ - os.path.join(helper.OSSFUZZ_DIR, 'build', 'out', EXAMPLE_PROJECT, - 'do_stuff_fuzzer') - ]) + fuzz_targets = utils.get_fuzz_targets(TEST_OUT_DIR) + self.assertCountEqual(fuzz_targets, + [os.path.join(TEST_OUT_DIR, 'do_stuff_fuzzer')]) fuzz_targets = utils.get_fuzz_targets( os.path.join(helper.OSSFUZZ_DIR, 'infra')) self.assertFalse(fuzz_targets) def test_invalid_filepath(self): """Tests what get_fuzz_targets return when invalid filepath is used.""" - utils.chdir_to_root() - helper.build_fuzzers_impl(EXAMPLE_PROJECT, - True, - 'libfuzzer', - 'address', - 'x86_64', [], - None, - no_cache=False, - mount_location=None) fuzz_targets = utils.get_fuzz_targets('not/a/valid/file/path') self.assertFalse(fuzz_targets)