From 9f52d142aae40abc094d7f1e651857a52413ee9d Mon Sep 17 00:00:00 2001 From: Leo Neat Date: Wed, 12 Feb 2020 14:44:11 -0800 Subject: [PATCH] [CIFuzz] Only report reproducible crashes (#3376) * Tests for Reproduce * Leo comments * Maxs comments pt.2 * Olivers comments * Olivers comments * Add fuzz target module tests * Formatting * Small punct and spelling * Test update * Format --- infra/cifuzz/cifuzz_test.py | 46 +++++++++++++++- infra/cifuzz/fuzz_target.py | 29 +++++++++- infra/cifuzz/fuzz_target_test.py | 91 ++++++++++++++++++++++++++++++++ 3 files changed, 163 insertions(+), 3 deletions(-) create mode 100644 infra/cifuzz/fuzz_target_test.py diff --git a/infra/cifuzz/cifuzz_test.py b/infra/cifuzz/cifuzz_test.py index b0dd2fdc3..c62a95a69 100644 --- a/infra/cifuzz/cifuzz_test.py +++ b/infra/cifuzz/cifuzz_test.py @@ -20,10 +20,12 @@ import os import sys import tempfile import unittest +import unittest.mock # pylint: disable=wrong-import-position sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) import cifuzz +import fuzz_target # NOTE: This integration test relies on # https://github.com/google/oss-fuzz/tree/master/projects/example project @@ -156,7 +158,7 @@ class RunFuzzersIntegrationTest(unittest.TestCase): class ParseOutputUnitTest(unittest.TestCase): """Test parse_fuzzer_output function in the cifuzz module.""" - def parse_valid_output(self): + def test_parse_valid_output(self): """Checks that the parse fuzzer output can correctly parse output.""" test_case_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'test_files') @@ -175,12 +177,52 @@ class ParseOutputUnitTest(unittest.TestCase): real_summary = bug_summary.read() self.assertEqual(detected_summary, real_summary) - def parse_invalid_output(self): + def test_parse_invalid_output(self): """Checks that no files are created when an invalid input was given.""" with tempfile.TemporaryDirectory() as tmp_dir: cifuzz.parse_fuzzer_output('not a valid output_string', tmp_dir) self.assertEqual(len(os.listdir(tmp_dir)), 0) +class ReproduceIntegrationTest(unittest.TestCase): + """Test that only reproducible bugs are reported by CIFuzz.""" + + def test_reproduce_true(self): + """Checks CIFuzz reports an error when a crash is reproducible.""" + with tempfile.TemporaryDirectory() as tmp_dir: + out_path = os.path.join(tmp_dir, 'out') + os.mkdir(out_path) + self.assertTrue( + cifuzz.build_fuzzers( + EXAMPLE_PROJECT, + 'oss-fuzz', + tmp_dir, + commit_sha='0b95fe1039ed7c38fea1f97078316bfc1030c523')) + with unittest.mock.patch.object(fuzz_target.FuzzTarget, + 'is_reproducible', + return_value=True): + run_success, bug_found = cifuzz.run_fuzzers(5, tmp_dir) + self.assertTrue(run_success) + self.assertTrue(bug_found) + + def test_reproduce_false(self): + """Checks CIFuzz doesn't report an error when a crash isn't reproducible.""" + with tempfile.TemporaryDirectory() as tmp_dir: + out_path = os.path.join(tmp_dir, 'out') + os.mkdir(out_path) + self.assertTrue( + cifuzz.build_fuzzers( + EXAMPLE_PROJECT, + 'oss-fuzz', + tmp_dir, + commit_sha='0b95fe1039ed7c38fea1f97078316bfc1030c523')) + with unittest.mock.patch.object(fuzz_target.FuzzTarget, + 'is_reproducible', + return_value=False): + run_success, bug_found = cifuzz.run_fuzzers(5, tmp_dir) + self.assertTrue(run_success) + self.assertFalse(bug_found) + + if __name__ == '__main__': unittest.main() diff --git a/infra/cifuzz/fuzz_target.py b/infra/cifuzz/fuzz_target.py index cfb01a4ed..f3ed8f4bf 100644 --- a/infra/cifuzz/fuzz_target.py +++ b/infra/cifuzz/fuzz_target.py @@ -30,6 +30,9 @@ logging.basicConfig( LIBFUZZER_OPTIONS = '-seed=1337 -len_control=0' +# The number of reproduce attempts for a crash. +REPRODUCE_ATTEMPTS = 10 + class FuzzTarget: """A class to manage a single fuzz target. @@ -92,7 +95,31 @@ class FuzzTarget: if not test_case: logging.error('No test case found in stack trace.', file=sys.stderr) return None, None - return test_case, err_str + if self.is_reproducible(test_case): + return test_case, err_str + logging.error('A crash was found but it was not reproducible.') + return None, None + + def is_reproducible(self, test_case): + """Checks if the test case reproduces. + + Args: + test_case: The path to the test case to be tested. + + Returns: + True if crash is reproducible. + """ + command = [ + 'docker', 'run', '--rm', '--privileged', '-v', + '%s:/out' % os.path.dirname(self.target_path), '-v', + '%s:/testcase' % test_case, '-t', 'gcr.io/oss-fuzz-base/base-runner', + 'reproduce', self.target_name, '-runs=100' + ] + for _ in range(REPRODUCE_ATTEMPTS): + _, _, err_code = utils.execute(command) + if err_code: + return True + return False def get_test_case(self, error_string): """Gets the file from a fuzzer run stack trace. diff --git a/infra/cifuzz/fuzz_target_test.py b/infra/cifuzz/fuzz_target_test.py new file mode 100644 index 000000000..9656e55fe --- /dev/null +++ b/infra/cifuzz/fuzz_target_test.py @@ -0,0 +1,91 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Test the functionality of the fuzz_target module.""" + +import os +import sys +import unittest +import unittest.mock + +# Pylint has issue importing utils which is why error suppression is required. +# pylint: disable=wrong-import-position +# pylint: disable=import-error +sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +import fuzz_target +import utils + +# NOTE: This integration test relies on +# https://github.com/google/oss-fuzz/tree/master/projects/example project +EXAMPLE_PROJECT = 'example' + + +class IsReproducibleUnitTest(unittest.TestCase): + """Test is_reproducible function in the fuzz_target module.""" + + def setUp(self): + """Sets up dummy fuzz target to test is_reproducible method.""" + self.test_target = fuzz_target.FuzzTarget('/example/path', 10, + '/example/outdir') + + def test_with_reproducible(self): + """Tests that a is_reproducible will return true if crash is detected.""" + test_all_success = [(0, 0, 1)] * 10 + all_success_mock = unittest.mock.Mock() + all_success_mock.side_effect = test_all_success + utils.execute = all_success_mock + self.assertTrue(self.test_target.is_reproducible('/fake/path/to/testcase')) + self.assertEqual(1, all_success_mock.call_count) + + test_one_success = [(0, 0, 0)] * 9 + [(0, 0, 1)] + one_success_mock = unittest.mock.Mock() + one_success_mock.side_effect = test_one_success + utils.execute = one_success_mock + self.assertTrue(self.test_target.is_reproducible('/fake/path/to/testcase')) + self.assertEqual(10, one_success_mock.call_count) + + def test_with_not_reproducible(self): + """Tests that a is_reproducible will return False if crash not detected.""" + test_all_fail = [(0, 0, 0)] * 10 + all_fail_mock = unittest.mock.Mock() + all_fail_mock.side_effect = test_all_fail + utils.execute = all_fail_mock + self.assertFalse(self.test_target.is_reproducible('/fake/path/to/testcase')) + + +class GetTestCaseUnitTest(unittest.TestCase): + """Test get_test_case function in the fuzz_target module.""" + + def setUp(self): + """Sets up dummy fuzz target to test get_test_case method.""" + self.test_target = fuzz_target.FuzzTarget('/example/path', 10, + '/example/outdir') + + def test_with_valid_error_string(self): + """Tests that get_test_case returns the correct test case give an error.""" + test_case_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), + 'test_files', 'example_fuzzer_output.txt') + with open(test_case_path, 'r') as test_fuzz_output: + parsed_test_case = self.test_target.get_test_case(test_fuzz_output.read()) + self.assertEqual( + parsed_test_case, + '/example/outdir/crash-ad6700613693ef977ff3a8c8f4dae239c3dde6f5') + + def test_with_invalid_error_string(self): + """Tests that get_test_case will return None with a bad error string.""" + self.assertIsNone(self.test_target.get_test_case('')) + self.assertIsNone(self.test_target.get_test_case(' Example crash string.')) + + +if __name__ == '__main__': + unittest.main()