[CIFuzz] Only report reproducible crashes (#3376)

* Tests for Reproduce

* Leo comments

* Maxs comments pt.2

* Olivers comments

* Olivers comments

* Add fuzz target module tests

* Formatting

* Small punct and spelling

* Test update

* Format

infra/cifuzz/cifuzz_test.py

@@ -20,10 +20,12 @@ import os
 import sys
 import tempfile
 import unittest
+import unittest.mock
 
 # pylint: disable=wrong-import-position
 sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 import cifuzz
+import fuzz_target
 
 # NOTE: This integration test relies on
 # https://github.com/google/oss-fuzz/tree/master/projects/example project
@@ -156,7 +158,7 @@ class RunFuzzersIntegrationTest(unittest.TestCase):
 class ParseOutputUnitTest(unittest.TestCase):
   """Test parse_fuzzer_output function in the cifuzz module."""
 
-  def parse_valid_output(self):
+  def test_parse_valid_output(self):
     """Checks that the parse fuzzer output can correctly parse output."""
     test_case_path = os.path.join(os.path.dirname(os.path.abspath(__file__)),
                                   'test_files')
@@ -175,12 +177,52 @@ class ParseOutputUnitTest(unittest.TestCase):
         real_summary = bug_summary.read()
       self.assertEqual(detected_summary, real_summary)
 
-  def parse_invalid_output(self):
+  def test_parse_invalid_output(self):
     """Checks that no files are created when an invalid input was given."""
     with tempfile.TemporaryDirectory() as tmp_dir:
       cifuzz.parse_fuzzer_output('not a valid output_string', tmp_dir)
       self.assertEqual(len(os.listdir(tmp_dir)), 0)
 
 
+class ReproduceIntegrationTest(unittest.TestCase):
+  """Test that only reproducible bugs are reported by CIFuzz."""
+
+  def test_reproduce_true(self):
+    """Checks CIFuzz reports an error when a crash is reproducible."""
+    with tempfile.TemporaryDirectory() as tmp_dir:
+      out_path = os.path.join(tmp_dir, 'out')
+      os.mkdir(out_path)
+      self.assertTrue(
+          cifuzz.build_fuzzers(
+              EXAMPLE_PROJECT,
+              'oss-fuzz',
+              tmp_dir,
+              commit_sha='0b95fe1039ed7c38fea1f97078316bfc1030c523'))
+      with unittest.mock.patch.object(fuzz_target.FuzzTarget,
+                                      'is_reproducible',
+                                      return_value=True):
+        run_success, bug_found = cifuzz.run_fuzzers(5, tmp_dir)
+        self.assertTrue(run_success)
+        self.assertTrue(bug_found)
+
+  def test_reproduce_false(self):
+    """Checks CIFuzz doesn't report an error when a crash isn't reproducible."""
+    with tempfile.TemporaryDirectory() as tmp_dir:
+      out_path = os.path.join(tmp_dir, 'out')
+      os.mkdir(out_path)
+      self.assertTrue(
+          cifuzz.build_fuzzers(
+              EXAMPLE_PROJECT,
+              'oss-fuzz',
+              tmp_dir,
+              commit_sha='0b95fe1039ed7c38fea1f97078316bfc1030c523'))
+      with unittest.mock.patch.object(fuzz_target.FuzzTarget,
+                                      'is_reproducible',
+                                      return_value=False):
+        run_success, bug_found = cifuzz.run_fuzzers(5, tmp_dir)
+        self.assertTrue(run_success)
+        self.assertFalse(bug_found)
+
+
 if __name__ == '__main__':
   unittest.main()

infra/cifuzz/fuzz_target.py

@@ -30,6 +30,9 @@ logging.basicConfig(
 
 LIBFUZZER_OPTIONS = '-seed=1337 -len_control=0'
 
+# The number of reproduce attempts for a crash.
+REPRODUCE_ATTEMPTS = 10
+
 
 class FuzzTarget:
   """A class to manage a single fuzz target.
@@ -92,7 +95,31 @@ class FuzzTarget:
     if not test_case:
       logging.error('No test case found in stack trace.', file=sys.stderr)
       return None, None
-    return test_case, err_str
+    if self.is_reproducible(test_case):
+      return test_case, err_str
+    logging.error('A crash was found but it was not reproducible.')
+    return None, None
+
+  def is_reproducible(self, test_case):
+    """Checks if the test case reproduces.
+
+      Args:
+        test_case: The path to the test case to be tested.
+
+      Returns:
+        True if crash is reproducible.
+    """
+    command = [
+        'docker', 'run', '--rm', '--privileged', '-v',
+        '%s:/out' % os.path.dirname(self.target_path), '-v',
+        '%s:/testcase' % test_case, '-t', 'gcr.io/oss-fuzz-base/base-runner',
+        'reproduce', self.target_name, '-runs=100'
+    ]
+    for _ in range(REPRODUCE_ATTEMPTS):
+      _, _, err_code = utils.execute(command)
+      if err_code:
+        return True
+    return False
 
   def get_test_case(self, error_string):
     """Gets the file from a fuzzer run stack trace.

infra/cifuzz/fuzz_target_test.py

@@ -0,0 +1,91 @@
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Test the functionality of the fuzz_target module."""
+
+import os
+import sys
+import unittest
+import unittest.mock
+
+# Pylint has issue importing utils which is why error suppression is required.
+# pylint: disable=wrong-import-position
+# pylint: disable=import-error
+sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+import fuzz_target
+import utils
+
+# NOTE: This integration test relies on
+# https://github.com/google/oss-fuzz/tree/master/projects/example project
+EXAMPLE_PROJECT = 'example'
+
+
+class IsReproducibleUnitTest(unittest.TestCase):
+  """Test is_reproducible function in the fuzz_target module."""
+
+  def setUp(self):
+    """Sets up dummy fuzz target to test is_reproducible method."""
+    self.test_target = fuzz_target.FuzzTarget('/example/path', 10,
+                                              '/example/outdir')
+
+  def test_with_reproducible(self):
+    """Tests that a is_reproducible will return true if crash is detected."""
+    test_all_success = [(0, 0, 1)] * 10
+    all_success_mock = unittest.mock.Mock()
+    all_success_mock.side_effect = test_all_success
+    utils.execute = all_success_mock
+    self.assertTrue(self.test_target.is_reproducible('/fake/path/to/testcase'))
+    self.assertEqual(1, all_success_mock.call_count)
+
+    test_one_success = [(0, 0, 0)] * 9 + [(0, 0, 1)]
+    one_success_mock = unittest.mock.Mock()
+    one_success_mock.side_effect = test_one_success
+    utils.execute = one_success_mock
+    self.assertTrue(self.test_target.is_reproducible('/fake/path/to/testcase'))
+    self.assertEqual(10, one_success_mock.call_count)
+
+  def test_with_not_reproducible(self):
+    """Tests that a is_reproducible will return False if crash not detected."""
+    test_all_fail = [(0, 0, 0)] * 10
+    all_fail_mock = unittest.mock.Mock()
+    all_fail_mock.side_effect = test_all_fail
+    utils.execute = all_fail_mock
+    self.assertFalse(self.test_target.is_reproducible('/fake/path/to/testcase'))
+
+
+class GetTestCaseUnitTest(unittest.TestCase):
+  """Test get_test_case function in the fuzz_target module."""
+
+  def setUp(self):
+    """Sets up dummy fuzz target to test get_test_case method."""
+    self.test_target = fuzz_target.FuzzTarget('/example/path', 10,
+                                              '/example/outdir')
+
+  def test_with_valid_error_string(self):
+    """Tests that get_test_case returns the correct test case give an error."""
+    test_case_path = os.path.join(os.path.dirname(os.path.abspath(__file__)),
+                                  'test_files', 'example_fuzzer_output.txt')
+    with open(test_case_path, 'r') as test_fuzz_output:
+      parsed_test_case = self.test_target.get_test_case(test_fuzz_output.read())
+    self.assertEqual(
+        parsed_test_case,
+        '/example/outdir/crash-ad6700613693ef977ff3a8c8f4dae239c3dde6f5')
+
+  def test_with_invalid_error_string(self):
+    """Tests that get_test_case will return None with a bad error string."""
+    self.assertIsNone(self.test_target.get_test_case(''))
+    self.assertIsNone(self.test_target.get_test_case(' Example crash string.'))
+
+
+if __name__ == '__main__':
+  unittest.main()