[CIFuzz] Download code coverage (#3434)

This is the foundation for affected fuzzers. It provides the ability to map fuzzers to files. In the future we will use this functionality to pick which fuzzers to run during CI.

infra/cifuzz/cifuzz.py

@@ -18,10 +18,13 @@ This module helps CI tools do the following:
 Eventually it will be used to help CI tools determine which fuzzers to run.
 """
 
+import json
 import logging
 import os
 import shutil
 import sys
+import urllib.error
+import urllib.request
 
 import fuzz_target
 
@@ -67,6 +70,9 @@ DEFAULT_ENGINE = 'libfuzzer'
 DEFAULT_SANITIZER = 'address'
 DEFAULT_ARCHITECTURE = 'x86_64'
 
+# The path to get project's latest report json files.
+LATEST_REPORT_INFO_PATH = 'oss-fuzz-coverage/latest_report_info/'
+
 # TODO: Turn default logging to WARNING when CIFuzz is stable
 logging.basicConfig(
     format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
@@ -243,6 +249,117 @@ def check_fuzzer_build(out_dir):
   return True
 
 
+def get_latest_cov_report_info(project_name):
+  """Gets latest coverage report info for a specific OSS-Fuzz project from GCS.
+
+  Args:
+    project_name: The name of the relevant OSS-Fuzz project.
+
+  Returns:
+    The projects coverage report info in json dict or None on failure.
+  """
+  latest_report_info_url = fuzz_target.url_join(fuzz_target.GCS_BASE_URL,
+                                                LATEST_REPORT_INFO_PATH,
+                                                project_name + '.json')
+  latest_cov_info_json = get_json_from_url(latest_report_info_url)
+  if not latest_cov_info_json:
+    logging.error('Could not get the coverage report json from url: %s.',
+                  latest_report_info_url)
+    return None
+  return latest_cov_info_json
+
+
+def get_target_coverage_report(latest_cov_info, target_name):
+  """Get the coverage report for a specific fuzz target.
+
+  Args:
+    latest_cov_info: A dict containing a project's latest cov report info.
+    target_name: The name of the fuzz target whose coverage is requested.
+
+  Returns:
+    The targets coverage json dict or None on failure.
+  """
+  if 'fuzzer_stats_dir' not in latest_cov_info:
+    logging.error('The latest coverage report information did not contain'
+                  '\'fuzzer_stats_dir\' key.')
+    return None
+  fuzzer_report_url_segment = latest_cov_info['fuzzer_stats_dir']
+
+  # Converting gs:// to http://
+  fuzzer_report_url_segment = fuzzer_report_url_segment.replace('gs://', '')
+  target_url = fuzz_target.url_join(fuzz_target.GCS_BASE_URL,
+                                    fuzzer_report_url_segment,
+                                    target_name + '.json')
+  return get_json_from_url(target_url)
+
+
+def get_files_covered_by_target(latest_cov_info, target_name,
+                                oss_fuzz_project_base):
+  """Gets a list of files covered by the specific fuzz target.
+
+  Args:
+    latest_cov_info: A dict containing a project's latest cov report info.
+    target_name: The name of the fuzz target whose coverage is requested.
+    oss_fuzz_project_base: The location where OSS-Fuzz project is cloned to for
+      the projects build.
+
+  Returns:
+    A list of files that the fuzzer covers or None.
+
+  Raises:
+    ValueError: When the oss_fuzz_project_base is not defined.
+  """
+  if not oss_fuzz_project_base:
+    return None
+  target_cov = get_target_coverage_report(latest_cov_info, target_name)
+  if not target_cov:
+    return None
+  coverage_per_file = target_cov['data'][0]['files']
+  if not coverage_per_file:
+    logging.info('No files found in coverage report.')
+    return None
+
+  # Cases like curl there is /src/curl and /src/curl_fuzzers/ are handled.
+  if not oss_fuzz_project_base.endswith('/'):
+    oss_fuzz_project_base += '/'
+
+  affected_file_list = []
+  for file in coverage_per_file:
+    if not file['filename'].startswith(oss_fuzz_project_base):
+      continue
+    if not file['summary']['regions']['count']:
+      # Don't consider a file affected if code in it is never executed.
+      continue
+
+    relative_path = file['filename'].replace(oss_fuzz_project_base, '')
+    affected_file_list.append(relative_path)
+  if not affected_file_list:
+    return None
+  return affected_file_list
+
+
+def get_json_from_url(url):
+  """Gets a json object from a specified http url.
+
+  Args:
+    url: The url of the json to be downloaded.
+
+  Returns:
+    Json dict or None on failure.
+  """
+  try:
+    response = urllib.request.urlopen(url)
+  except urllib.error.HTTPError:
+    logging.error('HTTP error with url %s.', url)
+    return None
+  try:
+    result_json = json.load(response)
+  except ValueError as excp:
+    logging.error('Loading json from url %s failed with: %s.', url, str(excp))
+    return None
+  return result_json
+
+
 def parse_fuzzer_output(fuzzer_output, out_dir):
   """Parses the fuzzer output from a fuzz target binary.
 

infra/cifuzz/cifuzz_test.py

@@ -15,8 +15,9 @@
 1. Building fuzzers.
 2. Running fuzzers.
 """
-
+import json
 import os
+import pickle
 import shutil
 import sys
 import tempfile
@@ -247,5 +248,118 @@ class CheckFuzzerBuildUnitTest(unittest.TestCase):
     self.assertFalse(cifuzz.check_fuzzer_build(TEST_FILES_PATH))
 
 
+class GetFilesCoveredByTargetUnitTest(unittest.TestCase):
+  """Test to get the files covered by a fuzz target in the cifuzz module."""
+
+  example_cov_json = 'example_curl_cov.json'
+  example_fuzzer_cov_json = 'example_curl_fuzzer_cov.json'
+  example_fuzzer = 'curl_fuzzer'
+  example_curl_file_list = 'example_curl_file_list'
+
+  def setUp(self):
+    with open(os.path.join(TEST_FILES_PATH, self.example_cov_json),
+              'r') as file:
+      self.proj_cov_report_example = json.loads(file.read())
+    with open(os.path.join(TEST_FILES_PATH, self.example_fuzzer_cov_json),
+              'r') as file:
+      self.fuzzer_cov_report_example = json.loads(file.read())
+
+  def test_valid_target(self):
+    """Tests that covered files can be retrieved from a coverage report."""
+
+    with unittest.mock.patch.object(
+        cifuzz,
+        'get_target_coverage_report',
+        return_value=self.fuzzer_cov_report_example):
+      file_list = cifuzz.get_files_covered_by_target(
+          self.proj_cov_report_example, self.example_fuzzer, '/src/curl')
+
+    with open(os.path.join(TEST_FILES_PATH, 'example_curl_file_list'),
+              'rb') as file_handle:
+      true_files_list = pickle.load(file_handle)
+    self.assertCountEqual(file_list, true_files_list)
+
+  def test_invalid_target(self):
+    """Test asserts an invalid fuzzer returns None."""
+    self.assertIsNone(
+        cifuzz.get_files_covered_by_target(self.proj_cov_report_example,
+                                           'not-a-fuzzer', '/src/curl'))
+    self.assertIsNone(
+        cifuzz.get_files_covered_by_target(self.proj_cov_report_example, '',
+                                           '/src/curl'))
+
+  def test_invalid_project_build_dir(self):
+    """Test asserts an invalid build dir returns None."""
+    self.assertIsNone(
+        cifuzz.get_files_covered_by_target(self.proj_cov_report_example,
+                                           self.example_fuzzer, '/no/pe'))
+    self.assertIsNone(
+        cifuzz.get_files_covered_by_target(self.proj_cov_report_example,
+                                           self.example_fuzzer, ''))
+
+
+class GetTargetCoverageReporUnitTest(unittest.TestCase):
+  """Test get_target_coverage_report function in the cifuzz module."""
+
+  example_cov_json = 'example_curl_cov.json'
+  example_fuzzer = 'curl_fuzzer'
+
+  def setUp(self):
+    with open(os.path.join(TEST_FILES_PATH, self.example_cov_json),
+              'r') as file:
+      self.cov_exmp = json.loads(file.read())
+
+  def test_valid_target(self):
+    """Test a target's coverage report can be downloaded and parsed."""
+    with unittest.mock.patch.object(cifuzz,
+                                    'get_json_from_url',
+                                    return_value='{}') as mock_get_json:
+      cifuzz.get_target_coverage_report(self.cov_exmp, self.example_fuzzer)
+      (url,), _ = mock_get_json.call_args
+      self.assertEqual(
+          'https://storage.googleapis.com/oss-fuzz-coverage/'
+          'curl/fuzzer_stats/20200226/curl_fuzzer.json', url)
+
+  def test_invalid_target(self):
+    """Test an invalid target coverage report will be None."""
+    self.assertIsNone(
+        cifuzz.get_target_coverage_report(self.cov_exmp, 'not-valid-target'))
+    self.assertIsNone(cifuzz.get_target_coverage_report(self.cov_exmp, ''))
+
+  def test_invalid_project_json(self):
+    """Test a project json coverage report will be None."""
+    self.assertIsNone(
+        cifuzz.get_target_coverage_report('not-a-proj', self.example_fuzzer))
+    self.assertIsNone(cifuzz.get_target_coverage_report('',
+                                                        self.example_fuzzer))
+
+
+class GetLatestCoverageReportUnitTest(unittest.TestCase):
+  """Test get_latest_cov_report_info function in the cifuzz module."""
+
+  test_project = 'curl'
+
+  def test_get_valid_project(self):
+    """Tests that a project's coverage report can be downloaded and parsed.
+
+    NOTE: This test relies on the test_project repo's coverage report.
+    Example was not used because it has no coverage reports.
+    """
+    with unittest.mock.patch.object(cifuzz,
+                                    'get_json_from_url',
+                                    return_value='{}') as mock_fun:
+
+      cifuzz.get_latest_cov_report_info(self.test_project)
+      (url,), _ = mock_fun.call_args
+      self.assertEqual(
+          'https://storage.googleapis.com/oss-fuzz-coverage/'
+          'latest_report_info/curl.json', url)
+
+  def test_get_invalid_project(self):
+    """Tests a project's coverage report will return None if bad project."""
+    self.assertIsNone(cifuzz.get_latest_cov_report_info('not-a-proj'))
+    self.assertIsNone(cifuzz.get_latest_cov_report_info(''))
+
+
 if __name__ == '__main__':
   unittest.main()

infra/cifuzz/test_files/example_curl_cov.json

@@ -0,0 +1 @@
+{"report_summary_path": "gs://oss-fuzz-coverage/curl/reports/20200226/linux/summary.json", "html_report_url": "https://storage.googleapis.com/oss-fuzz-coverage/curl/reports/20200226/linux/index.html", "report_date": "20200226", "fuzzer_stats_dir": "gs://oss-fuzz-coverage/curl/fuzzer_stats/20200226"}