Source-based code coverage reporting for Jazzer.js (#9758)

infra/base-images/base-builder/compile_javascript_fuzzer

@@ -35,6 +35,6 @@ fuzzer_basename=$(basename -s .js $fuzz_target)
 echo "#!/bin/bash
 # LLVMFuzzerTestOneInput so that the wrapper script is recognized as a fuzz target for 'check_build'.
 this_dir=\$(dirname \"\$0\")
-\$this_dir/$project/node_modules/@jazzer.js/core/dist/cli.js $project/$fuzz_target $jazzerjs_args -- \$@" > $OUT/$fuzzer_basename
+\$this_dir/$project/node_modules/@jazzer.js/core/dist/cli.js $project/$fuzz_target $jazzerjs_args \$JAZZERJS_EXTRA_ARGS -- \$@" > $OUT/$fuzzer_basename
 
 chmod +x $OUT/$fuzzer_basename

infra/base-images/base-runner/Dockerfile

@@ -91,6 +91,7 @@ COPY bad_build_check \
     coverage_helper \
     download_corpus \
     jacoco_report_converter.py \
+    nyc_report_converter.py \
     rcfilt \
     reproduce \
     run_fuzzer \

infra/base-images/base-runner/coverage

@@ -31,6 +31,8 @@ fi
 COVERAGE_OUTPUT_DIR=${COVERAGE_OUTPUT_DIR:-$OUT}
 
 DUMPS_DIR="$COVERAGE_OUTPUT_DIR/dumps"
+FUZZERS_COVERAGE_DUMPS_DIR="$DUMPS_DIR/fuzzers_coverage"
+MERGED_COVERAGE_DIR="$COVERAGE_OUTPUT_DIR/merged_coverage"
 FUZZER_STATS_DIR="$COVERAGE_OUTPUT_DIR/fuzzer_stats"
 TEXTCOV_REPORT_DIR="$COVERAGE_OUTPUT_DIR/textcov_reports"
 LOGS_DIR="$COVERAGE_OUTPUT_DIR/logs"
@@ -40,7 +42,7 @@ PLATFORM=linux
 REPORT_PLATFORM_DIR="$COVERAGE_OUTPUT_DIR/report/$PLATFORM"
 
 for directory in $DUMPS_DIR $FUZZER_STATS_DIR $LOGS_DIR $REPORT_ROOT_DIR $TEXTCOV_REPORT_DIR\
-                 $REPORT_PLATFORM_DIR $REPORT_BY_TARGET_ROOT_DIR; do
+                 $REPORT_PLATFORM_DIR $REPORT_BY_TARGET_ROOT_DIR $FUZZERS_COVERAGE_DUMPS_DIR $MERGED_COVERAGE_DIR; do
   rm -rf $directory
   mkdir -p $directory
 done
@@ -222,6 +224,51 @@ function run_java_fuzz_target {
   jacoco_report_converter.py $xml_report $summary_file
 }
 
+function run_javascript_fuzz_target {
+  local target=$1
+  local corpus_real="$CORPUS_DIR/${target}"
+
+  # -merge=1 requires an output directory, create a new, empty dir for that.
+  local corpus_dummy="$OUT/dummy_corpus_dir_for_${target}"
+  rm -rf $corpus_dummy && mkdir -p $corpus_dummy
+
+  # IstanbulJS currently does not work when the tested program creates
+  # subprocesses. For this reason, we first minimize the corpus removing
+  # any crashing inputs so that we can report source-based code coverage
+  # with a single sweep over the minimized corpus
+  local merge_args="-merge=1 -timeout=100 $corpus_dummy $corpus_real"
+  timeout $TIMEOUT $OUT/$target $merge_args &> $LOGS_DIR/$target.log
+
+  # nyc saves the coverage reports in a directory with the default name "coverage"
+  local coverage_dir="$DUMPS_DIR/coverage_dir_for_${target}"
+  rm -rf $coverage_dir && mkdir -p $coverage_dir
+
+  local nyc_json_coverage_file="$coverage_dir/coverage-final.json"
+  local nyc_json_summary_file="$coverage_dir/coverage-summary.json"
+
+  local args="-runs=0 $corpus_dummy"
+  local jazzerjs_args="--coverage --coverageDirectory $coverage_dir --coverageReporters json --coverageReporters json-summary"
+
+  JAZZERJS_EXTRA_ARGS=$jazzerjs_args $OUT/$target $args &> $LOGS_DIR/$target.log
+
+  if (( $? != 0 )); then
+    echo "Error occured while running $target:"
+    cat $LOGS_DIR/$target.log
+  fi
+
+  if [ ! -s $nyc_json_coverage_file ]; then
+    # Skip fuzz targets that failed to produce coverage-final.json file.
+    echo "$target failed to produce coverage-final.json file."
+    return 0
+  fi
+
+  cp $nyc_json_coverage_file $FUZZERS_COVERAGE_DUMPS_DIR/$target.json
+
+  local summary_file="$FUZZER_STATS_DIR/$target.json"
+
+  nyc_report_converter.py $nyc_json_summary_file $summary_file
+}
+
 function generate_html {
   local profdata=$1
   local shared_libraries=$2
@@ -265,6 +312,14 @@ for fuzz_target in $FUZZ_TARGETS; do
 
     echo "Running $fuzz_target"
     run_java_fuzz_target $fuzz_target &
+  elif [[ $FUZZING_LANGUAGE == "javascript" ]]; then
+    # Continue if not a fuzz target.
+    if [[ $FUZZING_ENGINE != "none" ]]; then
+      grep "LLVMFuzzerTestOneInput" $fuzz_target > /dev/null 2>&1 || continue
+    fi
+
+    echo "Running $fuzz_target"
+    run_javascript_fuzz_target $fuzz_target &
   else
     # Continue if not a fuzz target.
     if [[ $FUZZING_ENGINE != "none" ]]; then
@@ -387,6 +442,22 @@ elif [[ $FUZZING_LANGUAGE == "jvm" ]]; then
   # Write llvm-cov summary file.
   jacoco_report_converter.py $xml_report $SUMMARY_FILE
 
+  set +e
+elif [[ $FUZZING_LANGUAGE == "javascript" ]]; then
+
+  # From this point on the script does not tolerate any errors.
+  set -e
+
+  json_report=$MERGED_COVERAGE_DIR/coverage.json
+  nyc merge $FUZZERS_COVERAGE_DUMPS_DIR $json_report
+
+  nyc report -t $MERGED_COVERAGE_DIR --report-dir $REPORT_PLATFORM_DIR --reporter=html --reporter=json-summary
+
+  nyc_json_summary_file=$REPORT_PLATFORM_DIR/coverage-summary.json
+
+  # Write llvm-cov summary file.
+  nyc_report_converter.py $nyc_json_summary_file $SUMMARY_FILE
+
   set +e
 else
 

infra/base-images/base-runner/install_javascript.sh

@@ -19,3 +19,6 @@ apt-get update && apt-get install -y curl
 
 curl -fsSL https://deb.nodesource.com/setup_19.x | bash -
 apt-get update && apt-get install -y nodejs
+
+# Install latest versions of nyc for source-based coverage reporting
+npm install --global nyc

infra/base-images/base-runner/nyc_report_converter.py

@@ -0,0 +1,80 @@
+#!/usr/bin/env python3
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+################################################################################
+"""Helper script for creating a llvm-cov style JSON summary from a nyc
+JSON summary."""
+import json
+import sys
+
+
+def convert(nyc_json_summary):
+  """Turns a nyc JSON report into a llvm-cov JSON summary."""
+  summary = {
+      'type':
+          'oss-fuzz.javascript.coverage.json.export',
+      'version':
+          '1.0.0',
+      'data': [{
+          'totals':
+              file_summary(nyc_json_summary['total']),
+          'files': [{
+              'filename': src_file,
+              'summary': file_summary(nyc_json_summary[src_file])
+          } for src_file in nyc_json_summary if src_file != 'total'],
+      }],
+  }
+
+  return json.dumps(summary)
+
+
+def file_summary(nyc_file_summary):
+  """Returns a summary for a given file in the nyc JSON summary report."""
+  return {
+      'functions': element_summary(nyc_file_summary['functions']),
+      'lines': element_summary(nyc_file_summary['lines']),
+      'regions': element_summary(nyc_file_summary['branches'])
+  }
+
+
+def element_summary(element):
+  """Returns a summary of a coverage element in the nyc JSON summary
+  of the file"""
+  return {
+      'count': element['total'],
+      'covered': element['covered'],
+      'notcovered': element['total'] - element['covered'] - element['skipped'],
+      'percent': element['pct'] if element['pct'] != 'Unknown' else 0
+  }
+
+
+def main():
+  """Produces a llvm-cov style JSON summary from a nyc JSON summary."""
+  if len(sys.argv) != 3:
+    sys.stderr.write('Usage: %s <path_to_nyc_json_summary> <out_path_json>\n' %
+                     sys.argv[0])
+    return 1
+
+  with open(sys.argv[1], 'r') as nyc_json_summary_file:
+    nyc_json_summary = json.load(nyc_json_summary_file)
+  json_summary = convert(nyc_json_summary)
+  with open(sys.argv[2], 'w') as json_output_file:
+    json_output_file.write(json_summary)
+
+  return 0
+
+
+if __name__ == '__main__':
+  sys.exit(main())

infra/constants.py

@@ -32,7 +32,7 @@ LANGUAGES = [
     'swift',
 ]
 LANGUAGES_WITH_COVERAGE_SUPPORT = [
-    'c', 'c++', 'go', 'jvm', 'python', 'rust', 'swift'
+    'c', 'c++', 'go', 'jvm', 'python', 'rust', 'swift', 'javascript'
 ]
 SANITIZERS = [
     'address',