diff --git a/infra/base-images/base-builder/compile_javascript_fuzzer b/infra/base-images/base-builder/compile_javascript_fuzzer index 5082de7ec..abb325e5a 100755 --- a/infra/base-images/base-builder/compile_javascript_fuzzer +++ b/infra/base-images/base-builder/compile_javascript_fuzzer @@ -35,6 +35,6 @@ fuzzer_basename=$(basename -s .js $fuzz_target) echo "#!/bin/bash # LLVMFuzzerTestOneInput so that the wrapper script is recognized as a fuzz target for 'check_build'. this_dir=\$(dirname \"\$0\") -\$this_dir/$project/node_modules/@jazzer.js/core/dist/cli.js $project/$fuzz_target $jazzerjs_args -- \$@" > $OUT/$fuzzer_basename +\$this_dir/$project/node_modules/@jazzer.js/core/dist/cli.js $project/$fuzz_target $jazzerjs_args \$JAZZERJS_EXTRA_ARGS -- \$@" > $OUT/$fuzzer_basename chmod +x $OUT/$fuzzer_basename diff --git a/infra/base-images/base-runner/Dockerfile b/infra/base-images/base-runner/Dockerfile index 963d524b2..32f3b3d03 100755 --- a/infra/base-images/base-runner/Dockerfile +++ b/infra/base-images/base-runner/Dockerfile @@ -91,6 +91,7 @@ COPY bad_build_check \ coverage_helper \ download_corpus \ jacoco_report_converter.py \ + nyc_report_converter.py \ rcfilt \ reproduce \ run_fuzzer \ diff --git a/infra/base-images/base-runner/coverage b/infra/base-images/base-runner/coverage index 2e509cf3c..48a5775b5 100755 --- a/infra/base-images/base-runner/coverage +++ b/infra/base-images/base-runner/coverage @@ -31,6 +31,8 @@ fi COVERAGE_OUTPUT_DIR=${COVERAGE_OUTPUT_DIR:-$OUT} DUMPS_DIR="$COVERAGE_OUTPUT_DIR/dumps" +FUZZERS_COVERAGE_DUMPS_DIR="$DUMPS_DIR/fuzzers_coverage" +MERGED_COVERAGE_DIR="$COVERAGE_OUTPUT_DIR/merged_coverage" FUZZER_STATS_DIR="$COVERAGE_OUTPUT_DIR/fuzzer_stats" TEXTCOV_REPORT_DIR="$COVERAGE_OUTPUT_DIR/textcov_reports" LOGS_DIR="$COVERAGE_OUTPUT_DIR/logs" @@ -40,7 +42,7 @@ PLATFORM=linux REPORT_PLATFORM_DIR="$COVERAGE_OUTPUT_DIR/report/$PLATFORM" for directory in $DUMPS_DIR $FUZZER_STATS_DIR $LOGS_DIR $REPORT_ROOT_DIR $TEXTCOV_REPORT_DIR\ - $REPORT_PLATFORM_DIR $REPORT_BY_TARGET_ROOT_DIR; do + $REPORT_PLATFORM_DIR $REPORT_BY_TARGET_ROOT_DIR $FUZZERS_COVERAGE_DUMPS_DIR $MERGED_COVERAGE_DIR; do rm -rf $directory mkdir -p $directory done @@ -222,6 +224,51 @@ function run_java_fuzz_target { jacoco_report_converter.py $xml_report $summary_file } +function run_javascript_fuzz_target { + local target=$1 + local corpus_real="$CORPUS_DIR/${target}" + + # -merge=1 requires an output directory, create a new, empty dir for that. + local corpus_dummy="$OUT/dummy_corpus_dir_for_${target}" + rm -rf $corpus_dummy && mkdir -p $corpus_dummy + + # IstanbulJS currently does not work when the tested program creates + # subprocesses. For this reason, we first minimize the corpus removing + # any crashing inputs so that we can report source-based code coverage + # with a single sweep over the minimized corpus + local merge_args="-merge=1 -timeout=100 $corpus_dummy $corpus_real" + timeout $TIMEOUT $OUT/$target $merge_args &> $LOGS_DIR/$target.log + + # nyc saves the coverage reports in a directory with the default name "coverage" + local coverage_dir="$DUMPS_DIR/coverage_dir_for_${target}" + rm -rf $coverage_dir && mkdir -p $coverage_dir + + local nyc_json_coverage_file="$coverage_dir/coverage-final.json" + local nyc_json_summary_file="$coverage_dir/coverage-summary.json" + + local args="-runs=0 $corpus_dummy" + local jazzerjs_args="--coverage --coverageDirectory $coverage_dir --coverageReporters json --coverageReporters json-summary" + + JAZZERJS_EXTRA_ARGS=$jazzerjs_args $OUT/$target $args &> $LOGS_DIR/$target.log + + if (( $? != 0 )); then + echo "Error occured while running $target:" + cat $LOGS_DIR/$target.log + fi + + if [ ! -s $nyc_json_coverage_file ]; then + # Skip fuzz targets that failed to produce coverage-final.json file. + echo "$target failed to produce coverage-final.json file." + return 0 + fi + + cp $nyc_json_coverage_file $FUZZERS_COVERAGE_DUMPS_DIR/$target.json + + local summary_file="$FUZZER_STATS_DIR/$target.json" + + nyc_report_converter.py $nyc_json_summary_file $summary_file +} + function generate_html { local profdata=$1 local shared_libraries=$2 @@ -265,6 +312,14 @@ for fuzz_target in $FUZZ_TARGETS; do echo "Running $fuzz_target" run_java_fuzz_target $fuzz_target & + elif [[ $FUZZING_LANGUAGE == "javascript" ]]; then + # Continue if not a fuzz target. + if [[ $FUZZING_ENGINE != "none" ]]; then + grep "LLVMFuzzerTestOneInput" $fuzz_target > /dev/null 2>&1 || continue + fi + + echo "Running $fuzz_target" + run_javascript_fuzz_target $fuzz_target & else # Continue if not a fuzz target. if [[ $FUZZING_ENGINE != "none" ]]; then @@ -387,6 +442,22 @@ elif [[ $FUZZING_LANGUAGE == "jvm" ]]; then # Write llvm-cov summary file. jacoco_report_converter.py $xml_report $SUMMARY_FILE + set +e +elif [[ $FUZZING_LANGUAGE == "javascript" ]]; then + + # From this point on the script does not tolerate any errors. + set -e + + json_report=$MERGED_COVERAGE_DIR/coverage.json + nyc merge $FUZZERS_COVERAGE_DUMPS_DIR $json_report + + nyc report -t $MERGED_COVERAGE_DIR --report-dir $REPORT_PLATFORM_DIR --reporter=html --reporter=json-summary + + nyc_json_summary_file=$REPORT_PLATFORM_DIR/coverage-summary.json + + # Write llvm-cov summary file. + nyc_report_converter.py $nyc_json_summary_file $SUMMARY_FILE + set +e else diff --git a/infra/base-images/base-runner/install_javascript.sh b/infra/base-images/base-runner/install_javascript.sh index 7985df71a..2d2bea7cb 100755 --- a/infra/base-images/base-runner/install_javascript.sh +++ b/infra/base-images/base-runner/install_javascript.sh @@ -19,3 +19,6 @@ apt-get update && apt-get install -y curl curl -fsSL https://deb.nodesource.com/setup_19.x | bash - apt-get update && apt-get install -y nodejs + +# Install latest versions of nyc for source-based coverage reporting +npm install --global nyc diff --git a/infra/base-images/base-runner/nyc_report_converter.py b/infra/base-images/base-runner/nyc_report_converter.py new file mode 100755 index 000000000..53044754c --- /dev/null +++ b/infra/base-images/base-runner/nyc_report_converter.py @@ -0,0 +1,80 @@ +#!/usr/bin/env python3 +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +################################################################################ +"""Helper script for creating a llvm-cov style JSON summary from a nyc +JSON summary.""" +import json +import sys + + +def convert(nyc_json_summary): + """Turns a nyc JSON report into a llvm-cov JSON summary.""" + summary = { + 'type': + 'oss-fuzz.javascript.coverage.json.export', + 'version': + '1.0.0', + 'data': [{ + 'totals': + file_summary(nyc_json_summary['total']), + 'files': [{ + 'filename': src_file, + 'summary': file_summary(nyc_json_summary[src_file]) + } for src_file in nyc_json_summary if src_file != 'total'], + }], + } + + return json.dumps(summary) + + +def file_summary(nyc_file_summary): + """Returns a summary for a given file in the nyc JSON summary report.""" + return { + 'functions': element_summary(nyc_file_summary['functions']), + 'lines': element_summary(nyc_file_summary['lines']), + 'regions': element_summary(nyc_file_summary['branches']) + } + + +def element_summary(element): + """Returns a summary of a coverage element in the nyc JSON summary + of the file""" + return { + 'count': element['total'], + 'covered': element['covered'], + 'notcovered': element['total'] - element['covered'] - element['skipped'], + 'percent': element['pct'] if element['pct'] != 'Unknown' else 0 + } + + +def main(): + """Produces a llvm-cov style JSON summary from a nyc JSON summary.""" + if len(sys.argv) != 3: + sys.stderr.write('Usage: %s \n' % + sys.argv[0]) + return 1 + + with open(sys.argv[1], 'r') as nyc_json_summary_file: + nyc_json_summary = json.load(nyc_json_summary_file) + json_summary = convert(nyc_json_summary) + with open(sys.argv[2], 'w') as json_output_file: + json_output_file.write(json_summary) + + return 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/infra/constants.py b/infra/constants.py index db4953ecc..e085700c9 100644 --- a/infra/constants.py +++ b/infra/constants.py @@ -32,7 +32,7 @@ LANGUAGES = [ 'swift', ] LANGUAGES_WITH_COVERAGE_SUPPORT = [ - 'c', 'c++', 'go', 'jvm', 'python', 'rust', 'swift' + 'c', 'c++', 'go', 'jvm', 'python', 'rust', 'swift', 'javascript' ] SANITIZERS = [ 'address',