mirror of https://github.com/google/oss-fuzz.git
[test_all] Rewrite in Python (#4769)
Rewrite test_all in python. Bash is quite annoying to write and test. One issue with bash is it is even worse than Python for parallelism (which may be causing #4707). Rewrite test_all in python and optimize base-runner/Dockerfile for fast development. Also, combine some docker layers.
This commit is contained in:
jonathanmetzman
GitHub
parent
8169d41e43
commit
c8d2319aa8
|
|
@ -20,12 +20,14 @@ FROM gcr.io/oss-fuzz-base/base-clang AS base-clang
|
|||
FROM gcr.io/oss-fuzz-base/base-image
|
||||
|
||||
# Copy the binaries needed for code coverage and crash symbolization.
|
||||
COPY --from=base-clang /usr/local/bin/llvm-cov /usr/local/bin/
|
||||
COPY --from=base-clang /usr/local/bin/llvm-profdata /usr/local/bin/
|
||||
COPY --from=base-clang /usr/local/bin/llvm-symbolizer /usr/local/bin/
|
||||
COPY --from=base-clang /usr/local/bin/llvm-cov \
|
||||
/usr/local/bin/llvm-profdata \
|
||||
/usr/local/bin/llvm-symbolizer \
|
||||
/usr/local/bin/
|
||||
|
||||
RUN apt-get update
|
||||
RUN apt-get install -y \
|
||||
# TODO(metzman): Install libc6-i386 lib32gcc1 instead of libc6-dev-i386 for
|
||||
# consistency with ClusterFuzz image and to reduce size.
|
||||
RUN apt-get update && apt-get install -y \
|
||||
binutils \
|
||||
file \
|
||||
fonts-dejavu \
|
||||
|
|
@ -37,24 +39,8 @@ RUN apt-get install -y \
|
|||
wget \
|
||||
zip
|
||||
|
||||
RUN git clone https://chromium.googlesource.com/chromium/src/tools/code_coverage /opt/code_coverage
|
||||
RUN pip3 install -r /opt/code_coverage/requirements.txt
|
||||
|
||||
COPY bad_build_check \
|
||||
collect_dft \
|
||||
coverage \
|
||||
coverage_helper \
|
||||
dataflow_tracer.py \
|
||||
download_corpus \
|
||||
minijail0 \
|
||||
reproduce \
|
||||
run_fuzzer \
|
||||
run_minijail \
|
||||
parse_options.py \
|
||||
targets_list \
|
||||
test_all \
|
||||
test_one \
|
||||
/usr/local/bin/
|
||||
RUN git clone https://chromium.googlesource.com/chromium/src/tools/code_coverage /opt/code_coverage && \
|
||||
pip3 install -r /opt/code_coverage/requirements.txt
|
||||
|
||||
# Default environment options for various sanitizers.
|
||||
# Note that these match the settings used in ClusterFuzz and
|
||||
|
|
@ -81,3 +67,20 @@ ENV PATH $PATH:/root/.go/bin:$GOPATH/bin
|
|||
|
||||
# Set up Golang coverage modules.
|
||||
RUN go get github.com/google/oss-fuzz/infra/go/coverage/...
|
||||
|
||||
# Do this last to make developing these files easier/faster due to caching.
|
||||
COPY bad_build_check \
|
||||
collect_dft \
|
||||
coverage \
|
||||
coverage_helper \
|
||||
dataflow_tracer.py \
|
||||
download_corpus \
|
||||
minijail0 \
|
||||
reproduce \
|
||||
run_fuzzer \
|
||||
run_minijail \
|
||||
parse_options.py \
|
||||
targets_list \
|
||||
test_all.py \
|
||||
test_one \
|
||||
/usr/local/bin/
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ docker run -ti gcr.io/oss-fuzz-base/base-runner <command> <args>
|
|||
|---------|-------------|
|
||||
| `reproduce <fuzzer_name> <fuzzer_options>` | build all fuzz targets and run specified one with testcase `/testcase` and given options.
|
||||
| `run_fuzzer <fuzzer_name> <fuzzer_options>` | runs specified fuzzer combining options with `.options` file |
|
||||
| `test_all` | runs every binary in `/out` as a fuzzer for a while to ensure it works. |
|
||||
| `test_all.py` | runs every binary in `/out` as a fuzzer for a while to ensure it works. |
|
||||
|
||||
# Examples
|
||||
|
||||
|
|
|
|||
|
|
@ -1,147 +0,0 @@
|
|||
#!/bin/bash -u
|
||||
# Copyright 2016 Google Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
################################################################################
|
||||
|
||||
# Percentage threshold that needs to be reached for marking a build as failed.
|
||||
ALLOWED_BROKEN_TARGETS_PERCENTAGE=${ALLOWED_BROKEN_TARGETS_PERCENTAGE:-10}
|
||||
|
||||
# Test all fuzz targets in the $OUT/ dir.
|
||||
TOTAL_TARGETS_COUNT=0
|
||||
|
||||
# Number of CPUs available, this is needed for running tests in parallel.
|
||||
NPROC=$(nproc)
|
||||
|
||||
# Directories where bad build check results will be written to.
|
||||
VALID_TARGETS_DIR="/tmp/valid_fuzz_targets"
|
||||
BROKEN_TARGETS_DIR="/tmp/broken_fuzz_targets"
|
||||
rm -rf $VALID_TARGETS_DIR
|
||||
rm -rf $BROKEN_TARGETS_DIR
|
||||
mkdir $VALID_TARGETS_DIR
|
||||
mkdir $BROKEN_TARGETS_DIR
|
||||
|
||||
# Move the directory the fuzzer is located in to somewhere that doesn't exist
|
||||
# on the builder to make it more likely that hardcoding /out fails here (since
|
||||
# it will fail on ClusterFuzz).
|
||||
TMP_FUZZER_DIR=/tmp/not-out
|
||||
rm -rf $TMP_FUZZER_DIR
|
||||
mkdir $TMP_FUZZER_DIR
|
||||
# Move contents of $OUT/ into $TMP_FUZZER_DIR. We can't move the directory
|
||||
# itself because it is a mount.
|
||||
mv $OUT/* $TMP_FUZZER_DIR
|
||||
INITIAL_OUT=$OUT
|
||||
export OUT=$TMP_FUZZER_DIR
|
||||
|
||||
|
||||
# Main loop that iterates through all fuzz targets and runs the check.
|
||||
for FUZZER_BINARY in $(find $TMP_FUZZER_DIR -maxdepth 1 -executable -type f); do
|
||||
if [ "$FUZZING_LANGUAGE" != "python" ]; then
|
||||
if file "$FUZZER_BINARY" | grep -v ELF > /dev/null 2>&1; then
|
||||
continue
|
||||
fi
|
||||
fi
|
||||
|
||||
# Continue if not a fuzz target.
|
||||
if [[ $FUZZING_ENGINE != "none" ]]; then
|
||||
grep "LLVMFuzzerTestOneInput" $FUZZER_BINARY > /dev/null 2>&1 || continue
|
||||
fi
|
||||
|
||||
FUZZER=$(basename $FUZZER_BINARY)
|
||||
if [[ "$FUZZER" == afl-* ]]; then
|
||||
continue
|
||||
fi
|
||||
|
||||
echo "INFO: performing bad build checks for $FUZZER_BINARY."
|
||||
|
||||
LOG_PATH_FOR_BROKEN_TARGET="${BROKEN_TARGETS_DIR}/${FUZZER}"
|
||||
|
||||
# Launch bad build check process in the background. Ignore the exit codes, as
|
||||
# we check the percentage of broken fuzz targets after running all the checks.
|
||||
bad_build_check $FUZZER_BINARY &> $LOG_PATH_FOR_BROKEN_TARGET &
|
||||
|
||||
# Count total number of fuzz targets being tested.
|
||||
TOTAL_TARGETS_COUNT=$[$TOTAL_TARGETS_COUNT+1]
|
||||
|
||||
# Do not spawn more processes than the number of CPUs available.
|
||||
n_child_proc=$(jobs -rp | wc -l)
|
||||
while [ "$n_child_proc" -eq "$NPROC" ]; do
|
||||
sleep 4
|
||||
n_child_proc=$(jobs -rp | wc -l)
|
||||
done
|
||||
done
|
||||
|
||||
# Wait for background processes to finish.
|
||||
wait
|
||||
|
||||
# Restore OUT
|
||||
export OUT=$INITIAL_OUT
|
||||
mv $TMP_FUZZER_DIR/* $OUT
|
||||
|
||||
# Sanity check in case there are no fuzz targets in the $OUT/ dir.
|
||||
if [ "$TOTAL_TARGETS_COUNT" -eq "0" ]; then
|
||||
echo "ERROR: no fuzzers found in $OUT/"
|
||||
ls -al $OUT
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# An empty log file indicated that corresponding fuzz target is not broken.
|
||||
find $BROKEN_TARGETS_DIR -empty -exec mv {} $VALID_TARGETS_DIR \;
|
||||
|
||||
# Calculate number of valid and broken fuzz targets.
|
||||
VALID_TARGETS_COUNT=$(ls $VALID_TARGETS_DIR | wc -l)
|
||||
BROKEN_TARGETS_COUNT=$(ls $BROKEN_TARGETS_DIR | wc -l)
|
||||
|
||||
# Sanity check to make sure that bad build check doesn't skip any fuzz target.
|
||||
if [ "$TOTAL_TARGETS_COUNT" -ne "$[$VALID_TARGETS_COUNT+$BROKEN_TARGETS_COUNT]" ]; then
|
||||
echo "ERROR: bad_build_check seems to have a bug, total number of fuzz" \
|
||||
"does not match number of fuzz targets tested."
|
||||
echo "Total fuzz targets ($TOTAL_TARGETS_COUNT):"
|
||||
ls -al $OUT
|
||||
echo "Valid fuzz targets ($VALID_TARGETS_COUNT):"
|
||||
ls -al $VALID_TARGETS_DIR
|
||||
echo "Total fuzz targets ($BROKEN_TARGETS_COUNT):"
|
||||
ls -al $BROKEN_TARGETS_DIR
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Build info about all broken fuzz targets (if any).
|
||||
if [ "$BROKEN_TARGETS_COUNT" -gt "0" ]; then
|
||||
echo "Broken fuzz targets ($BROKEN_TARGETS_COUNT):"
|
||||
for target in $(ls $BROKEN_TARGETS_DIR); do
|
||||
echo "${target}:"
|
||||
cat ${BROKEN_TARGETS_DIR}/${target}
|
||||
done
|
||||
fi
|
||||
|
||||
# Calculate the percentage of broken fuzz targets and make the finel decision.
|
||||
BROKEN_TARGETS_PERCENTAGE=$[$BROKEN_TARGETS_COUNT*100/$TOTAL_TARGETS_COUNT]
|
||||
|
||||
|
||||
if [ "$BROKEN_TARGETS_PERCENTAGE" -gt "$ALLOWED_BROKEN_TARGETS_PERCENTAGE" ]; then
|
||||
echo "ERROR: $BROKEN_TARGETS_PERCENTAGE% of fuzz targets seem to be broken." \
|
||||
"See the list above for a detailed information."
|
||||
|
||||
# TODO: figure out how to not fail the "special" cases handled below. Those
|
||||
# are from "example" and "c-ares" projects and are too small targets to pass.
|
||||
if [ "$(ls $OUT/do_stuff_fuzzer $OUT/ares_*_fuzzer $OUT/checksum_fuzzer $OUT/fuzz_dump $OUT/fuzz_keyring $OUT/xmltest $OUT/fuzz_compression_sas_rle 2>/dev/null | wc -l)" -gt "0" ]; then
|
||||
exit 0
|
||||
fi
|
||||
|
||||
exit 1
|
||||
else
|
||||
echo "$TOTAL_TARGETS_COUNT fuzzers total, $BROKEN_TARGETS_COUNT seem to be" \
|
||||
"broken ($BROKEN_TARGETS_PERCENTAGE%)."
|
||||
exit 0
|
||||
fi
|
||||
|
|
@ -0,0 +1,206 @@
|
|||
#!/usr/bin/env python3
|
||||
# Copyright 2020 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
################################################################################
|
||||
"""Does bad_build_check on all fuzz targets in $OUT."""
|
||||
|
||||
import contextlib
|
||||
import multiprocessing
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import stat
|
||||
import sys
|
||||
|
||||
TMP_FUZZER_DIR = '/tmp/not-out'
|
||||
|
||||
EXECUTABLE = stat.S_IEXEC | stat.S_IXGRP | stat.S_IXOTH
|
||||
|
||||
IGNORED_TARGETS = [
|
||||
r'do_stuff_fuzzer', r'checksum_fuzzer', r'fuzz_dump', r'fuzz_keyring',
|
||||
r'xmltest', r'fuzz_compression_sas_rle', r'ares_*_fuzzer'
|
||||
]
|
||||
|
||||
IGNORED_TARGETS_RE = re.compile('^' + r'$|^'.join(IGNORED_TARGETS) + '$')
|
||||
|
||||
|
||||
def recreate_directory(directory):
|
||||
"""Creates |directory|. If it already exists than deletes it first before
|
||||
creating."""
|
||||
if os.path.exists(directory):
|
||||
shutil.rmtree(directory)
|
||||
os.mkdir(directory)
|
||||
|
||||
|
||||
def move_directory_contents(src_directory, dst_directory):
|
||||
"""Moves contents of |src_directory| to |dst_directory|."""
|
||||
src_files = os.listdir(src_directory)
|
||||
for filename in src_files:
|
||||
src_path = os.path.join(src_directory, filename)
|
||||
shutil.move(src_path, dst_directory)
|
||||
|
||||
|
||||
def is_elf(filepath):
|
||||
"""Returns True if |filepath| is an ELF file."""
|
||||
result = subprocess.run(['file', filepath],
|
||||
stdout=subprocess.PIPE,
|
||||
check=False)
|
||||
return b'ELF' in result.stdout
|
||||
|
||||
|
||||
def find_fuzz_targets(directory, fuzzing_language):
|
||||
"""Returns paths to fuzz targets in |directory|."""
|
||||
# TODO(https://github.com/google/oss-fuzz/issues/4585): Use libClusterFuzz for
|
||||
# this.
|
||||
fuzz_targets = []
|
||||
for filename in os.listdir(directory):
|
||||
path = os.path.join(directory, filename)
|
||||
if filename.startswith('afl-'):
|
||||
continue
|
||||
if not os.path.isfile(path):
|
||||
continue
|
||||
if not os.stat(path).st_mode & EXECUTABLE:
|
||||
continue
|
||||
with open(path, 'rb') as file_handle:
|
||||
binary_contents = file_handle.read()
|
||||
if b'LLVMFuzzerTestOneInput' not in binary_contents:
|
||||
continue
|
||||
if fuzzing_language != 'python' and not is_elf(path):
|
||||
continue
|
||||
fuzz_targets.append(path)
|
||||
return fuzz_targets
|
||||
|
||||
|
||||
def do_bad_build_check(fuzz_target):
|
||||
"""Runs bad_build_check on |fuzz_target|. Returns a
|
||||
Subprocess.ProcessResult."""
|
||||
print('INFO: performing bad build checks for', fuzz_target)
|
||||
command = ['bad_build_check', fuzz_target]
|
||||
return subprocess.run(command,
|
||||
stderr=subprocess.PIPE,
|
||||
stdout=subprocess.PIPE,
|
||||
check=False)
|
||||
|
||||
|
||||
def get_broken_fuzz_targets(bad_build_results, fuzz_targets):
|
||||
"""Returns a list of broken fuzz targets and their process results in
|
||||
|fuzz_targets| where each item in |bad_build_results| is the result of
|
||||
bad_build_check on the corresponding element in |fuzz_targets|."""
|
||||
broken = []
|
||||
for result, fuzz_target in zip(bad_build_results, fuzz_targets):
|
||||
if result.returncode != 0:
|
||||
broken.append((fuzz_target, result))
|
||||
return broken
|
||||
|
||||
|
||||
def has_ignored_targets(out_dir):
|
||||
"""Returns True if |out_dir| has any fuzz targets we are supposed to ignore
|
||||
bad build checks of."""
|
||||
out_files = set(os.listdir(out_dir))
|
||||
for filename in out_files:
|
||||
if re.match(IGNORED_TARGETS_RE, filename):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def use_different_out_dir():
|
||||
"""Context manager that moves OUT to TMP_FUZZER_DIR. This is useful for
|
||||
catching hardcoding. Note that this sets the environment variable OUT and
|
||||
therefore must be run before multiprocessing.Pool is created. Resets OUT at
|
||||
the end."""
|
||||
# Use a fake OUT directory to catch path hardcoding that breaks on
|
||||
# ClusterFuzz.
|
||||
out = os.getenv('OUT')
|
||||
initial_out = out
|
||||
recreate_directory(TMP_FUZZER_DIR)
|
||||
out = TMP_FUZZER_DIR
|
||||
# Set this so that run_fuzzer which is called by bad_build_check works
|
||||
# properly.
|
||||
os.environ['OUT'] = out
|
||||
# We move the contents of the directory because we can't move the
|
||||
# directory itself because it is a mount.
|
||||
move_directory_contents(initial_out, out)
|
||||
try:
|
||||
yield out
|
||||
finally:
|
||||
move_directory_contents(out, initial_out)
|
||||
shutil.rmtree(out)
|
||||
os.environ['OUT'] = initial_out
|
||||
|
||||
|
||||
def test_all_outside_out(fuzzing_language, allowed_broken_targets_percentage):
|
||||
"""Wrapper around test_all that changes OUT and returns the result."""
|
||||
with use_different_out_dir() as out:
|
||||
return test_all(out, fuzzing_language, allowed_broken_targets_percentage)
|
||||
|
||||
|
||||
def test_all(out, fuzzing_language, allowed_broken_targets_percentage):
|
||||
"""Do bad_build_check on all fuzz targets."""
|
||||
# TODO(metzman): Refactor so that we can convert test_one to python.
|
||||
fuzz_targets = find_fuzz_targets(out, fuzzing_language)
|
||||
pool = multiprocessing.Pool()
|
||||
bad_build_results = pool.map(do_bad_build_check, fuzz_targets)
|
||||
broken_targets = get_broken_fuzz_targets(bad_build_results, fuzz_targets)
|
||||
broken_targets_count = len(broken_targets)
|
||||
if not broken_targets_count:
|
||||
return True
|
||||
|
||||
print('Broken fuzz targets', broken_targets_count)
|
||||
total_targets_count = len(fuzz_targets)
|
||||
broken_targets_percentage = 100 * broken_targets_count / total_targets_count
|
||||
for broken_target, result in broken_targets:
|
||||
print(broken_target)
|
||||
# Use write because we can't print binary strings.
|
||||
sys.stdout.buffer.write(result.stdout + result.stderr + b'\n')
|
||||
|
||||
if broken_targets_percentage > allowed_broken_targets_percentage:
|
||||
print('ERROR: {broken_targets_percentage}% of fuzz targets seem to be '
|
||||
'broken. See the list above for a detailed information.'.format(
|
||||
broken_targets_percentage=broken_targets_percentage))
|
||||
if has_ignored_targets(out):
|
||||
print('Build check automatically passing because of ignored targets.')
|
||||
return True
|
||||
return False
|
||||
print('{total_targets_count} fuzzers total, {broken_targets_count} '
|
||||
'seem to be broken ({broken_targets_percentage}%).'.format(
|
||||
total_targets_count=total_targets_count,
|
||||
broken_targets_count=broken_targets_count,
|
||||
broken_targets_percentage=broken_targets_percentage))
|
||||
return True
|
||||
|
||||
|
||||
def get_allowed_broken_targets_percentage():
|
||||
"""Returns the value of the environment value
|
||||
'ALLOWED_BROKEN_TARGETS_PERCENTAGE' as an int or returns a reasonable
|
||||
default."""
|
||||
return int(os.getenv('ALLOWED_BROKEN_TARGETS_PERCENTAGE', '10'))
|
||||
|
||||
|
||||
def main():
|
||||
"""Does bad_build_check on all fuzz targets in parallel. Returns 0 on success.
|
||||
Returns 1 on failure."""
|
||||
# Set these environment variables here so that stdout
|
||||
fuzzing_language = os.getenv('FUZZING_LANGUAGE')
|
||||
allowed_broken_targets_percentage = get_allowed_broken_targets_percentage()
|
||||
if not test_all_outside_out(fuzzing_language,
|
||||
allowed_broken_targets_percentage):
|
||||
return 1
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
|
|
@ -314,7 +314,7 @@ def check_fuzzer_build(out_dir, sanitizer='address'):
|
|||
command += ['-e', 'OUT=' + out_dir, '--volumes-from', container]
|
||||
else:
|
||||
command += ['-v', '%s:/out' % out_dir]
|
||||
command.extend(['-t', 'gcr.io/oss-fuzz-base/base-runner', 'test_all'])
|
||||
command.extend(['-t', 'gcr.io/oss-fuzz-base/base-runner', 'test_all.py'])
|
||||
exit_code = helper.docker_run(command)
|
||||
if exit_code:
|
||||
logging.error('Check fuzzer build failed.')
|
||||
|
|
|
|||
|
|
@ -277,7 +277,8 @@ def get_dockerfile_path(project_name):
|
|||
|
||||
|
||||
def _get_corpus_dir(project_name=''):
|
||||
"""Creates and returns path to /corpus directory for the given project (if specified)."""
|
||||
"""Creates and returns path to /corpus directory for the given project (if
|
||||
specified)."""
|
||||
directory = os.path.join(BUILD_DIR, 'corpus', project_name)
|
||||
os.makedirs(directory, exist_ok=True)
|
||||
|
||||
|
|
@ -285,7 +286,8 @@ def _get_corpus_dir(project_name=''):
|
|||
|
||||
|
||||
def _get_output_dir(project_name=''):
|
||||
"""Creates and returns path to /out directory for the given project (if specified)."""
|
||||
"""Creates and returns path to /out directory for the given project (if
|
||||
specified)."""
|
||||
directory = os.path.join(BUILD_DIR, 'out', project_name)
|
||||
os.makedirs(directory, exist_ok=True)
|
||||
|
||||
|
|
@ -293,7 +295,8 @@ def _get_output_dir(project_name=''):
|
|||
|
||||
|
||||
def _get_work_dir(project_name=''):
|
||||
"""Creates and returns path to /work directory for the given project (if specified)."""
|
||||
"""Creates and returns path to /work directory for the given project (if
|
||||
specified)."""
|
||||
directory = os.path.join(BUILD_DIR, 'work', project_name)
|
||||
os.makedirs(directory, exist_ok=True)
|
||||
|
||||
|
|
@ -627,7 +630,7 @@ def check_build(args):
|
|||
if args.fuzzer_name:
|
||||
run_args += ['test_one', os.path.join('/out', args.fuzzer_name)]
|
||||
else:
|
||||
run_args.append('test_all')
|
||||
run_args.append('test_all.py')
|
||||
|
||||
exit_code = docker_run(run_args)
|
||||
if exit_code == 0:
|
||||
|
|
|
|||
Loading…
Reference in New Issue