Rooba
/
oss-fuzz
mirror of https://github.com/google/oss-fuzz.git
1
1
Fork 0
Code Issues Packages Projects Releases Wiki Activity

[Infra] Add tests to presubmit (#3405) 

Adds unit/integration tests to travis presubmit.
This commit is contained in:
Leo Neat 2020-02-21 08:47:13 -08:00 committed by GitHub
parent dd5ad3b20e
commit 1522a7428c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 107 additions and 114 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.travis.yml
View File

@ -10,12 +10,15 @@ before_install:
install:
- pip install -r infra/travis/requirements.txt
matrix:
include:
- name: "presubmit"
install:
- pip install -r infra/dev-requirements.txt
script: ./infra/presubmit.py
- name: "tests"
script: sudo ./infra/presubmit.py test
- name: "libfuzzer address x86_64"
env:
- TRAVIS_ENGINE=libfuzzer

1
infra/bisector_test.py
View File

@ -29,6 +29,7 @@ import test_repos
TEST_DIR_PATH = os.path.dirname(os.path.realpath(__file__))
@unittest.skip('Test is too long to be run with presubmit.')
class BisectIntegrationTests(unittest.TestCase):
"""Class to test the functionality of bisection method."""

102
infra/cifuzz/cifuzz_test.py
View File

@ -17,6 +17,7 @@
"""
import os
import shutil
import sys
import tempfile
import unittest
@ -28,9 +29,16 @@ import cifuzz
import fuzz_target
# NOTE: This integration test relies on
# https://github.com/google/oss-fuzz/tree/master/projects/example project
# https://github.com/google/oss-fuzz/tree/master/projects/example project.
EXAMPLE_PROJECT = 'example'
# Location of files used for testing.
TEST_FILES_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)),
'test_files')
# An example fuzzer that triggers an error.
EXAMPLE_FUZZER = 'do_stuff_fuzzer'
class BuildFuzzersIntegrationTest(unittest.TestCase):
"""Test build_fuzzers function in the utils module."""
@ -46,7 +54,7 @@ class BuildFuzzersIntegrationTest(unittest.TestCase):
'oss-fuzz',
tmp_dir,
commit_sha='0b95fe1039ed7c38fea1f97078316bfc1030c523'))
self.assertTrue(os.path.exists(os.path.join(out_path, 'do_stuff_fuzzer')))
self.assertTrue(os.path.exists(os.path.join(out_path, EXAMPLE_FUZZER)))
def test_valid_pull_request(self):
"""Test building fuzzers with valid pull request."""
@ -58,7 +66,7 @@ class BuildFuzzersIntegrationTest(unittest.TestCase):
'oss-fuzz',
tmp_dir,
pr_ref='refs/pull/1757/merge'))
self.assertTrue(os.path.exists(os.path.join(out_path, 'do_stuff_fuzzer')))
self.assertTrue(os.path.exists(os.path.join(out_path, EXAMPLE_FUZZER)))
def test_invalid_pull_request(self):
"""Test building fuzzers with invalid pull request."""
@ -114,60 +122,53 @@ class BuildFuzzersIntegrationTest(unittest.TestCase):
class RunFuzzersIntegrationTest(unittest.TestCase):
"""Test build_fuzzers function in the cifuzz module."""
def tearDown(self):
"""Remove any existing crashes and test files."""
out_dir = os.path.join(TEST_FILES_PATH, 'out')
for out_file in os.listdir(out_dir):
out_path = os.path.join(out_dir, out_file)
if out_file == EXAMPLE_FUZZER:
continue
if os.path.isdir(out_path):
shutil.rmtree(out_path)
else:
os.remove(out_path)
def test_new_bug_found(self):
"""Test run_fuzzers with a valid build."""
with tempfile.TemporaryDirectory() as tmp_dir:
out_path = os.path.join(tmp_dir, 'out')
os.mkdir(out_path)
self.assertTrue(
cifuzz.build_fuzzers(
EXAMPLE_PROJECT,
'oss-fuzz',
tmp_dir,
commit_sha='0b95fe1039ed7c38fea1f97078316bfc1030c523'))
self.assertTrue(os.path.exists(os.path.join(out_path, 'do_stuff_fuzzer')))
# Setting the first return value to True, then the second to False to
# emulate a bug existing in the current PR but not on the downloaded
# OSS-Fuzz build.
with unittest.mock.patch.object(fuzz_target.FuzzTarget,
'is_reproducible',
side_effect=[True, False]):
run_success, bug_found = cifuzz.run_fuzzers(5, tmp_dir, EXAMPLE_PROJECT)
build_dir = os.path.join(tmp_dir, 'out', 'oss_fuzz_latest')
self.assertTrue(os.path.exists(build_dir))
self.assertNotEqual(0, len(os.listdir(build_dir)))
self.assertTrue(run_success)
self.assertTrue(bug_found)
# Setting the first return value to True, then the second to False to
# emulate a bug existing in the current PR but not on the downloaded
# OSS-Fuzz build.
with unittest.mock.patch.object(fuzz_target.FuzzTarget,
'is_reproducible',
side_effect=[True, False]):
run_success, bug_found = cifuzz.run_fuzzers(100, TEST_FILES_PATH,
EXAMPLE_PROJECT)
build_dir = os.path.join(TEST_FILES_PATH, 'out', 'oss_fuzz_latest')
self.assertTrue(os.path.exists(build_dir))
self.assertNotEqual(0, len(os.listdir(build_dir)))
self.assertTrue(run_success)
self.assertTrue(bug_found)
def test_old_bug_found(self):
"""Test run_fuzzers with a bug found in OSS-Fuzz before."""
with tempfile.TemporaryDirectory() as tmp_dir:
out_path = os.path.join(tmp_dir, 'out')
os.mkdir(out_path)
self.assertTrue(
cifuzz.build_fuzzers(
EXAMPLE_PROJECT,
'oss-fuzz',
tmp_dir,
commit_sha='0b95fe1039ed7c38fea1f97078316bfc1030c523'))
self.assertTrue(os.path.exists(os.path.join(out_path, 'do_stuff_fuzzer')))
with unittest.mock.patch.object(fuzz_target.FuzzTarget,
'is_reproducible',
side_effect=[True, True]):
run_success, bug_found = cifuzz.run_fuzzers(5, tmp_dir, EXAMPLE_PROJECT)
build_dir = os.path.join(tmp_dir, 'out', 'oss_fuzz_latest')
self.assertTrue(os.path.exists(build_dir))
self.assertNotEqual(0, len(os.listdir(build_dir)))
self.assertTrue(run_success)
self.assertFalse(bug_found)
with unittest.mock.patch.object(fuzz_target.FuzzTarget,
'is_reproducible',
side_effect=[True, True]):
run_success, bug_found = cifuzz.run_fuzzers(100, TEST_FILES_PATH,
EXAMPLE_PROJECT)
build_dir = os.path.join(TEST_FILES_PATH, 'out', 'oss_fuzz_latest')
self.assertTrue(os.path.exists(build_dir))
self.assertNotEqual(0, len(os.listdir(build_dir)))
self.assertTrue(run_success)
self.assertFalse(bug_found)
def test_invalid_build(self):
"""Test run_fuzzers with an invalid build."""
with tempfile.TemporaryDirectory() as tmp_dir:
out_path = os.path.join(tmp_dir, 'out')
os.mkdir(out_path)
run_success, bug_found = cifuzz.run_fuzzers(5, tmp_dir, EXAMPLE_PROJECT)
run_success, bug_found = cifuzz.run_fuzzers(100, tmp_dir, EXAMPLE_PROJECT)
self.assertFalse(run_success)
self.assertFalse(bug_found)
@ -182,7 +183,7 @@ class RunFuzzersIntegrationTest(unittest.TestCase):
def test_invalid_out_dir(self):
"""Tests run_fuzzers with an invalid out directory."""
run_success, bug_found = cifuzz.run_fuzzers(5, 'not/a/valid/path',
run_success, bug_found = cifuzz.run_fuzzers(100, 'not/a/valid/path',
EXAMPLE_PROJECT)
self.assertFalse(run_success)
self.assertFalse(bug_found)
@ -193,10 +194,9 @@ class ParseOutputUnitTest(unittest.TestCase):
def test_parse_valid_output(self):
"""Checks that the parse fuzzer output can correctly parse output."""
test_case_path = os.path.join(os.path.dirname(os.path.abspath(__file__)),
'test_files')
test_output_path = os.path.join(test_case_path, 'example_fuzzer_output.txt')
test_summary_path = os.path.join(test_case_path, 'bug_summary_example.txt')
test_output_path = os.path.join(TEST_FILES_PATH,
'example_fuzzer_output.txt')
test_summary_path = os.path.join(TEST_FILES_PATH, 'bug_summary_example.txt')
with tempfile.TemporaryDirectory() as tmp_dir:
with open(test_output_path, 'r') as test_fuzz_output:
cifuzz.parse_fuzzer_output(test_fuzz_output.read(), tmp_dir)

2
infra/cifuzz/fuzz_target.py
View File

@ -113,7 +113,7 @@ class FuzzTarget:
err_str = err.decode('ascii')
test_case = self.get_test_case(err_str)
if not test_case:
logging.error('No test case found in stack trace. %s.', sys.stderr)
logging.error('No test case found in stack trace: %s.', err_str)
return None, None
if self.check_reproducibility_and_regression(test_case):
return test_case, err_str

40
infra/cifuzz/fuzz_target_test.py
View File

@ -42,32 +42,32 @@ class IsReproducibleUnitTest(unittest.TestCase):
def test_with_reproducible(self):
"""Tests that a is_reproducible will return true if crash is detected."""
test_all_success = [(0, 0, 1)] * 10
all_success_mock = unittest.mock.Mock()
all_success_mock.side_effect = test_all_success
utils.execute = all_success_mock
self.assertTrue(
self.test_target.is_reproducible('/fake/path/to/testcase',
'/fake/target'))
self.assertEqual(1, all_success_mock.call_count)
with unittest.mock.patch.object(utils,
'execute',
side_effect=test_all_success) as patch:
self.assertTrue(
self.test_target.is_reproducible('/fake/path/to/testcase',
'/fake/target'))
self.assertEqual(1, patch.call_count)
test_one_success = [(0, 0, 0)] * 9 + [(0, 0, 1)]
one_success_mock = unittest.mock.Mock()
one_success_mock.side_effect = test_one_success
utils.execute = one_success_mock
self.assertTrue(
self.test_target.is_reproducible('/fake/path/to/testcase',
'/fake/target'))
self.assertEqual(10, one_success_mock.call_count)
with unittest.mock.patch.object(utils,
'execute',
side_effect=test_one_success) as patch:
self.assertTrue(
self.test_target.is_reproducible('/fake/path/to/testcase',
'/fake/target'))
self.assertEqual(10, patch.call_count)
def test_with_not_reproducible(self):
"""Tests that a is_reproducible will return False if crash not detected."""
test_all_fail = [(0, 0, 0)] * 10
all_fail_mock = unittest.mock.Mock()
all_fail_mock.side_effect = test_all_fail
utils.execute = all_fail_mock
self.assertFalse(
self.test_target.is_reproducible('/fake/path/to/testcase',
'/fake/target'))
with unittest.mock.patch.object(utils, 'execute',
side_effect=test_all_fail) as patch:
self.assertFalse(
self.test_target.is_reproducible('/fake/path/to/testcase',
'/fake/target'))
self.assertEqual(10, patch.call_count)
class GetTestCaseUnitTest(unittest.TestCase):

BIN
infra/cifuzz/test_files/out/do_stuff_fuzzer Executable file
View File

Binary file not shown.

22
infra/presubmit.py
View File

@ -20,6 +20,7 @@ import argparse
import os
import subprocess
import sys
import unittest
import yaml
_SRC_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
@ -320,12 +321,25 @@ def get_changed_files():
]
def run_tests():
"""Run all unit tests in directories that are different from HEAD."""
changed_dirs = set()
for file in get_changed_files():
changed_dirs.add(os.path.dirname(file))
suite_list = []
for change_dir in changed_dirs:
suite_list.append(unittest.TestLoader().discover(change_dir,
pattern='*_test.py'))
full_suite = unittest.TestSuite(suite_list)
unittest.TextTestRunner().run(full_suite)
def main():
"""Check changes on a branch for common issues before submitting."""
# Get program arguments.
parser = argparse.ArgumentParser(description='Presubmit script for oss-fuzz.')
parser.add_argument('command',
choices=['format', 'lint', 'license'],
choices=['format', 'lint', 'license', 'test'],
nargs='?')
args = parser.parse_args()
@ -346,8 +360,12 @@ def main():
success = check_license(changed_files)
return bool_to_returncode(success)
# Otherwise, do all of them.
if args.command == 'test':
return run_tests()
# Do all the checks (but no tests).
success = do_checks(changed_files)
return bool_to_returncode(success)

4
infra/repo_manager_test.py
View File

@ -116,9 +116,9 @@ class RepoManagerCheckoutPullRequestUnitTests(unittest.TestCase):
"""Tests that the git checkout pull request works."""
with tempfile.TemporaryDirectory() as tmp_dir:
test_repo_manager = repo_manager.RepoManager(OSS_FUZZ_REPO, tmp_dir)
test_repo_manager.checkout_pr('refs/pull/1757/merge')
test_repo_manager.checkout_pr('refs/pull/3415/merge')
self.assertEqual(test_repo_manager.get_current_commit(),
'2a2b11cc3d370db8f7bdf73046f3290a39615347')
'314c9249a54a08e764a5bbcb7333294ae7c1f9ed')
def test_checkout_invalid_pull_request(self):
"""Tests that the git checkout invalid pull request fails."""

47
infra/utils_test.py
View File

@ -22,6 +22,9 @@ import helper
EXAMPLE_PROJECT = 'example'
TEST_OUT_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)),
'cifuzz', 'test_files', 'out')
class IsFuzzTargetLocalUnitTest(unittest.TestCase):
"""Test is_fuzz_target_local function in the utils module."""
@ -37,22 +40,11 @@ class IsFuzzTargetLocalUnitTest(unittest.TestCase):
def test_valid_filepath(self):
"""Checks is_fuzz_target_local function with a valid filepath."""
utils.chdir_to_root()
helper.build_fuzzers_impl(EXAMPLE_PROJECT,
True,
'libfuzzer',
'address',
'x86_64', [],
None,
no_cache=False,
mount_location=None)
is_local = utils.is_fuzz_target_local(
os.path.join(helper.OSSFUZZ_DIR, 'build', 'out', EXAMPLE_PROJECT,
'do_stuff_fuzzer'))
os.path.join(TEST_OUT_DIR, 'do_stuff_fuzzer'))
self.assertTrue(is_local)
is_local = utils.is_fuzz_target_local(
os.path.join(helper.OSSFUZZ_DIR, 'build', 'out', EXAMPLE_PROJECT,
'do_stuff_fuzzer.dict'))
is_local = utils.is_fuzz_target_local(TEST_OUT_DIR)
self.assertFalse(is_local)
@ -61,36 +53,15 @@ class GetFuzzTargetsUnitTest(unittest.TestCase):
def test_valid_filepath(self):
"""Tests that fuzz targets can be retrieved once the fuzzers are built."""
utils.chdir_to_root()
helper.build_fuzzers_impl(EXAMPLE_PROJECT,
True,
'libfuzzer',
'address',
'x86_64', [],
None,
no_cache=False,
mount_location=None)
fuzz_targets = utils.get_fuzz_targets(
os.path.join(helper.OSSFUZZ_DIR, 'build', 'out', EXAMPLE_PROJECT))
self.assertCountEqual(fuzz_targets, [
os.path.join(helper.OSSFUZZ_DIR, 'build', 'out', EXAMPLE_PROJECT,
'do_stuff_fuzzer')
])
fuzz_targets = utils.get_fuzz_targets(TEST_OUT_DIR)
self.assertCountEqual(fuzz_targets,
[os.path.join(TEST_OUT_DIR, 'do_stuff_fuzzer')])
fuzz_targets = utils.get_fuzz_targets(
os.path.join(helper.OSSFUZZ_DIR, 'infra'))
self.assertFalse(fuzz_targets)
def test_invalid_filepath(self):
"""Tests what get_fuzz_targets return when invalid filepath is used."""
utils.chdir_to_root()
helper.build_fuzzers_impl(EXAMPLE_PROJECT,
True,
'libfuzzer',
'address',
'x86_64', [],
None,
no_cache=False,
mount_location=None)
fuzz_targets = utils.get_fuzz_targets('not/a/valid/file/path')
self.assertFalse(fuzz_targets)