From 192bf5f9511caa8d41b0f8d5d3e51d7c88a220d7 Mon Sep 17 00:00:00 2001 From: jonathanmetzman <31354670+jonathanmetzman@users.noreply.github.com> Date: Tue, 27 Jul 2021 12:11:27 -0700 Subject: [PATCH] [CIFuzz] Add script to run cifuzz (#6118) This can be used by non-github users. Related: #6100 --- infra/cifuzz/build_fuzzers.py | 4 +- infra/cifuzz/docker.py | 2 + infra/cifuzz/run_cifuzz.py | 94 +++++++++++++++++++ .../Dockerfile | 4 +- .../build.sh | 0 5 files changed, 100 insertions(+), 4 deletions(-) create mode 100644 infra/cifuzz/run_cifuzz.py rename infra/cifuzz/test_data/external-project/{oss-fuzz => cifuzz-build-integration}/Dockerfile (91%) rename infra/cifuzz/test_data/external-project/{oss-fuzz => cifuzz-build-integration}/build.sh (100%) diff --git a/infra/cifuzz/build_fuzzers.py b/infra/cifuzz/build_fuzzers.py index 892d51abd..ef70c90d6 100644 --- a/infra/cifuzz/build_fuzzers.py +++ b/infra/cifuzz/build_fuzzers.py @@ -95,8 +95,8 @@ class Builder: # pylint: disable=too-many-instance-attributes ]) rm_path = os.path.join(self.image_repo_path, '*') image_src_path = os.path.dirname(self.image_repo_path) - bash_command = (f'rm -rf {rm_path} && cp -r {self.host_repo_path} ' - f'{image_src_path} && compile') + bash_command = (f'cd / && rm -rf {rm_path} && cp -r {self.host_repo_path} ' + f'{image_src_path} && cd - && compile') docker_args.append(bash_command) logging.info('Building with %s sanitizer.', self.config.sanitizer) diff --git a/infra/cifuzz/docker.py b/infra/cifuzz/docker.py index 1a649b189..9621c7994 100644 --- a/infra/cifuzz/docker.py +++ b/infra/cifuzz/docker.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. """Module for dealing with docker.""" +import logging import os import sys @@ -68,6 +69,7 @@ def get_base_docker_run_args(workspace, sanitizer='address', language='c++'): '-e', 'OUT=' + workspace.out ] docker_container = utils.get_container_name() + logging.info('Docker container: %s.', docker_container) if docker_container: # Don't map specific volumes if in a docker container, it breaks when # running a sibling container. diff --git a/infra/cifuzz/run_cifuzz.py b/infra/cifuzz/run_cifuzz.py new file mode 100644 index 000000000..4ad4a7a97 --- /dev/null +++ b/infra/cifuzz/run_cifuzz.py @@ -0,0 +1,94 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Script for running CIFuzz end-to-end. This is meant to work outside any +docker image. This cannot depend on any CIFuzz code or third party packages.""" +import os +import subprocess +import tempfile +import logging + +INFRA_DIR = os.path.dirname(os.path.dirname(__file__)) + +DEFAULT_ENVS = [('DRY_RUN', '0'), ('SANITIZER', 'address')] + +BASE_CIFUZZ_TAG = 'gcr.io/oss-fuzz-base/' + + +def set_default_env_var_if_unset(env_var, default_value): + """Sets the value of |env_var| in the environment to |default_value| if it was + not already set.""" + if env_var not in os.environ: + os.environ[env_var] = default_value + + +def docker_run(name, workdir, project_src_path): + """Runs a CIFuzz docker container with |name|.""" + command = [ + 'docker', 'run', '--name', name, '--rm', '-e', 'PROJECT_SRC_PATH', '-e', + 'BUILD_INTEGRATION_PATH', '-e', 'OSS_FUZZ_PROJECT_NAME', '-e', + 'GITHUB_WORKSPACE', '-e', 'GITHUB_EVENT_NAME', '-e', 'GITHUB_REPOSITORY', + '-e', 'GITHUB_EVENT_NAME', '-e', 'DRY_RUN', '-e', 'CI', '-e', 'SANITIZER', + '-e', 'GITHUB_SHA' + ] + if project_src_path: + command += ['-v', f'{project_src_path}:{project_src_path}'] + command += [ + '-v', '/var/run/docker.sock:/var/run/docker.sock', '-v', + f'{workdir}:{workdir}', f'gcr.io/oss-fuzz-base/{name}' + ] + print('Running docker command:', command) + subprocess.run(command, check=True) + + +def docker_build(image): + """Builds the CIFuzz |image|. Only suitable for building CIFuzz images.""" + command = [ + 'docker', 'build', '-t', BASE_CIFUZZ_TAG + image, '--file', + f'{image}.Dockerfile', '.' + ] + subprocess.run(command, check=True, cwd=INFRA_DIR) + + +def main(): + """Builds and runs fuzzers using CIFuzz.""" + for env_var, default_value in DEFAULT_ENVS: + set_default_env_var_if_unset(env_var, default_value) + + if 'GITHUB_REPOSITORY' not in os.environ: + repository = os.getenv('REPOSITORY') + assert repository + os.environ['GITHUB_REPOSITORY'] = repository + + project_src_path = os.getenv('PROJECT_SRC_PATH') + + with tempfile.TemporaryDirectory() as temp_dir: + if 'GITHUB_WORKSPACE' not in os.environ: + if 'WORKSPACE' not in os.environ: + os.environ['GITHUB_WORKSPACE'] = temp_dir + else: + os.environ['GITHUB_WORKSPACE'] = os.environ['WORKSPACE'] + + workdir = os.environ['GITHUB_WORKSPACE'] + + docker_build('build_fuzzers') + docker_run('build_fuzzers', workdir, project_src_path) + docker_build('run_fuzzers') + try: + docker_run('run_fuzzers', workdir, project_src_path) + except subprocess.CalledProcessError: + logging.error('run_fuzzers failed') + + +if __name__ == '__main__': + main() diff --git a/infra/cifuzz/test_data/external-project/oss-fuzz/Dockerfile b/infra/cifuzz/test_data/external-project/cifuzz-build-integration/Dockerfile similarity index 91% rename from infra/cifuzz/test_data/external-project/oss-fuzz/Dockerfile rename to infra/cifuzz/test_data/external-project/cifuzz-build-integration/Dockerfile index e9dc33031..c2c147999 100644 --- a/infra/cifuzz/test_data/external-project/oss-fuzz/Dockerfile +++ b/infra/cifuzz/test_data/external-project/cifuzz-build-integration/Dockerfile @@ -18,5 +18,5 @@ FROM gcr.io/oss-fuzz-base/base-builder RUN apt-get update && apt-get install -y make COPY . $SRC/external-project -WORKDIR external-project -COPY oss-fuzz/build.sh $SRC/ +WORKDIR $SRC/external-project +COPY cifuzz-build-integration/build.sh $SRC/ diff --git a/infra/cifuzz/test_data/external-project/oss-fuzz/build.sh b/infra/cifuzz/test_data/external-project/cifuzz-build-integration/build.sh similarity index 100% rename from infra/cifuzz/test_data/external-project/oss-fuzz/build.sh rename to infra/cifuzz/test_data/external-project/cifuzz-build-integration/build.sh