From e16ee1785bdcbddb724cd1e4262aa8782674e269 Mon Sep 17 00:00:00 2001 From: Mike Aizatsky Date: Tue, 29 Nov 2016 11:22:48 -0800 Subject: [PATCH] [infra] using project instead of target --- infra/README.md | 12 +-- infra/auto-setup/build_status.py | 4 +- infra/base-images/base-libfuzzer/README.md | 20 ++-- infra/helper.py | 117 ++++++--------------- infra/templates.py | 6 +- 5 files changed, 53 insertions(+), 106 deletions(-) diff --git a/infra/README.md b/infra/README.md index e26cb8904..e12fbd1a5 100644 --- a/infra/README.md +++ b/infra/README.md @@ -4,14 +4,14 @@ Core infrastructure: * [`base-images`](base-images/) - docker images for building fuzzers & corresponding jenkins pipeline. - + Continuous Integration infrastracture: * [`libfuzzer-pipeline.groovy`](libfuzzer-pipeline.groovy/) - jenkins pipeline that runs for each oss-fuzz project. * [`docker-cleanup`](docker-cleanup/) - jenkins pipeline to clean stale docker images & processes. * [`push-images`](push-images/) - jenkins pipeline to push built base images. -* [`jenkins-cluster`](jenkins-cluster/) - kubernetes cluster definition for our jenkins-based build (not operational yet, +* [`jenkins-cluster`](jenkins-cluster/) - kubernetes cluster definition for our jenkins-based build (not operational yet, [#10](https://github.com/google/oss-fuzz/issues/10)). ## helper.py @@ -19,9 +19,9 @@ Continuous Integration infrastracture: | Command | Description | |---------|------------- -| `generate` | Generates skeleton files for a new target | -| `build_image` | Builds a docker image for a given target | -| `build_fuzzers` | Builds fuzzers for a given target | +| `generate` | Generates skeleton files for a new project | +| `build_image` | Builds a docker image for a given project | +| `build_fuzzers` | Builds fuzzers for a given project | | `run_fuzzer` | Runs a fuzzer in a docker container | | `coverage` | Runs a fuzzer in a docker container and computes a coverage report | -| `shell` | Starts a shell inside the docker image for a target | +| `shell` | Starts a shell inside the docker image for a project | diff --git a/infra/auto-setup/build_status.py b/infra/auto-setup/build_status.py index 3c71b1273..778ff1e73 100755 --- a/infra/auto-setup/build_status.py +++ b/infra/auto-setup/build_status.py @@ -31,11 +31,11 @@ def get_build_results(server): for job in server.get_jobs(1): try: name = job['fullname'] - if not name.startswith('targets/'): + if not name.startswith('projects/'): continue print name - library = name[len('targets/'):] + library = name[len('projects/'):] info = server.get_job_info(name) last_build_number = info['lastCompletedBuild']['number'] diff --git a/infra/base-images/base-libfuzzer/README.md b/infra/base-images/base-libfuzzer/README.md index 6eedad1df..4e93a9ee3 100644 --- a/infra/base-images/base-libfuzzer/README.md +++ b/infra/base-images/base-libfuzzer/README.md @@ -1,12 +1,12 @@ # base-libfuzzer > Abstract base image for libfuzzer builders. -Every target image supports multiple commands that can be invoked through docker after building the image: +Every project image supports multiple commands that can be invoked through docker after building the image: 
-docker run --rm -ti ossfuzz/$target <command> <arguments...>
+docker run --rm -ti ossfuzz/$project <command> <arguments...>
- + # Supported Commands | Command | Description | @@ -18,17 +18,17 @@ docker run --rm -ti ossfuzz/$target <command> < # Examples -- *reproduce an issue using the latest OSS-Fuzz build:* +- *reproduce an issue using the latest OSS-Fuzz build:* 
-docker run --rm -ti -v $testcase_file:/testcase ossfuzz/$target reproduce $fuzzer
+docker run --rm -ti -v $testcase_file:/testcase ossfuzz/$project reproduce $fuzzer
- *reproduce using the local source code:* 
-    docker run --rm -ti -v $target_checkout_dir:/src/$target \
-                        -v $testcase_file:/testcase ossfuzz/$target reproduce $fuzzer
+    docker run --rm -ti -v $project_checkout_dir:/src/$project \
+                        -v $testcase_file:/testcase ossfuzz/$project reproduce $fuzzer
@@ -48,7 +48,7 @@ provided to be able to write retargetable scripts. ## Compiler Flags -You *must* use special compiler flags to build your target and fuzzers. +You *must* use special compiler flags to build your project and fuzzers. These flags are provided in following environment variables: | Env Variable | Description @@ -69,7 +69,7 @@ passing them manually to a build tool might be required. Child image has to checkout all sources it needs to compile fuzzers into `$SRC` directory. When the image is executed, a directory could be mounted on top of these with local checkouts using -`docker run -v $HOME/my_target:/src/my_target ...`. +`docker run -v $HOME/my_project:/src/my_project ...`. ## Other Required Files @@ -77,4 +77,4 @@ Following files have to be added by child images: | File Location | Description | | ------------- | ----------- | -| `$SRC/build.sh` | build script to build the target and its fuzzers | +| `$SRC/build.sh` | build script to build the project and its fuzzers | diff --git a/infra/helper.py b/infra/helper.py index 9122d5170..11ab7a7d8 100755 --- a/infra/helper.py +++ b/infra/helper.py @@ -73,18 +73,18 @@ def _is_base_image(image_name): return os.path.exists(os.path.join('infra', 'base-images', image_name)) -def _check_target_exists(target_name): - """Checks if a target exists.""" - if not os.path.exists(os.path.join(OSSFUZZ_DIR, 'targets', target_name)): - print(target_name, 'does not exist', file=sys.stderr) +def _check_project_exists(project_name): + """Checks if a project exists.""" + if not os.path.exists(os.path.join(OSSFUZZ_DIR, 'projects', project_name)): + print(project_name, 'does not exist', file=sys.stderr) return False return True -def _check_fuzzer_exists(target_name, fuzzer_name): +def _check_fuzzer_exists(project_name, fuzzer_name): """Checks if a fuzzer exists.""" - if not os.path.exists(os.path.join(BUILD_DIR, 'out', target_name, + if not os.path.exists(os.path.join(BUILD_DIR, 'out', project_name, fuzzer_name)): print(fuzzer_name, 'does not seem to exist. Please run build_fuzzers first.', @@ -105,10 +105,10 @@ def _build_image(image_name): if _is_base_image(image_name): dockerfile_dir = os.path.join('infra', 'base-images', image_name) else: - if not _check_target_exists(image_name): + if not _check_project_exists(image_name): return False - dockerfile_dir = os.path.join('targets', image_name) + dockerfile_dir = os.path.join('projects', image_name) command = [ 'docker', 'build', '--pull', '-t', 'ossfuzz/' + image_name, @@ -128,10 +128,10 @@ def _build_image(image_name): def build_image(build_args): """Build docker image.""" parser = argparse.ArgumentParser('helper.py build_image') - parser.add_argument('target_name') + parser.add_argument('project_name') args = parser.parse_args(build_args) - if _build_image(args.target_name): + if _build_image(args.project_name): return 0 return 1 @@ -140,16 +140,16 @@ def build_image(build_args): def build_fuzzers(build_args): """Build fuzzers.""" parser = argparse.ArgumentParser('helper.py build_fuzzers') - parser.add_argument('target_name') + parser.add_argument('project_name') args = parser.parse_args(build_args) - if not _build_image(args.target_name): + if not _build_image(args.project_name): return 1 command = [ 'docker', 'run', '-i', - '-v', '%s:/out' % os.path.join(BUILD_DIR, 'out', args.target_name), - '-t', 'ossfuzz/' + args.target_name, + '-v', '%s:/out' % os.path.join(BUILD_DIR, 'out', args.project_name), + '-t', 'ossfuzz/' + args.project_name, ] print('Running:', _get_command_string(command)) @@ -166,16 +166,16 @@ def build_fuzzers(build_args): def run_fuzzer(run_args): """Runs a fuzzer in the container.""" parser = argparse.ArgumentParser('helper.py run_fuzzer') - parser.add_argument('target_name', help='name of the target') + parser.add_argument('project_name', help='name of the project') parser.add_argument('fuzzer_name', help='name of the fuzzer') parser.add_argument('fuzzer_args', help='arguments to pass to the fuzzer', nargs=argparse.REMAINDER) args = parser.parse_args(run_args) - if not _check_target_exists(args.target_name): + if not _check_project_exists(args.project_name): return 1 - if not _check_fuzzer_exists(args.target_name, args.fuzzer_name): + if not _check_fuzzer_exists(args.project_name, args.fuzzer_name): return 1 if not _build_image('libfuzzer-runner'): @@ -183,7 +183,7 @@ def run_fuzzer(run_args): command = [ 'docker', 'run', '-i', - '-v', '%s:/out' % os.path.join(BUILD_DIR, 'out', args.target_name), + '-v', '%s:/out' % os.path.join(BUILD_DIR, 'out', args.project_name), '-t', 'ossfuzz/libfuzzer-runner', 'run_fuzzer', '/out/%s' % args.fuzzer_name, @@ -198,16 +198,16 @@ def coverage(run_args): parser = argparse.ArgumentParser('helper.py coverage') parser.add_argument('--run_time', default=60, help='time in seconds to run fuzzer') - parser.add_argument('target_name', help='name of the target') + parser.add_argument('project_name', help='name of the project') parser.add_argument('fuzzer_name', help='name of the fuzzer') parser.add_argument('fuzzer_args', help='arguments to pass to the fuzzer', nargs=argparse.REMAINDER) args = parser.parse_args(run_args) - if not _check_target_exists(args.target_name): + if not _check_project_exists(args.project_name): return 1 - if not _check_fuzzer_exists(args.target_name, args.fuzzer_name): + if not _check_fuzzer_exists(args.project_name, args.fuzzer_name): return 1 if not _build_image('libfuzzer-runner'): @@ -217,7 +217,7 @@ def coverage(run_args): command = [ 'docker', 'run', '-i', - '-v', '%s:/out' % os.path.join(BUILD_DIR, 'out', args.target_name), + '-v', '%s:/out' % os.path.join(BUILD_DIR, 'out', args.project_name), '-v', '%s:/cov' % temp_dir, '-w', '/cov', '-e', 'ASAN_OPTIONS=coverage=1,detect_leaks=0', @@ -232,11 +232,11 @@ def coverage(run_args): command = [ 'docker', 'run', '-i', - '-v', '%s:/out' % os.path.join(BUILD_DIR, 'out', args.target_name), + '-v', '%s:/out' % os.path.join(BUILD_DIR, 'out', args.project_name), '-v', '%s:/cov' % temp_dir, '-w', '/cov', '-p', '8001:8001', - '-t', 'ossfuzz/%s' % args.target_name, + '-t', 'ossfuzz/%s' % args.project_name, 'coverage_report', '/out/%s' % args.fuzzer_name, ] @@ -246,11 +246,11 @@ def coverage(run_args): def generate(generate_args): - """Generate empty target files.""" + """Generate empty project files.""" parser = argparse.ArgumentParser('helper.py generate') - parser.add_argument('target_name') + parser.add_argument('project_name') args = parser.parse_args(generate_args) - dir = os.path.join('targets', args.target_name) + dir = os.path.join('projects', args.project_name) try: os.mkdir(dir) @@ -263,7 +263,7 @@ def generate(generate_args): print('Writing new files to', dir) template_args = { - 'target_name' : args.target_name + 'project_name' : args.project_name } with open(os.path.join(dir, 'Jenkinsfile'), 'w') as f: f.write(templates.JENKINS_TEMPLATE % template_args) @@ -275,9 +275,6 @@ def generate(generate_args): with open(build_sh_path, 'w') as f: f.write(templates.BUILD_TEMPLATE % template_args) - targets_readme_path = os.path.join('targets', 'README.md') - update_targets_readme(targets_readme_path, args.target_name, dir) - os.chmod(build_sh_path, 0o755) return 0 @@ -285,16 +282,16 @@ def generate(generate_args): def shell(shell_args): """Runs a shell within a docker image.""" parser = argparse.ArgumentParser('helper.py shell') - parser.add_argument('target_name', help='name of the target') + parser.add_argument('project_name', help='name of the project') args = parser.parse_args(shell_args) - if not _build_image(args.target_name): + if not _build_image(args.project_name): return 1 command = [ 'docker', 'run', '-i', - '-v', '%s:/out' % os.path.join(BUILD_DIR, 'out', args.target_name), - '-t', 'ossfuzz/' + args.target_name, + '-v', '%s:/out' % os.path.join(BUILD_DIR, 'out', args.project_name), + '-t', 'ossfuzz/' + args.project_name, '/bin/bash' ] print('Running:', _get_command_string(command)) @@ -302,55 +299,5 @@ def shell(shell_args): pipe.communicate() -def update_targets_readme(readme_path, target_name, fuzzers_location): - """Add new target name and fuzzers location to the given README.md file.""" - readme_lines = [] - with open(readme_path) as f: - readme_lines += f.readlines() - - if not readme_lines: - print('ERROR: empty %s file' % readme_path) - return - - TARGETS_LIST_START_TOKEN = '| Target |' - first_target_line_number = -1 - for i, line in enumerate(readme_lines): - if line.startswith(TARGETS_LIST_START_TOKEN): - first_target_line_number = i + 2 - break - - if first_target_line_number < 0: - print('ERROR: list of targets is not found in %s file' % readme_path) - return - - def sanitize_line(line): - while line and not line[0] in string.ascii_letters + string.digits: - line = line[1:] - return line - - sanitized_lines = readme_lines[first_target_line_number : ] - sanitized_lines = [sanitize_line(line) for line in sanitized_lines] - - position_to_insert = -1 - for i in xrange(0, len(sanitized_lines)): - if target_name > sanitized_lines[i] and target_name < sanitized_lines[i+1]: - position_to_insert = i + 1 - break - - if position_to_insert < 0: - print('ERROR: please update %s file manually' % readme_path) - return - - position_to_insert += first_target_line_number - updated_readme_lines = readme_lines[ : position_to_insert] - updated_readme_lines.append('| %s | [/%s](%s) |\n' % (target_name, - fuzzers_location, - target_name)) - updated_readme_lines += readme_lines[position_to_insert : ] - - with open(readme_path, 'w') as f: - f.write(''.join(updated_readme_lines)) - - if __name__ == '__main__': sys.exit(main()) diff --git a/infra/templates.py b/infra/templates.py index defcf15b7..ecb1f6ea3 100755 --- a/infra/templates.py +++ b/infra/templates.py @@ -59,8 +59,8 @@ DOCKER_TEMPLATE = """\ FROM ossfuzz/base-libfuzzer MAINTAINER your@email.com RUN apt-get install -y make autoconf automake libtool -RUN git clone %(target_name)s # or use other version control -WORKDIR %(target_name)s +RUN git clone %(project_name)s # or use other version control +WORKDIR %(project_name)s COPY build.sh $src/ """ @@ -82,7 +82,7 @@ BUILD_TEMPLATE = """\ # ################################################################################ -# build the target. +# build the project. # e.g. # # ./autogen.sh