diff --git a/infra/base-images/msan-builder/Dockerfile b/infra/base-images/msan-builder/Dockerfile index 51aa94193..b61491724 100644 --- a/infra/base-images/msan-builder/Dockerfile +++ b/infra/base-images/msan-builder/Dockerfile @@ -17,7 +17,7 @@ FROM gcr.io/oss-fuzz-base/base-clang MAINTAINER ochang@google.com RUN sed -i -r 's/#\s*deb-src/deb-src/g' /etc/apt/sources.list -RUN apt-get update && apt-get install -y python dpkg-dev patchelf +RUN apt-get update && apt-get install -y python dpkg-dev patchelf python-apt # Take all libraries from lib/msan RUN cp -R /usr/msan/lib/* /usr/lib/ diff --git a/infra/base-images/msan-builder/msan_build.py b/infra/base-images/msan-builder/msan_build.py index a15414f8b..c6961090f 100755 --- a/infra/base-images/msan-builder/msan_build.py +++ b/infra/base-images/msan-builder/msan_build.py @@ -19,10 +19,13 @@ from __future__ import print_function import argparse import imp import os +import re import shutil import subprocess import tempfile +import apt + from packages import package SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) @@ -205,6 +208,48 @@ def PatchRpath(path, output_directory): processed_rpath, path]) +def _CollectDependencies(apt_cache, pkg, cache, dependencies): + """Collect dependencies that need to be built.""" + C_OR_CXX_DEPS = [ + 'libc++1', + 'libc6', + 'libc++abi1', + 'libgcc1', + 'libstdc++6', + ] + + if pkg.name in C_OR_CXX_DEPS: + return True + + is_c_or_cxx = False + for dependency in pkg.versions[0].dependencies: + dependency = dependency[0] + if dependency.pre_depend: + continue + + if dependency.name in cache: + is_c_or_cxx |= cache[dependency.name] + else: + is_c_or_cxx |= _CollectDependencies(apt_cache, apt_cache[dependency.name], + cache, dependencies) + if is_c_or_cxx: + dependencies.append(pkg.name) + + cache[pkg.name] = is_c_or_cxx + return is_c_or_cxx + + + +def GetBuildList(package_name): + """Get list of packages that need to be built including dependencies.""" + apt_cache = apt.Cache() + pkg = apt_cache[package_name] + + dependencies = [] + _CollectDependencies(apt_cache, pkg, {}, dependencies) + return dependencies + + class MSanBuilder(object): """MSan builder.""" @@ -265,18 +310,35 @@ def main(): parser.add_argument('--debug', action='store_true', help='Enable debug mode.') parser.add_argument('--log-path', help='Log path for debugging.') parser.add_argument('--work-dir', help='Work directory.') - + parser.add_argument('--build-deps', action='store_true', + help='Build dependencies as well.') args = parser.parse_args() if not os.path.exists(args.output_dir): os.makedirs(args.output_dir) + if args.build_deps: + all_packages = set() + package_names = [] + for package_name in args.package_names: + for dep in GetBuildList(package_name): + if dep in all_packages: + continue + + all_packages.add(dep) + package_names.append(dep) + else: + package_names = args.package_names + + print('Going to build:') + for package_name in package_names: + print('\t', package_name) + with MSanBuilder(debug=args.debug, log_path=args.log_path, work_dir=args.work_dir) as builder: - for package_name in args.package_names: + for package_name in package_names: builder.Build(package_name, args.output_dir) if __name__ == '__main__': main() - diff --git a/infra/base-images/msan-builder/packages/package.py b/infra/base-images/msan-builder/packages/package.py index c2ad02321..75c051d1b 100644 --- a/infra/base-images/msan-builder/packages/package.py +++ b/infra/base-images/msan-builder/packages/package.py @@ -1,15 +1,11 @@ import os import subprocess +import apt + SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) -def FindDirs(directory): - """Find sub directories.""" - return [subdir for subdir in os.listdir(directory) - if os.path.isdir(os.path.join(directory, subdir))] - - def ApplyPatch(source_directory, patch_name): """Apply custom patch.""" subprocess.check_call(['patch', '-p1', '-i', @@ -46,20 +42,11 @@ class Package(object): def DownloadSource(self, download_directory): """Download the source for a package.""" self.PreDownload(download_directory) - before = FindDirs(download_directory) - subprocess.check_call( - ['apt-get', 'source', self.name], - stderr=subprocess.STDOUT, cwd=download_directory) - after = FindDirs(download_directory) - new_dirs = [subdir for subdir in after - if subdir not in before] + apt_cache = apt.Cache() + source_directory = apt_cache[self.name].versions[0].fetch_source( + download_directory) - if len(new_dirs) != 1: - raise PackageException( - 'Found more than one new directory after downloading apt-get source.') - - source_directory = os.path.join(download_directory, new_dirs[0]) self.PostDownload(source_directory) return source_directory