diff --git a/infra/base-images/msan-builder/Dockerfile b/infra/base-images/msan-builder/Dockerfile index 0d00ea19b..9597e1f45 100644 --- a/infra/base-images/msan-builder/Dockerfile +++ b/infra/base-images/msan-builder/Dockerfile @@ -23,4 +23,5 @@ RUN apt-get update && apt-get install -y python dpkg-dev RUN cp -R /usr/msan/lib/* /usr/lib/ COPY compiler_wrapper.py msan_build.py /usr/local/bin/ +COPY packages /usr/local/bin/packages WORKDIR $SRC diff --git a/infra/base-images/msan-builder/msan_build.py b/infra/base-images/msan-builder/msan_build.py index 60ee9c815..ca6db4966 100755 --- a/infra/base-images/msan-builder/msan_build.py +++ b/infra/base-images/msan-builder/msan_build.py @@ -17,12 +17,16 @@ from __future__ import print_function import argparse +import imp import os import shutil import subprocess import tempfile +from packages import package + SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) +PACKAGES_DIR = os.path.join(SCRIPT_DIR, 'packages') INJECTED_ARGS = [ '-fsanitize=memory', @@ -99,41 +103,6 @@ def SetUpEnvironment(work_dir): return env -def InstallBuildDeps(package_name): - """Install build dependencies for a package.""" - subprocess.check_call(['apt-get', 'build-dep', '-y', package_name]) - - -def DownloadPackageSource(package_name, download_directory): - """Download the source for a package.""" - before = FindDirs(download_directory) - subprocess.check_call( - ['apt-get', 'source', package_name], - stderr=subprocess.STDOUT, cwd=download_directory) - - after = FindDirs(download_directory) - new_dirs = [subdir for subdir in after - if subdir not in before] - - if len(new_dirs) != 1: - raise MSanBuildException( - 'Found more than one new directory after downloading apt-get source.') - - return os.path.join(download_directory, new_dirs[0]) - - -def FindDirs(directory): - """Find sub directories.""" - return [subdir for subdir in os.listdir(directory) - if os.path.isdir(os.path.join(directory, subdir))] - - -def BuildDebianPackage(source_directory, env): - """Build .deb packages.""" - subprocess.check_call( - ['dpkg-buildpackage', '-us', '-uc', '-b'], cwd=source_directory, env=env) - - def ExtractSharedLibraries(work_directory, output_directory): """Extract all shared libraries from .deb packages.""" extract_directory = os.path.join(work_directory, 'extracted') @@ -153,17 +122,30 @@ def ExtractSharedLibraries(work_directory, output_directory): shutil.copy2(file_path, output_directory) +def GetPackage(package_name): + custom_package_path = os.path.join(PACKAGES_DIR, package_name) + '.py' + if not os.path.exists(custom_package_path): + print('Using default package build steps.') + return package.Package(package_name) + + print('Using custom package build steps.') + module = imp.load_source('packages.' + package_name, custom_package_path) + return module.Package() + + class MSanBuilder(object): """MSan builder.""" - def __init__(self, debug=False, log_path=None): + def __init__(self, debug=False, log_path=None, work_dir=None): self.debug = debug self.log_path = log_path - self.work_dir = None + self.work_dir = work_dir self.env = None def __enter__(self): - self.work_dir = tempfile.mkdtemp() + if not self.work_dir: + self.work_dir = tempfile.mkdtemp(dir=self.work_dir) + self.env = SetUpEnvironment(self.work_dir) if self.debug and self.log_path: @@ -175,13 +157,15 @@ class MSanBuilder(object): if not self.debug: shutil.rmtree(self.work_dir, ignore_errors=True) - def build(self, package_name, output_directory): + def Build(self, package_name, output_directory): """Build the package and write results into the output directory.""" - InstallBuildDeps(package_name) - source_directory = DownloadPackageSource(package_name, self.work_dir) + pkg = GetPackage(package_name) + + pkg.InstallBuildDeps() + source_directory = pkg.DownloadSource(self.work_dir) print('Source downloaded to', source_directory) - BuildDebianPackage(source_directory, self.env) + pkg.Build(source_directory, self.env) ExtractSharedLibraries(self.work_dir, output_directory) @@ -191,14 +175,16 @@ def main(): parser.add_argument('output_dir', help='Output directory.') parser.add_argument('--debug', action='store_true', help='Enable debug mode.') parser.add_argument('--log-path', help='Log path for debugging.') + parser.add_argument('--work-dir', help='Work directory.') args = parser.parse_args() if not os.path.exists(args.output_dir): os.makedirs(args.output_dir) - with MSanBuilder(debug=args.debug, log_path=args.log_path) as builder: - builder.build(args.package_name, args.output_dir) + with MSanBuilder(debug=args.debug, log_path=args.log_path, + work_dir=args.work_dir) as builder: + builder.Build(args.package_name, args.output_dir) if __name__ == '__main__': diff --git a/infra/base-images/msan-builder/packages/__init__.py b/infra/base-images/msan-builder/packages/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/infra/base-images/msan-builder/packages/package.py b/infra/base-images/msan-builder/packages/package.py new file mode 100644 index 000000000..c2ad02321 --- /dev/null +++ b/infra/base-images/msan-builder/packages/package.py @@ -0,0 +1,74 @@ +import os +import subprocess + +SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) + + +def FindDirs(directory): + """Find sub directories.""" + return [subdir for subdir in os.listdir(directory) + if os.path.isdir(os.path.join(directory, subdir))] + + +def ApplyPatch(source_directory, patch_name): + """Apply custom patch.""" + subprocess.check_call(['patch', '-p1', '-i', + os.path.join(SCRIPT_DIR, patch_name)], + cwd=source_directory) + + +class PackageException(Exception): + """Base package exception.""" + + +class Package(object): + """Base package.""" + + def __init__(self, name): + self.name = name + + def PreBuild(self, source_directory, env): + return + + def PostBuild(self, source_directory, env): + return + + def PreDownload(self, download_directory): + return + + def PostDownload(self, source_directory): + return + + def InstallBuildDeps(self): + """Install build dependencies for a package.""" + subprocess.check_call(['apt-get', 'build-dep', '-y', self.name]) + + def DownloadSource(self, download_directory): + """Download the source for a package.""" + self.PreDownload(download_directory) + before = FindDirs(download_directory) + subprocess.check_call( + ['apt-get', 'source', self.name], + stderr=subprocess.STDOUT, cwd=download_directory) + + after = FindDirs(download_directory) + new_dirs = [subdir for subdir in after + if subdir not in before] + + if len(new_dirs) != 1: + raise PackageException( + 'Found more than one new directory after downloading apt-get source.') + + source_directory = os.path.join(download_directory, new_dirs[0]) + self.PostDownload(source_directory) + return source_directory + + def Build(self, source_directory, env): + """Build .deb packages.""" + self.PreBuild(source_directory, env) + subprocess.check_call( + ['dpkg-buildpackage', '-us', '-uc', '-b'], + cwd=source_directory, env=env) + self.PostBuild(source_directory, env) + + diff --git a/infra/base-images/msan-builder/packages/pulseaudio.py b/infra/base-images/msan-builder/packages/pulseaudio.py new file mode 100644 index 000000000..11dae9514 --- /dev/null +++ b/infra/base-images/msan-builder/packages/pulseaudio.py @@ -0,0 +1,25 @@ +from __future__ import print_function +import glob +import os +import subprocess + +import package + + +class Package(package.Package): + """PulseAudio package.""" + + def __init__(self): + super(Package, self).__init__('pulseaudio') + + def PostDownload(self, source_directory): + """Remove blacklisted patches.""" + # Fix *droid* patches. + bad_patch_path = os.path.join( + source_directory, 'debian', 'patches', + '0600-droid-sync-with-upstream-for-Android-5-support-and-b.patch') + if not os.path.exists(bad_patch_path): + return + + print('Applying custom patches.') + package.ApplyPatch(source_directory, 'pulseaudio_fix_android.patch') diff --git a/infra/base-images/msan-builder/packages/pulseaudio_fix_android.patch b/infra/base-images/msan-builder/packages/pulseaudio_fix_android.patch new file mode 100644 index 000000000..e86f79823 --- /dev/null +++ b/infra/base-images/msan-builder/packages/pulseaudio_fix_android.patch @@ -0,0 +1,39 @@ +--- pulseaudio-8.0/src/modules/droid/module-droid-card.c 2017-11-27 22:09:42.533589970 +0000 ++++ pulseaudio-8.0.fixed/src/modules/droid/module-droid-card.c 2017-11-27 22:28:23.847250467 +0000 +@@ -66,10 +66,11 @@ + #include "droid-extcon.h" + #endif + +-#if ANDROID_VERSION_MAJOR == 4 && ANDROID_VERSION_MINOR == 2 ++#if ANDROID_VERSION_MAJOR == 4 && ANDROID_VERSION_MINOR == 4 + #include "module-droid-card-19-symdef.h" + #elif ANDROID_VERSION_MAJOR == 5 && ANDROID_VERSION_MINOR == 1 + #include "module-droid-card-22-symdef.h" ++#else + #endif + + PA_MODULE_AUTHOR("Juho Hämäläinen"); +diff -ru pulseaudio-8.0/src/modules/droid/module-droid-sink.c pulseaudio-8.0.fixed/src/modules/droid/module-droid-sink.c +--- pulseaudio-8.0/src/modules/droid/module-droid-sink.c 2017-11-27 22:09:42.533589970 +0000 ++++ pulseaudio-8.0.fixed/src/modules/droid/module-droid-sink.c 2017-11-27 22:29:53.776348900 +0000 +@@ -40,7 +40,7 @@ + #include "droid-util.h" + #include "droid-sink.h" + +-#if ANDROID_VERSION_MAJOR == 4 && ANDROID_VERSION_MINOR == 2 ++#if ANDROID_VERSION_MAJOR == 4 && ANDROID_VERSION_MINOR == 4 + #include "module-droid-sink-19-symdef.h" + #elif ANDROID_VERSION_MAJOR == 5 && ANDROID_VERSION_MINOR == 1 + #include "module-droid-sink-22-symdef.h" +diff -ru pulseaudio-8.0/src/modules/droid/module-droid-source.c pulseaudio-8.0.fixed/src/modules/droid/module-droid-source.c +--- pulseaudio-8.0/src/modules/droid/module-droid-source.c 2017-11-27 22:09:42.533589970 +0000 ++++ pulseaudio-8.0.fixed/src/modules/droid/module-droid-source.c 2017-11-27 22:30:03.920472828 +0000 +@@ -40,7 +40,7 @@ + #include "droid-util.h" + #include "droid-source.h" + +-#if ANDROID_VERSION_MAJOR == 4 && ANDROID_VERSION_MINOR == 2 ++#if ANDROID_VERSION_MAJOR == 4 && ANDROID_VERSION_MINOR == 4 + #include "module-droid-source-19-symdef.h" + #elif ANDROID_VERSION_MAJOR == 5 && ANDROID_VERSION_MINOR == 1 + #include "module-droid-source-22-symdef.h"