Rooba
/
oss-fuzz
mirror of https://github.com/google/oss-fuzz.git
1
1
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Support C/C++ projects on AArch64. (#8184) 

Related: #8164
This commit is contained in:
jonathanmetzman 2022-08-09 10:18:06 -04:00 committed by GitHub
parent 4051b12f4d
commit f02edd2c9a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 179 additions and 127 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
infra/base-images/base-builder/Dockerfile
View File

@ -17,23 +17,8 @@
ARG parent_image=gcr.io/oss-fuzz-base/base-clang
FROM $parent_image
RUN dpkg --add-architecture i386 && \
apt-get update && \
apt-get install -y software-properties-common && \
add-apt-repository ppa:git-core/ppa && \
apt-get update && \
apt-get install -y \
binutils-dev \
build-essential \
curl \
wget \
git \
jq \
libc6-dev-i386 \
patchelf \
rsync \
subversion \
zip
COPY install_deps.sh /
RUN /install_deps.sh && rm /install_deps.sh
# Build and install latest Python 3 (3.8.3).
ENV PYTHON_VERSION 3.8.3

47
infra/base-images/base-builder/install_deps.sh Executable file
View File

@ -0,0 +1,47 @@
#!/bin/bash -eux
# Copyright 2022 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
################################################################################
# Install base-builder's dependencies in a architecture-aware way.
case $(uname -m) in
x86_64)
dpkg --add-architecture i386
;;
esac
apt-get update && \
apt-get install -y software-properties-common && \
add-apt-repository ppa:git-core/ppa && \
apt-get update && \
apt-get install -y \
binutils-dev \
build-essential \
curl \
wget \
git \
jq \
patchelf \
rsync \
subversion \
zip
case $(uname -m) in
x86_64)
apt-get install -y libc6-dev-i386
;;
esac

10
infra/base-images/base-clang/Dockerfile
View File

@ -18,6 +18,8 @@
FROM gcr.io/oss-fuzz-base/base-image
ARG arch=x86_64
ARG introspector
ENV INTROSPECTOR_PATCHES=$introspector
ENV FUZZINTRO_OUTDIR=$SRC
@ -26,10 +28,10 @@ ENV FUZZ_INTROSPECTOR=$introspector
# Install newer cmake.
ENV CMAKE_VERSION 3.21.1
RUN apt-get update && apt-get install -y wget sudo && \
wget https://github.com/Kitware/CMake/releases/download/v$CMAKE_VERSION/cmake-$CMAKE_VERSION-Linux-x86_64.sh && \
chmod +x cmake-$CMAKE_VERSION-Linux-x86_64.sh && \
./cmake-$CMAKE_VERSION-Linux-x86_64.sh --skip-license --prefix="/usr/local" && \
rm cmake-$CMAKE_VERSION-Linux-x86_64.sh && \
wget https://github.com/Kitware/CMake/releases/download/v$CMAKE_VERSION/cmake-$CMAKE_VERSION-Linux-$arch.sh && \
chmod +x cmake-$CMAKE_VERSION-Linux-$arch.sh && \
./cmake-$CMAKE_VERSION-Linux-$arch.sh --skip-license --prefix="/usr/local" && \
rm cmake-$CMAKE_VERSION-Linux-$arch.sh && \
SUDO_FORCE_REMOVE=yes apt-get remove --purge -y wget sudo && \
rm -rf /usr/local/doc/cmake /usr/local/bin/cmake-gui

225
infra/base-images/base-clang/checkout_build_install_llvm.sh
View File

@ -17,8 +17,33 @@
NPROC=$(nproc)
TARGET_TO_BUILD=
case $(uname -m) in
x86_64)
TARGET_TO_BUILD=X86
ARCHITECTURE_DEPS="g++-multilib"
# Use chromium's clang revision.
export CC=$WORK/llvm-stage1/bin/clang
export CXX=$WORK/llvm-stage1/bin/clang++
;;
aarch64)
TARGET_TO_BUILD=AArch64
# g++ multilib is not needed on AArch64 because we don't care about i386.
# We need to install clang and lld using apt because the binary downloaded
# from Chrome's developer tools doesn't support AArch64.
# TODO(metzman): Make x86_64 use the distro's clang for consistency once
# we support AArch64 fully.
ARCHITECTURE_DEPS="clang lld g++"
export CC=clang
export CXX=clang++
;;
*)
echo "Error: unsupported target $(uname -m)"
exit 1
;;
esac
# zlib1g-dev is needed for llvm-profdata to handle coverage data from rust compiler
LLVM_DEP_PACKAGES="build-essential make ninja-build git python3 python3-distutils g++-multilib binutils-dev zlib1g-dev"
LLVM_DEP_PACKAGES="build-essential make ninja-build git python3 python3-distutils binutils-dev zlib1g-dev $ARCHITECTURE_DEPS"
apt-get update && apt-get install -y $LLVM_DEP_PACKAGES --no-install-recommends
INTROSPECTOR_DEP_PACKAGES="texinfo bison flex"
@ -26,6 +51,24 @@ if [ -n "$INTROSPECTOR_PATCHES" ]; then
apt-get install -y $INTROSPECTOR_DEP_PACKAGES
fi
# For manual bumping.
OUR_LLVM_REVISION=llvmorg-14-init-7378-gaee49255
mkdir $SRC/chromium_tools
cd $SRC/chromium_tools
git clone https://chromium.googlesource.com/chromium/src/tools/clang
cd clang
# Pin clang due to https://github.com/google/oss-fuzz/issues/7617
git checkout 946a41a51f44207941b3729a0733dfc1e236644e
# To allow for manual downgrades. Set to 0 to use Chrome's clang version (i.e.
# *not* force a manual downgrade). Set to 1 to force a manual downgrade.
# DO NOT CHANGE THIS UNTIL https://github.com/google/oss-fuzz/issues/7273 is
# RESOLVED.
FORCE_OUR_REVISION=1
LLVM_REVISION=$(grep -Po "CLANG_REVISION = '\K([^']+)" scripts/update.py)
LLVM_SRC=$SRC/llvm-project
# Checkout
CHECKOUT_RETRIES=10
function clone_with_retries {
@ -48,7 +91,9 @@ function clone_with_retries {
set -e
return $CHECKOUT_RETURN_CODE
}
clone_with_retries https://github.com/llvm/llvm-project.git $LLVM_SRC
PROJECTS_TO_BUILD="libcxx;libcxxabi;compiler-rt;clang;lld"
function cmake_llvm {
extra_args="$@"
cmake -G "Ninja" \
@ -63,28 +108,6 @@ function cmake_llvm {
$LLVM_SRC/llvm
}
# Use chromium's clang revision
mkdir $SRC/chromium_tools
cd $SRC/chromium_tools
git clone https://chromium.googlesource.com/chromium/src/tools/clang
cd clang
# Pin clang due to https://github.com/google/oss-fuzz/issues/7617
git checkout 946a41a51f44207941b3729a0733dfc1e236644e
LLVM_SRC=$SRC/llvm-project
# For manual bumping.
OUR_LLVM_REVISION=llvmorg-14-init-7378-gaee49255
# To allow for manual downgrades. Set to 0 to use Chrome's clang version (i.e.
# *not* force a manual downgrade). Set to 1 to force a manual downgrade.
# DO NOT CHANGE THIS UNTIL https://github.com/google/oss-fuzz/issues/7273 is
# RESOLVED.
FORCE_OUR_REVISION=1
LLVM_REVISION=$(grep -Po "CLANG_REVISION = '\K([^']+)" scripts/update.py)
clone_with_retries https://github.com/llvm/llvm-project.git $LLVM_SRC
set +e
git -C $LLVM_SRC merge-base --is-ancestor $OUR_LLVM_REVISION $LLVM_REVISION
IS_OUR_REVISION_ANCESTOR_RETCODE=$?
@ -114,37 +137,91 @@ if [ -n "$INTROSPECTOR_PATCHES" ]; then
cd $OLD_WORKING_DIR
fi
# Build & install.
mkdir -p $WORK/llvm-stage2 $WORK/llvm-stage1
python3 $SRC/chromium_tools/clang/scripts/update.py --output-dir $WORK/llvm-stage1
TARGET_TO_BUILD=
case $(uname -m) in
x86_64)
TARGET_TO_BUILD=X86
;;
aarch64)
TARGET_TO_BUILD=AArch64
;;
*)
echo "Error: unsupported target $(uname -m)"
exit 1
;;
esac
PROJECTS_TO_BUILD="libcxx;libcxxabi;compiler-rt;clang;lld"
cd $WORK/llvm-stage2
export CC=$WORK/llvm-stage1/bin/clang
export CXX=$WORK/llvm-stage1/bin/clang++
cmake_llvm
ninja -j $NPROC
ninja install
rm -rf $WORK/llvm-stage1 $WORK/llvm-stage2
# libFuzzer sources.
cp -r $LLVM_SRC/compiler-rt/lib/fuzzer $SRC/libfuzzer
# Use the clang we just built from now on.
CMAKE_EXTRA_ARGS="-DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++"
function free_disk_space {
rm -rf $LLVM_SRC $SRC/chromium_tools
apt-get remove --purge -y $LLVM_DEP_PACKAGES
if [ -n "$INTROSPECTOR_PATCHES" ]; then
apt-get remove --purge -y $INTROSPECTOR_DEP_PACKAGES
fi
apt-get autoremove -y
# Delete unneeded parts of LLVM to reduce image size.
# See https://github.com/google/oss-fuzz/issues/5170
LLVM_TOOLS_TMPDIR=/tmp/llvm-tools
mkdir $LLVM_TOOLS_TMPDIR
# Move binaries with llvm- prefix that we want into LLVM_TOOLS_TMPDIR.
mv \
/usr/local/bin/llvm-ar \
/usr/local/bin/llvm-as \
/usr/local/bin/llvm-config \
/usr/local/bin/llvm-cov \
/usr/local/bin/llvm-objcopy \
/usr/local/bin/llvm-nm \
/usr/local/bin/llvm-profdata \
/usr/local/bin/llvm-ranlib \
/usr/local/bin/llvm-symbolizer \
/usr/local/bin/llvm-undname \
$LLVM_TOOLS_TMPDIR
# Delete remaining llvm- binaries.
rm -rf /usr/local/bin/llvm-*
# Restore the llvm- binaries we want to keep.
mv $LLVM_TOOLS_TMPDIR/* /usr/local/bin/
rm -rf $LLVM_TOOLS_TMPDIR
# Remove binaries from LLVM build that we don't need.
rm -f \
/usr/local/bin/bugpoint \
/usr/local/bin/llc \
/usr/local/bin/lli \
/usr/local/bin/clang-check \
/usr/local/bin/clang-refactor \
/usr/local/bin/clang-offload-wrapper \
/usr/local/bin/clang-offload-bundler \
/usr/local/bin/clang-check \
/usr/local/bin/clang-refactor \
/usr/local/bin/c-index-test \
/usr/local/bin/clang-rename \
/usr/local/bin/clang-scan-deps \
/usr/local/bin/clang-extdef-mapping \
/usr/local/bin/diagtool \
/usr/local/bin/sanstats \
/usr/local/bin/dsymutil \
/usr/local/bin/verify-uselistorder \
/usr/local/bin/clang-format
# Remove unneeded clang libs, CMake files from LLVM build, lld libs, and the
# libraries.
# Note: we need fuzzer_no_main libraries for atheris. Don't delete.
rm -rf \
/usr/local/lib/libclang* \
/usr/local/lib/liblld* \
/usr/local/lib/cmake/
}
if [ "$TARGET_TO_BUILD" == "AArch64" ]
then
free_disk_space
# Exit now on AArch64. We don't need to rebuild libc++ because on AArch64 we
# do not support MSAN nor do we care about i386.
exit 0
fi
function cmake_libcxx {
extra_args="$@"
cmake -G "Ninja" \
@ -189,66 +266,4 @@ ninja -j $NPROC cxx
ninja install-cxx
rm -rf $WORK/msan
# libFuzzer sources.
cp -r $LLVM_SRC/compiler-rt/lib/fuzzer $SRC/libfuzzer
# Cleanup
rm -rf $LLVM_SRC
rm -rf $SRC/chromium_tools
apt-get remove --purge -y $LLVM_DEP_PACKAGES
if [ -n "$INTROSPECTOR_PATCHES" ]; then
apt-get remove --purge -y $INTROSPECTOR_DEP_PACKAGES
fi
apt-get autoremove -y
# Delete unneeded parts of LLVM to reduce image size.
# See https://github.com/google/oss-fuzz/issues/5170
LLVM_TOOLS_TMPDIR=/tmp/llvm-tools
mkdir $LLVM_TOOLS_TMPDIR
# Move binaries with llvm- prefix that we want into LLVM_TOOLS_TMPDIR
mv \
/usr/local/bin/llvm-ar \
/usr/local/bin/llvm-as \
/usr/local/bin/llvm-config \
/usr/local/bin/llvm-cov \
/usr/local/bin/llvm-objcopy \
/usr/local/bin/llvm-nm \
/usr/local/bin/llvm-profdata \
/usr/local/bin/llvm-ranlib \
/usr/local/bin/llvm-symbolizer \
/usr/local/bin/llvm-undname \
$LLVM_TOOLS_TMPDIR
# Delete remaining llvm- binaries.
rm -rf /usr/local/bin/llvm-*
# Restore the llvm- binaries we want to keep.
mv $LLVM_TOOLS_TMPDIR/* /usr/local/bin/
rm -rf $LLVM_TOOLS_TMPDIR
# Remove binaries from LLVM build that we don't need.
rm -f \
/usr/local/bin/bugpoint \
/usr/local/bin/llc \
/usr/local/bin/lli \
/usr/local/bin/clang-check \
/usr/local/bin/clang-refactor \
/usr/local/bin/clang-offload-wrapper \
/usr/local/bin/clang-offload-bundler \
/usr/local/bin/clang-check \
/usr/local/bin/clang-refactor \
/usr/local/bin/c-index-test \
/usr/local/bin/clang-rename \
/usr/local/bin/clang-scan-deps \
/usr/local/bin/clang-extdef-mapping \
/usr/local/bin/diagtool \
/usr/local/bin/sanstats \
/usr/local/bin/dsymutil \
/usr/local/bin/verify-uselistorder \
/usr/local/bin/clang-format
# Remove unneeded clang libs, CMake files from LLVM build, lld libs, and the
# libraries.
# Note: we need fuzzer_no_main libraries for atheris. Don't delete.
rm -rf \
/usr/local/lib/libclang* \
/usr/local/lib/liblld* \
/usr/local/lib/cmake/
free_disk_space

5
infra/base-images/base-image/Dockerfile
View File

@ -16,7 +16,10 @@
# Base image for all other images.
FROM ubuntu:20.04
ARG parent_image=ubuntu:20.04
FROM $parent_image
ENV DEBIAN_FRONTEND noninteractive
RUN apt-get update && \
apt-get upgrade -y && \