From 7d950a1008fa0f3829d3730482e561eb1345d477 Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Fri, 20 Aug 2021 02:46:54 +0300 Subject: [PATCH] Resurrect ClickHouse fuzzing (#6244) --- projects/clickhouse/Dockerfile | 45 +++++++++++++ projects/clickhouse/build.sh | 111 +++++++++++++++++++++++++++++++ projects/clickhouse/project.yaml | 8 +++ 3 files changed, 164 insertions(+) create mode 100644 projects/clickhouse/Dockerfile create mode 100755 projects/clickhouse/build.sh diff --git a/projects/clickhouse/Dockerfile b/projects/clickhouse/Dockerfile new file mode 100644 index 000000000..8de1d4da0 --- /dev/null +++ b/projects/clickhouse/Dockerfile @@ -0,0 +1,45 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +################################################################################ + +FROM gcr.io/oss-fuzz-base/base-builder + +RUN apt-get update -y \ + && env DEBIAN_FRONTEND=noninteractive \ + apt-get install --yes --no-install-recommends \ + bash \ + wget \ + curl \ + ccache \ + expect \ + ninja-build \ + perl \ + pkg-config \ + python \ + python-lxml \ + python-requests \ + python-termcolor \ + sudo \ + tzdata + + +RUN wget https://github.com/Kitware/CMake/releases/download/v3.21.1/cmake-3.21.1-Linux-x86_64.sh; \ + chmod +x cmake-3.21.1-Linux-x86_64.sh; \ + ./cmake-3.21.1-Linux-x86_64.sh --skip-license --prefix="/usr" + +RUN git clone --recursive https://github.com/ClickHouse/ClickHouse $SRC/ClickHouse +WORKDIR $SRC/ClickHouse + +COPY build.sh $SRC/ diff --git a/projects/clickhouse/build.sh b/projects/clickhouse/build.sh new file mode 100755 index 000000000..dea084bea --- /dev/null +++ b/projects/clickhouse/build.sh @@ -0,0 +1,111 @@ +#!/bin/bash -eu +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +################################################################################ + +mkdir $SRC/ClickHouse/build +cd $SRC/ClickHouse/build + +sed -i -e '/warnings.cmake)/d' $SRC/ClickHouse/CMakeLists.txt + +# It will be hard to maintain any compilation fails (if any) in two repositories. +# Also ClickHouse won't compile without this. +# It is very strange, because we have as many warnings as you could imagine. +sed -i -e 's/add_warning(/no_warning(/g' $SRC/ClickHouse/CMakeLists.txt + +# This files contain some errors. +# It wasn't build in our CI. So, it will be removed soon from upstream. +# P.S. Sorry for my Bash skills. +sed -i -e '$d' $SRC/ClickHouse/src/Common/examples/CMakeLists.txt +sed -i -e '$d' $SRC/ClickHouse/src/Common/examples/CMakeLists.txt +sed -i -e '$d' $SRC/ClickHouse/src/Common/examples/CMakeLists.txt +sed -i -e '$d' $SRC/ClickHouse/src/Common/examples/CMakeLists.txt +rm -rf $SRC/ClickHouse/src/Common/examples/YAML_fuzzer.cpp + +sed -i -e '$d' $SRC/ClickHouse/src/Parsers/examples/CMakeLists.txt +sed -i -e '$d' $SRC/ClickHouse/src/Parsers/examples/CMakeLists.txt +sed -i -e '$d' $SRC/ClickHouse/src/Parsers/examples/CMakeLists.txt +sed -i -e '$d' $SRC/ClickHouse/src/Parsers/examples/CMakeLists.txt +sed -i -e '$d' $SRC/ClickHouse/src/Parsers/examples/CMakeLists.txt +sed -i -e '$d' $SRC/ClickHouse/src/Parsers/examples/CMakeLists.txt +sed -i -e '$d' $SRC/ClickHouse/src/Parsers/examples/CMakeLists.txt +sed -i -e '$d' $SRC/ClickHouse/src/Parsers/examples/CMakeLists.txt +sed -i -e '$d' $SRC/ClickHouse/src/Parsers/examples/CMakeLists.txt +sed -i -e '$d' $SRC/ClickHouse/src/Parsers/examples/CMakeLists.txt +sed -i -e '$d' $SRC/ClickHouse/src/Parsers/examples/CMakeLists.txt + +rm -rf $SRC/ClickHouse/src/Parsers/examples/lexer_fuzzer.cpp +rm -rf $SRC/ClickHouse/src/Parsers/examples/create_parser_fuzzer.cpp +rm -rf $SRC/ClickHouse/src/Parsers/examples/select_parser_fuzzer.cpp + + +# Turn off all libraries, but turn on only necessary +cmake -G Ninja $SRC/ClickHouse \ + -DCMAKE_CXX_COMPILER_LAUNCHER=/usr/bin/ccache \ + -DCMAKE_C_COMPILER=$CC \ + -DCMAKE_CXX_COMPILER=$CXX \ + -DCMAKE_BUILD_TYPE=RelWithDebInfo \ + -DSANITIZE=$SANITIZER \ + -DENABLE_THINLTO=0 \ + -DENABLE_TESTS=0 \ + -DENABLE_EXAMPLES=1 \ + -DENABLE_UTILS=0 \ + -DENABLE_JEMALLOC=0 \ + -DENABLE_FUZZING=1 \ + -DLIB_FUZZING_ENGINE:STRING="$LIB_FUZZING_ENGINE" \ + -DENABLE_EMBEDDED_COMPILER=0 \ + -DENABLE_CLICKHOUSE_ODBC_BRIDGE=OFF \ + -DENABLE_LIBRARIES=0 \ + -DUSE_YAML_CPP=1 + +NUM_JOBS=$(($(nproc || grep -c ^processor /proc/cpuinfo))) + +ninja -j $NUM_JOBS + +# copy out fuzzer binaries +find $SRC/ClickHouse/build -name '*_fuzzer' -exec cp -v '{}' $OUT ';' + +# copy out fuzzer options and dictionaries +cp $SRC/ClickHouse/tests/fuzz/*.dict $OUT/ +cp $SRC/ClickHouse/tests/fuzz/*.options $OUT/ + +# prepare corpus dirs +mkdir $SRC/ClickHouse/tests/fuzz/lexer_fuzzer.in/ +mkdir $SRC/ClickHouse/tests/fuzz/select_parser_fuzzer.in/ +mkdir $SRC/ClickHouse/tests/fuzz/create_parser_fuzzer.in/ + +# prepare corpus +cp $SRC/ClickHouse/tests/queries/0_stateless/*.sql $SRC/ClickHouse/tests/fuzz/lexer_fuzzer.in/ +cp $SRC/ClickHouse/tests/queries/0_stateless/*.sql $SRC/ClickHouse/tests/fuzz/select_parser_fuzzer.in/ +cp $SRC/ClickHouse/tests/queries/0_stateless/*.sql $SRC/ClickHouse/tests/fuzz/create_parser_fuzzer.in/ +cp $SRC/ClickHouse/tests/queries/1_stateful/*.sql $SRC/ClickHouse/tests/fuzz/lexer_fuzzer.in/ +cp $SRC/ClickHouse/tests/queries/1_stateful/*.sql $SRC/ClickHouse/tests/fuzz/select_parser_fuzzer.in/ +cp $SRC/ClickHouse/tests/queries/1_stateful/*.sql $SRC/ClickHouse/tests/fuzz/create_parser_fuzzer.in/ + +# copy out corpus +cd $SRC/ClickHouse/tests/fuzz +for dir in *_fuzzer.in; do + fuzzer=$(basename $dir .in) + zip -rj "$OUT/${fuzzer}_seed_corpus.zip" "${dir}/" +done + +# copy sources for code coverage if required +if [ "$SANITIZER" = "coverage" ]; then + mkdir -p $OUT/src/ClickHouse/ + cp -rL --parents $SRC/ClickHouse/src $OUT + cp -rL --parents $SRC/ClickHouse/base $OUT + cp -rL --parents $SRC/ClickHouse/programs $OUT + cp -rL --parents $SRC/ClickHouse/contrib $OUT +fi diff --git a/projects/clickhouse/project.yaml b/projects/clickhouse/project.yaml index 9c0c7cf92..bee17b229 100644 --- a/projects/clickhouse/project.yaml +++ b/projects/clickhouse/project.yaml @@ -4,3 +4,11 @@ primary_contact: "clickhouse-feedback@yandex-team.com" auto_ccs: - "security@yandex-team.com" - "kyprizel@gmail.com" + - "jakalletti@yandex-team.com" + - "jakalletti@gmail.com" +sanitizers: + - address + - memory: + experimental: True + - undefined +main_repo: 'https://github.com/ClickHouse/ClickHouse.git'