From ee87a8ebcc17bebe2efd0dc144e46b35ad7cf9a9 Mon Sep 17 00:00:00 2001
From: Catena cyber <35799796+catenacyber@users.noreply.github.com>
Date: Wed, 23 Nov 2022 14:27:32 +0100
Subject: [PATCH] ngolo-fuzzing: cleans while building (#9038)

cc @jonathanmetzman
This should consume less hard drive space for the build...
---
 projects/ngolo-fuzzing-x/Dockerfile   |  39 ++++++++++
 projects/ngolo-fuzzing-x/build.sh     | 106 ++++++++++++++++++++++++++
 projects/ngolo-fuzzing-x/project.yaml |   9 +++
 projects/ngolo-fuzzing/Dockerfile     |   6 --
 projects/ngolo-fuzzing/build.sh       |  31 +-------
 5 files changed, 156 insertions(+), 35 deletions(-)
 create mode 100644 projects/ngolo-fuzzing-x/Dockerfile
 create mode 100755 projects/ngolo-fuzzing-x/build.sh
 create mode 100644 projects/ngolo-fuzzing-x/project.yaml

diff --git a/projects/ngolo-fuzzing-x/Dockerfile b/projects/ngolo-fuzzing-x/Dockerfile
new file mode 100644
index 000000000..8e16742e3
--- /dev/null
+++ b/projects/ngolo-fuzzing-x/Dockerfile
@@ -0,0 +1,39 @@
+# Copyright 2021 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+################################################################################
+
+FROM gcr.io/oss-fuzz-base/base-builder-go
+
+RUN apt-get update && apt-get install -y protobuf-compiler libprotobuf-dev binutils cmake \
+   ninja-build liblzma-dev libz-dev pkg-config autoconf libtool
+RUN git clone --depth 1 https://github.com/google/libprotobuf-mutator.git
+RUN mkdir LPM; \
+  cd LPM; \
+  cmake $SRC/libprotobuf-mutator -GNinja -DLIB_PROTO_MUTATOR_DOWNLOAD_PROTOBUF=ON -DLIB_PROTO_MUTATOR_TESTING=OFF -DCMAKE_BUILD_TYPE=Release; \
+  ninja;
+
+RUN go install google.golang.org/protobuf/cmd/protoc-gen-go@latest
+
+RUN git clone --depth 1 https://go.googlesource.com/go goroot
+RUN git clone --depth 1 https://github.com/catenacyber/ngolo-fuzzing.git
+
+RUN mkdir $SRC/x
+RUN git clone --depth 1 https://github.com/golang/net x/net
+RUN git clone --depth 1 https://github.com/golang/image x/image
+RUN git clone --depth 1 https://github.com/golang/crypto x/crypto
+RUN git clone --depth 1 https://github.com/golang/text x/text
+
+COPY build.sh $SRC/
+WORKDIR $SRC/ngolo-fuzzing
diff --git a/projects/ngolo-fuzzing-x/build.sh b/projects/ngolo-fuzzing-x/build.sh
new file mode 100755
index 000000000..e6cb2bebb
--- /dev/null
+++ b/projects/ngolo-fuzzing-x/build.sh
@@ -0,0 +1,106 @@
+#!/bin/bash -eu
+# Copyright 2021 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+################################################################################
+
+# compile latest go from git
+(
+cd $SRC/goroot/src
+./make.bash
+)
+rm -Rf /root/.go/
+export PATH=$PATH:$SRC/goroot/bin/
+
+compile_package () {
+    pkg=$1
+    pkg_flat=`echo $pkg | sed 's/\//_/g' | sed 's/\./x/'`
+    args=`cat $SRC/ngolo-fuzzing/x/args.txt | grep "^$pkg_flat " | cut -d" " -f2-`
+    $SRC/ngolo-fuzzing/ngolo-fuzzing $args $pkg fuzz_ng_$pkg_flat
+    # applies special python patcher if any
+    ls $SRC/ngolo-fuzzing/x/$pkg_flat.py && (
+        python3 $SRC/ngolo-fuzzing/x/$pkg_flat.py fuzz_ng_$pkg_flat/fuzz_ng.go > fuzz_ng_$pkg_flat/fuzz_ngp.go
+        mv fuzz_ng_$pkg_flat/fuzz_ngp.go fuzz_ng_$pkg_flat/fuzz_ng.go
+    )
+    (
+        cd fuzz_ng_$pkg_flat
+        $SRC/LPM/external.protobuf/bin/protoc --go_out=./ ngolofuzz.proto
+        mkdir cpp
+        $SRC/LPM/external.protobuf/bin/protoc --cpp_out=./cpp ngolofuzz.proto
+        $CXX -DNDEBUG -stdlib=libc++ -c -I . -I $SRC/LPM/external.protobuf/include cpp/ngolofuzz.pb.cc
+        $CXX $CXXFLAGS -c -Icpp -I $SRC/libprotobuf-mutator/ -I $SRC/LPM/external.protobuf/include $SRC/ngolo-fuzzing/lpm/ngolofuzz.cc
+    )
+    if [ "$SANITIZER" = "coverage" ]
+    then
+        (
+        if [[ `echo $pkg | grep runtime | wc -l` == '1' ]]; then
+            continue
+        fi
+        cd fuzz_ng_$pkg_flat
+        GO_COV_ADD_PKG="$pkg" compile_go_fuzzer . FuzzNG_unsure fuzz_ngo_$pkg_flat
+        )
+    else
+        (
+        cd fuzz_ng_$pkg_flat
+        compile_go_fuzzer . FuzzNG_unsure fuzz_ngo_$pkg_flat
+        rm fuzz_ngo_$pkg_flat.a
+        )
+        $SRC/ngolo-fuzzing/go114-fuzz-build/go114-fuzz-build -func FuzzNG_valid -o fuzz_ng_$pkg_flat.a ./fuzz_ng_$pkg_flat
+
+        $CXX $CXXFLAGS $LIB_FUZZING_ENGINE fuzz_ng_$pkg_flat/ngolofuzz.pb.o fuzz_ng_$pkg_flat//ngolofuzz.o fuzz_ng_$pkg_flat.a  $SRC/LPM/src/libfuzzer/libprotobuf-mutator-libfuzzer.a $SRC/LPM/src/libprotobuf-mutator.a $SRC/LPM/external.protobuf/lib/libprotobuf.a -o $OUT/fuzz_ng_$pkg_flat
+        rm fuzz_ng_$pkg_flat.a
+    fi
+}
+
+# in $SRC/ngolo-fuzzing
+go build
+
+(
+cd go114-fuzz-build
+go build
+)
+
+# compile x packages
+cd $SRC/x
+ls | while read repo; do
+cd $repo
+find . -type d | while read pkg; do
+    if [[ `ls $pkg/*.go | wc -l` == '0' ]]; then
+        continue
+    fi
+    if [[ `echo $pkg | grep internal | wc -l` == '1' ]]; then
+        continue
+    fi
+    if [[ `echo $pkg | grep vendor | wc -l` == '1' ]]; then
+        continue
+    fi
+    if [[ `echo $pkg | grep testdata | wc -l` == '1' ]]; then
+        continue
+    fi
+    if compile_package $pkg; then
+        echo $pkg >> $SRC/ok.txt
+    else
+        echo "Failed for $pkg"
+        echo $pkg >> $SRC/ko.txt
+    fi
+
+done
+cd -
+done
+
+echo "Failed packages:"
+cat $SRC/ko.txt
+
+echo "Succesful packages:"
+cat $SRC/ok.txt
diff --git a/projects/ngolo-fuzzing-x/project.yaml b/projects/ngolo-fuzzing-x/project.yaml
new file mode 100644
index 000000000..4f44f6931
--- /dev/null
+++ b/projects/ngolo-fuzzing-x/project.yaml
@@ -0,0 +1,9 @@
+homepage: "'https://github.com/catenacyber/ngolo-fuzzing"
+primary_contact: "p.antoine@catenacyber.fr"
+language: go
+fuzzing_engines:
+  - libfuzzer
+sanitizers:
+  - address
+main_repo: 'https://github.com/catenacyber/ngolo-fuzzing'
+file_github_issue: True
diff --git a/projects/ngolo-fuzzing/Dockerfile b/projects/ngolo-fuzzing/Dockerfile
index 8e16742e3..303ea2f03 100644
--- a/projects/ngolo-fuzzing/Dockerfile
+++ b/projects/ngolo-fuzzing/Dockerfile
@@ -29,11 +29,5 @@ RUN go install google.golang.org/protobuf/cmd/protoc-gen-go@latest
 RUN git clone --depth 1 https://go.googlesource.com/go goroot
 RUN git clone --depth 1 https://github.com/catenacyber/ngolo-fuzzing.git
 
-RUN mkdir $SRC/x
-RUN git clone --depth 1 https://github.com/golang/net x/net
-RUN git clone --depth 1 https://github.com/golang/image x/image
-RUN git clone --depth 1 https://github.com/golang/crypto x/crypto
-RUN git clone --depth 1 https://github.com/golang/text x/text
-
 COPY build.sh $SRC/
 WORKDIR $SRC/ngolo-fuzzing
diff --git a/projects/ngolo-fuzzing/build.sh b/projects/ngolo-fuzzing/build.sh
index 689bc183d..19f16fe10 100755
--- a/projects/ngolo-fuzzing/build.sh
+++ b/projects/ngolo-fuzzing/build.sh
@@ -54,10 +54,12 @@ compile_package () {
         (
         cd fuzz_ng_$pkg_flat
         compile_go_fuzzer . FuzzNG_unsure fuzz_ngo_$pkg_flat
+        rm fuzz_ngo_$pkg_flat.a
         )
         $SRC/ngolo-fuzzing/go114-fuzz-build/go114-fuzz-build -func FuzzNG_valid -o fuzz_ng_$pkg_flat.a ./fuzz_ng_$pkg_flat
 
         $CXX $CXXFLAGS $LIB_FUZZING_ENGINE fuzz_ng_$pkg_flat/ngolofuzz.pb.o fuzz_ng_$pkg_flat//ngolofuzz.o fuzz_ng_$pkg_flat.a  $SRC/LPM/src/libfuzzer/libprotobuf-mutator-libfuzzer.a $SRC/LPM/src/libprotobuf-mutator.a $SRC/LPM/external.protobuf/lib/libprotobuf.a -o $OUT/fuzz_ng_$pkg_flat
+        rm fuzz_ng_$pkg_flat.a
     fi
 }
 
@@ -69,7 +71,6 @@ cd go114-fuzz-build
 go build
 )
 
-# maybe we should git clone --depth 1 https://github.com/golang/go.git
 find $SRC/goroot/src/ -type d | cut -d/ -f5- | while read pkg; do
     if [[ `ls $SRC/goroot/src/$pkg/*.go | wc -l` == '0' ]]; then
         continue
@@ -94,34 +95,6 @@ find $SRC/goroot/src/ -type d | cut -d/ -f5- | while read pkg; do
 
 done
 
-# compile x packages
-cd $SRC/x
-ls | while read repo; do
-cd $repo
-find . -type d | while read pkg; do
-    if [[ `ls $pkg/*.go | wc -l` == '0' ]]; then
-        continue
-    fi
-    if [[ `echo $pkg | grep internal | wc -l` == '1' ]]; then
-        continue
-    fi
-    if [[ `echo $pkg | grep vendor | wc -l` == '1' ]]; then
-        continue
-    fi
-    if [[ `echo $pkg | grep testdata | wc -l` == '1' ]]; then
-        continue
-    fi
-    if compile_package $pkg; then
-        echo $pkg >> $SRC/ok.txt
-    else
-        echo "Failed for $pkg"
-        echo $pkg >> $SRC/ko.txt
-    fi
-
-done
-cd -
-done
-
 echo "Failed packages:"
 cat $SRC/ko.txt