From 892cec4e75feb7e4fb8d4f23302417a5ea508c54 Mon Sep 17 00:00:00 2001 From: Abhishek Arya Date: Mon, 13 Apr 2020 21:57:30 -0700 Subject: [PATCH] Switch OSS projects to use native go-fuzz. (#3638) * Switch OSS projects to use native go-fuzz. * Fix go-json-iterator breakage, put source in package search dir. * Revert syzkaller change, track bug in #3639 --- .../new-project-guide/go_lang.md | 11 ++++++++--- projects/go-attestation/build.sh | 2 +- projects/go-dns/build.sh | 2 +- projects/go-json-iterator/Dockerfile | 6 +++--- projects/go-json-iterator/build.sh | 4 ++-- projects/golang/build.sh | 16 +++++++++++++++- projects/gonids/build.sh | 2 +- projects/kubernetes/Dockerfile | 18 ++++++++++++++++-- projects/kubernetes/build.sh | 2 +- projects/syzkaller/Dockerfile | 4 ++-- projects/syzkaller/build.sh | 10 +++++----- 11 files changed, 55 insertions(+), 22 deletions(-) diff --git a/docs/getting-started/new-project-guide/go_lang.md b/docs/getting-started/new-project-guide/go_lang.md index 939bae739..f58716dfc 100644 --- a/docs/getting-started/new-project-guide/go_lang.md +++ b/docs/getting-started/new-project-guide/go_lang.md @@ -22,13 +22,18 @@ process. The key specifics of integrating a Go project are outlined below. ## Go-fuzz support OSS-Fuzz supports **go-fuzz** in the -[libFuzzer compatible mode](https://github.com/dvyukov/go-fuzz#libfuzzer-support) +[libFuzzer compatible mode](https://github.com/mdempsky/go114-fuzz-build) only. In that mode, fuzz targets for Go use the libFuzzer engine with native Go coverage instrumentation. Binaries compiled in this mode provide the same libFuzzer command line interface as non-Go fuzz targets. ## Project files +First, you need to write a Go fuzz target that accepts a stream of bytes and +calls the program API with that. This fuzz target should reside in your project +repository +([example](https://github.com/golang/go/blob/4ad13555184eb0697c2e92c64c1b0bdb287ccc10/src/html/fuzz.go#L13)). + The structure of the project directory in OSS-Fuzz repository doesn't differ for projects written in Go. The project files have the following Go specific aspects. @@ -66,7 +71,7 @@ RUN go get github.com/ianlancetaylor/demangle ### build.sh -In order to build a Go fuzz target, you need to call `go-fuzz-build -libfuzzer` +In order to build a Go fuzz target, you need to call `go-fuzz` command first, and then link the resulting `.a` file against `$LIB_FUZZING_ENGINE` using the `$CXX $CXXFLAGS ...` command. [Example](https://github.com/google/oss-fuzz/blob/356f2b947670b7eb33a1f535c71bc5c87a60b0d1/projects/syzkaller/build.sh#L19): @@ -78,7 +83,7 @@ function compile_fuzzer { fuzzer=$3 # Instrument all Go files relevant to this fuzzer - go-fuzz-build -libfuzzer -func $function -o $fuzzer.a $path + go-fuzz -func $function -o $fuzzer.a $path # Instrumented, compiled Go ($fuzzer.a) + fuzzing engine = fuzzer binary $CXX $CXXFLAGS $LIB_FUZZING_ENGINE $fuzzer.a -lpthread -o $OUT/$fuzzer diff --git a/projects/go-attestation/build.sh b/projects/go-attestation/build.sh index 1aec57286..92d6de56c 100755 --- a/projects/go-attestation/build.sh +++ b/projects/go-attestation/build.sh @@ -22,7 +22,7 @@ function compile_fuzzer { fuzzer=$3 # Instrument all Go files relevant to this fuzzer - go-fuzz-build -libfuzzer -func $function -o $fuzzer.a $package + go-fuzz -func $function -o $fuzzer.a $package # Instrumented, compiled Go ($fuzzer.a) + fuzzing engine = fuzzer binary $CXX $CXXFLAGS $LIB_FUZZING_ENGINE $fuzzer.a -lpthread -o $OUT/$fuzzer diff --git a/projects/go-dns/build.sh b/projects/go-dns/build.sh index 26be8083e..e2c3f29ff 100755 --- a/projects/go-dns/build.sh +++ b/projects/go-dns/build.sh @@ -22,7 +22,7 @@ function compile_fuzzer { fuzzer=$3 # Instrument all Go files relevant to this fuzzer - go-fuzz-build -tags fuzz -libfuzzer -func $function -o $fuzzer.a $path + go-fuzz -tags fuzz -func $function -o $fuzzer.a $path # Instrumented, compiled Go ($fuzzer.a) + fuzzing engine = fuzzer binary $CXX $CXXFLAGS $LIB_FUZZING_ENGINE $fuzzer.a -lpthread -o $OUT/$fuzzer diff --git a/projects/go-json-iterator/Dockerfile b/projects/go-json-iterator/Dockerfile index eef259a0a..57cac6d56 100644 --- a/projects/go-json-iterator/Dockerfile +++ b/projects/go-json-iterator/Dockerfile @@ -18,7 +18,7 @@ FROM gcr.io/oss-fuzz-base/base-builder MAINTAINER taowen@gmail.com RUN go get github.com/json-iterator/go -RUN mkdir fuzz -COPY fuzz_json.go fuzz/ +RUN mkdir $GOPATH/src/fuzz +COPY fuzz_json.go $GOPATH/src/fuzz COPY build.sh $SRC/ -WORKDIR fuzz +WORKDIR $GOPATH/src/fuzz diff --git a/projects/go-json-iterator/build.sh b/projects/go-json-iterator/build.sh index c8edbc320..a6b121221 100755 --- a/projects/go-json-iterator/build.sh +++ b/projects/go-json-iterator/build.sh @@ -22,10 +22,10 @@ function compile_fuzzer { fuzzer=$3 # Instrument all Go files relevant to this fuzzer - go-fuzz-build -libfuzzer -func $function -o $fuzzer.a $path + go-fuzz -func $function -o $fuzzer.a $path # Instrumented, compiled Go ($fuzzer.a) + fuzzing engine = fuzzer binary $CXX $CXXFLAGS $LIB_FUZZING_ENGINE $fuzzer.a -lpthread -o $OUT/$fuzzer } -compile_fuzzer . Fuzz fuzz_json +compile_fuzzer fuzz Fuzz fuzz_json diff --git a/projects/golang/build.sh b/projects/golang/build.sh index efcbcfa1a..147192ebd 100755 --- a/projects/golang/build.sh +++ b/projects/golang/build.sh @@ -1,8 +1,22 @@ +# Copyright 2020 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + function compile_fuzzer { fuzzer=$(basename $1) # Instrument all Go files relevant to this fuzzer, compile and store in $fuzzer.a - go-fuzz-build -libfuzzer -o $fuzzer.a github.com/dvyukov/go-fuzz-corpus/$fuzzer + go-fuzz -o $fuzzer.a github.com/dvyukov/go-fuzz-corpus/$fuzzer # Instrumented, compiled Go ($fuzzer.a) + libFuzzer = fuzzer binary $CXX $CXXFLAGS $LIB_FUZZING_ENGINE $fuzzer.a -lpthread -o fuzzer-$fuzzer diff --git a/projects/gonids/build.sh b/projects/gonids/build.sh index e98d54ca6..5f241aaff 100755 --- a/projects/gonids/build.sh +++ b/projects/gonids/build.sh @@ -22,7 +22,7 @@ function compile_fuzzer { fuzzer=$3 # Instrument all Go files relevant to this fuzzer - go-fuzz-build -libfuzzer -func $function -o $fuzzer.a $path + go-fuzz -func $function -o $fuzzer.a $path # Instrumented, compiled Go ($fuzzer.a) + fuzzing engine = fuzzer binary $CXX $CXXFLAGS $LIB_FUZZING_ENGINE $fuzzer.a -lpthread -o $OUT/$fuzzer diff --git a/projects/kubernetes/Dockerfile b/projects/kubernetes/Dockerfile index e0c5afda2..8c095d29e 100644 --- a/projects/kubernetes/Dockerfile +++ b/projects/kubernetes/Dockerfile @@ -1,10 +1,24 @@ +# Copyright 2020 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +################################################################################ + FROM gcr.io/oss-fuzz-base/base-builder ENV GOPATH /gopath -RUN go get -u -d github.com/dvyukov/go-fuzz/... RUN go get github.com/ianlancetaylor/demangle - RUN git clone --depth 1 https://github.com/kubernetes/kubernetes.git /gopath/src/k8s.io/kubernetes WORKDIR /gopath/src/k8s.io/kubernetes diff --git a/projects/kubernetes/build.sh b/projects/kubernetes/build.sh index 539631bf4..ff4b0b328 100755 --- a/projects/kubernetes/build.sh +++ b/projects/kubernetes/build.sh @@ -27,7 +27,7 @@ function compile_fuzzer { local fuzzer="${pkg}_${function}" # Instrument all Go files relevant to this fuzzer - go-fuzz-build -libfuzzer -func "${function}" -o "${fuzzer}.a" "k8s.io/kubernetes/test/fuzz/${pkg}" + go-fuzz -func "${function}" -o "${fuzzer}.a" "k8s.io/kubernetes/test/fuzz/${pkg}" # Instrumented, compiled Go ($fuzzer.a) + fuzzing engine = fuzzer binary $CXX $CXXFLAGS $LIB_FUZZING_ENGINE "${fuzzer}.a" -lpthread -o "${OUT}/${fuzzer}" diff --git a/projects/syzkaller/Dockerfile b/projects/syzkaller/Dockerfile index 77bfde642..a02296570 100644 --- a/projects/syzkaller/Dockerfile +++ b/projects/syzkaller/Dockerfile @@ -22,6 +22,6 @@ RUN go get -u -d github.com/google/syzkaller/... # Dependency for one of the fuzz targets. RUN go get github.com/ianlancetaylor/demangle -RUN git clone --depth 1 https://github.com/google/syzkaller.git syzkaller -WORKDIR syzkaller +RUN git clone --depth 1 https://github.com/google/syzkaller.git $GOPATH/src/syzkaller +WORKDIR $GOPATH/src/syzkaller COPY build.sh $SRC/ diff --git a/projects/syzkaller/build.sh b/projects/syzkaller/build.sh index 44be203f7..b36f446d2 100755 --- a/projects/syzkaller/build.sh +++ b/projects/syzkaller/build.sh @@ -22,16 +22,16 @@ function compile_fuzzer { fuzzer=$3 # Instrument all Go files relevant to this fuzzer - go-fuzz-build -libfuzzer -func $function -o $fuzzer.a $path + go-fuzz-build -libfuzzer -func $function -o $fuzzer.a $path # Instrumented, compiled Go ($fuzzer.a) + fuzzing engine = fuzzer binary $CXX $CXXFLAGS $LIB_FUZZING_ENGINE $fuzzer.a -lpthread -o $OUT/$fuzzer } -compile_fuzzer ./pkg/compiler Fuzz compiler_fuzzer -compile_fuzzer ./prog/test FuzzDeserialize prog_deserialize_fuzzer -compile_fuzzer ./prog/test FuzzParseLog prog_parselog_fuzzer -compile_fuzzer ./pkg/report Fuzz report_fuzzer +compile_fuzzer syzkaller/pkg/compiler Fuzz compiler_fuzzer +compile_fuzzer syzkaller/prog/test FuzzDeserialize prog_deserialize_fuzzer +compile_fuzzer syzkaller/prog/test FuzzParseLog prog_parselog_fuzzer +compile_fuzzer syzkaller/pkg/report Fuzz report_fuzzer # This target is way too spammy and OOMs very quickly. # compile_fuzzer ./tools/syz-trace2syz/proggen Fuzz trace2syz_fuzzer