Go 1.16 more fixes (#5239)

* Fixes cilium build * Fixes dragonfly build * Fixes fasthttp build * Fixes fastjson build * golang build: change directory only temporary * Fixes gitea build * Fixes grpc-gateway build * Fixes hugo project build * Fixes ipfs build * Fixes jsonparser build * Fixes kubernetes build * Fixes loki build * Fixes minify build * Fixes nats build * Removes go get from the docs * Fixes quic-go build * Fixes radon build * Fixes syzkaller build * Fixes tidb build * Fixes vitess build
2021-02-23 00:25:47 +01:00 · 2021-02-23 00:25:47 +01:00 · 1e03eb94af
parent afb9e09ae2
commit 1e03eb94af
32 changed files with 50 additions and 80 deletions
--- a/docs/getting-started/new-project-guide/go_lang.md
+++ b/docs/getting-started/new-project-guide/go_lang.md
@ -60,21 +60,16 @@ sanitizers:
 ### Dockerfile

 The OSS-Fuzz builder image has the latest stable release of Golang installed. In
-order to install dependencies of your project, add `RUN go get ...` command to
+order to install dependencies of your project, add `RUN git clone ...` command to
 your Dockerfile.
 [Example](https://github.com/google/oss-fuzz/blob/356f2b947670b7eb33a1f535c71bc5c87a60b0d1/projects/syzkaller/Dockerfile#L23):

 ```dockerfile
 # Dependency for one of the fuzz targets.
-RUN go get github.com/ianlancetaylor/demangle
+RUN git clone --depth 1 https://github.com/ianlancetaylor/demangle
 ```

-In the case you are using modules, the best practice is to `git clone` the repository into the expected `$GOPATH/src` directory.
-
-A usage example from go-coredns project is
-```dockerfile
-RUN git clone --depth 1  https://github.com/coredns/coredns $GOPATH/src/github.com/coredns/coredns
-```
+go-fuzz will then automatically download the dependencies based on the go.mod file

 ### build.sh

--- a/docs/getting-started/new_project_guide.md
+++ b/docs/getting-started/new_project_guide.md
@ -192,7 +192,6 @@ For most projects, the image is simple:
 ```docker
 FROM gcr.io/oss-fuzz-base/base-builder       # base image with clang toolchain
 RUN apt-get update && apt-get install -y ... # install required packages to build your project
-RUN go get ...                               # install dependencies to build your Go project
 RUN git clone <git_url> <checkout_dir>       # checkout all sources needed to build your project
 WORKDIR <checkout_dir>                       # current directory for the build script
 COPY build.sh fuzzer.cc $SRC/                # copy build script and other fuzzer files in src dir
--- a/infra/base-images/base-builder/compile_go_fuzzer
+++ b/infra/base-images/base-builder/compile_go_fuzzer
@ -23,11 +23,13 @@ if [[ $#  -eq 4 ]]; then
  tags="-tags $4"
 fi

+# makes directory change temporary
+(
 cd $GOPATH/src/$path || true
 # in the case we are in the right directory, with go.mod but no go.sum
 go mod tidy || true
 # project was downloaded with go get if go list fails
-go list $tags $path || (cd $GOPATH/pkg/mod/ && cd `echo $path | cut -d/ -f1-3 | awk '{print $1"*"}'`)
+go list $tags $path || { cd $GOPATH/pkg/mod/ && cd `echo $path | cut -d/ -f1-3 | awk '{print $1"@*"}'`; }
 # project does not have go.mod if go list fails again
 go list $tags $path || go mod init $path

@ -52,3 +54,4 @@ else
  # Link Go code ($fuzzer.a) with fuzzing engine to produce fuzz target binary.
  $CXX $CXXFLAGS $LIB_FUZZING_ENGINE $fuzzer.a -o $OUT/$fuzzer
 fi
+)
--- a/projects/cilium/Dockerfile
+++ b/projects/cilium/Dockerfile
@ -21,6 +21,6 @@ RUN wget https://raw.githubusercontent.com/google/AFL/master/dictionaries/json.d
 RUN git clone --depth 1 https://github.com/dvyukov/go-fuzz-corpus
 RUN zip $OUT/fuzz_seed_corpus.zip go-fuzz-corpus/json/corpus/*

-RUN go get github.com/cilium/cilium/pkg/labels/...
-RUN cp $GOPATH/src/github.com/cilium/cilium/test/fuzzing/oss-fuzz-build.sh $SRC/build.sh
-WORKDIR $SRC
+RUN git clone https://github.com/cilium/cilium/ cilium
+RUN cp $SRC/cilium/test/fuzzing/oss-fuzz-build.sh $SRC/build.sh
+WORKDIR $SRC/cilium
--- a/projects/dragonfly/Dockerfile
+++ b/projects/dragonfly/Dockerfile
@ -30,4 +30,4 @@ RUN go get	github.com/go-openapi/swag \
 		github.com/willf/bitset
 RUN git clone https://github.com/dragonflyoss/Dragonfly
 COPY build.sh $SRC/
-WORKDIR $SRC/
+WORKDIR $SRC/Dragonfly
--- a/projects/dragonfly/build.sh
+++ b/projects/dragonfly/build.sh
@ -15,8 +15,5 @@
 #
 ################################################################################

-mkdir $GOPATH/src/github.com/dragonflyoss
-cp -r $SRC/Dragonfly $GOPATH/src/github.com/dragonflyoss/
-
 compile_go_fuzzer github.com/dragonflyoss/Dragonfly/dfget/core/uploader FuzzParseParams uploader_fuzz
 compile_go_fuzzer github.com/dragonflyoss/Dragonfly/supernode/daemon/mgr/cdn Fuzz cdn_fuzz
--- a/projects/fasthttp/Dockerfile
+++ b/projects/fasthttp/Dockerfile
@ -15,7 +15,7 @@
 ################################################################################

 FROM gcr.io/oss-fuzz-base/base-builder
-RUN go get github.com/valyala/fasthttp
+RUN git clone --depth 1 https://github.com/valyala/fasthttp

 COPY build.sh $SRC/
-WORKDIR $SRC/
+WORKDIR $SRC/fasthttp
--- a/projects/fasthttp/build.sh
+++ b/projects/fasthttp/build.sh
@ -18,7 +18,7 @@



-ls $GOPATH/src/github.com/valyala/fasthttp/fuzzit | while read target
+ls fuzzit/ | while read target
 do
    compile_go_fuzzer github.com/valyala/fasthttp/fuzzit/$target Fuzz fuzz_$target gofuzz
 done
--- a/projects/fastjson/Dockerfile
+++ b/projects/fastjson/Dockerfile
@ -15,7 +15,7 @@
 ################################################################################

 FROM gcr.io/oss-fuzz-base/base-builder
-RUN go get github.com/valyala/fastjson
+RUN git clone --depth 1 https://github.com/valyala/fastjson

 COPY build.sh $SRC/
-WORKDIR $SRC/
+WORKDIR $SRC/fastjson
--- a/projects/gitea/Dockerfile
+++ b/projects/gitea/Dockerfile
@ -17,4 +17,4 @@
 FROM gcr.io/oss-fuzz-base/base-builder
 RUN git clone https://github.com/go-gitea/gitea
 COPY build.sh $SRC/
-WORKDIR $SRC/
+WORKDIR $SRC/gitea
--- a/projects/gitea/build.sh
+++ b/projects/gitea/build.sh
@ -15,9 +15,5 @@
 #
 ################################################################################

-mkdir $GOPATH/src/code.gitea.io
-mv $SRC/gitea $GOPATH/src/code.gitea.io/
-cd $GOPATH/src/code.gitea.io/gitea && go get ./...
-
 compile_go_fuzzer code.gitea.io/gitea/tools FuzzMarkdownRenderRaw fuzz_markdown_render_raw gofuzz
 compile_go_fuzzer code.gitea.io/gitea/tools FuzzMarkupPostProcess fuzz_markup_post_process gofuzz
--- a/projects/grpc-gateway/Dockerfile
+++ b/projects/grpc-gateway/Dockerfile
@ -18,4 +18,4 @@ FROM gcr.io/oss-fuzz-base/base-builder
 ENV GO111MODULE on
 RUN git clone https://github.com/grpc-ecosystem/grpc-gateway
 COPY build.sh $SRC/
-WORKDIR $SRC/
+WORKDIR $SRC/grpc-gateway
--- a/projects/grpc-gateway/build.sh
+++ b/projects/grpc-gateway/build.sh
@ -15,10 +15,6 @@
 #
 ################################################################################

-mkdir $GOPATH/src/github.com/grpc-ecosystem
-mv $SRC/grpc-gateway $GOPATH/src/github.com/grpc-ecosystem/
-cd $GOPATH/src/github.com/grpc-ecosystem/grpc-gateway && go get ./...
-
 if [ "$SANITIZER" = "coverage" ]
 then
 	compile_go_fuzzer github.com/grpc-ecosystem/grpc-gateway/internal/httprule Fuzz fuzz gofuzz
--- a/projects/hugo/Dockerfile
+++ b/projects/hugo/Dockerfile
@ -19,4 +19,4 @@ FROM gcr.io/oss-fuzz-base/base-builder
 ENV GO111MODULE on
 RUN git clone https://github.com/gohugoio/hugo
 COPY build.sh $SRC/
-WORKDIR $SRC/
+WORKDIR $SRC/hugo
--- a/projects/hugo/build.sh
+++ b/projects/hugo/build.sh
@ -15,8 +15,5 @@
 #
 ################################################################################

-mkdir $GOPATH/src/github.com/gohugoio
-mv $SRC/hugo $GOPATH/src/github.com/gohugoio/
-cd $GOPATH/src/github.com/gohugoio/hugo

 compile_go_fuzzer github.com/gohugoio/hugo/tpl/transform FuzzMarkdownify fuzzmarkdownify
--- a/projects/ipfs/Dockerfile
+++ b/projects/ipfs/Dockerfile
@ -15,7 +15,7 @@
 ################################################################################

 FROM gcr.io/oss-fuzz-base/base-builder
-RUN go get -t github.com/ipfs/go-datastore
+RUN git clone --depth 1 https://github.com/ipfs/go-datastore

 COPY build.sh $SRC/
-WORKDIR $SRC/
+WORKDIR $SRC/go-datastore
--- a/projects/ipfs/build.sh
+++ b/projects/ipfs/build.sh
@ -15,8 +15,6 @@
 #
 ################################################################################

-cd $GOPATH/src/github.com/ipfs/go-datastore/fuzz
-
 function compile_ds_fuzzer {
  fuzzer=$1

--- a/projects/jsonparser/Dockerfile
+++ b/projects/jsonparser/Dockerfile
@ -15,7 +15,7 @@
 ################################################################################

 FROM gcr.io/oss-fuzz-base/base-builder
-RUN go get github.com/buger/jsonparser
+RUN git clone --depth 1 https://github.com/buger/jsonparser

 COPY build.sh $SRC/
-WORKDIR $SRC/
+WORKDIR $SRC/jsonparser
--- a/projects/jsonparser/build.sh
+++ b/projects/jsonparser/build.sh
@ -15,4 +15,4 @@
 #
 ################################################################################

-$GOPATH/src/github.com/buger/jsonparser/oss-fuzz-build.sh
+./oss-fuzz-build.sh
--- a/projects/kubernetes/Dockerfile
+++ b/projects/kubernetes/Dockerfile
@ -20,7 +20,7 @@ RUN go get github.com/ianlancetaylor/demangle
 RUN git clone --depth 1 https://github.com/kubernetes/kubernetes.git
 RUN git clone --depth 1 https://github.com/google/AFL
 RUN git clone --depth 1 https://github.com/dvyukov/go-fuzz-corpus
-RUN go get k8s.io/kops
+RUN git clone --depth 1 https://github.com/kubernetes/kops
 WORKDIR $SRC/

 COPY build.sh $SRC/
--- a/projects/kubernetes/build.sh
+++ b/projects/kubernetes/build.sh
@ -21,11 +21,13 @@ set -o errexit
 set -x

 # Compile kOps fuzzers
-$GOPATH/src/k8s.io/kops/tests/fuzz/build.sh
-
+(
+cd kops
+./tests/fuzz/build.sh
+)

 # Compile Kubernetes fuzzers
-mv $SRC/kubernetes $GOPATH/src/k8s.io/
+cd $SRC/kubernetes

 function compile_fuzzer {
  local pkg=$1
--- a/projects/loki/Dockerfile
+++ b/projects/loki/Dockerfile
@ -15,6 +15,6 @@
 ################################################################################

 FROM gcr.io/oss-fuzz-base/base-builder
-RUN go get github.com/grafana/loki/pkg/logql/...
+RUN git clone --depth 1 https://github.com/grafana/loki/
 COPY build.sh $SRC/
-WORKDIR $SRC/
+WORKDIR $SRC/loki
--- a/projects/minify/Dockerfile
+++ b/projects/minify/Dockerfile
@ -15,6 +15,6 @@
 ################################################################################

 FROM gcr.io/oss-fuzz-base/base-builder
-RUN go get -u github.com/tdewolff/minify
+RUN git clone --depth 1 https://github.com/tdewolff/minify
 COPY build.sh $SRC/
-WORKDIR $SRC/
+WORKDIR $SRC/minify
--- a/projects/minify/build.sh
+++ b/projects/minify/build.sh
@ -14,4 +14,4 @@
 # limitations under the License.
 #
 ################################################################################
-$GOPATH/src/github.com/tdewolff/minify/tests/oss-fuzz-build.sh
+./tests/oss-fuzz-build.sh
--- a/projects/nats/Dockerfile
+++ b/projects/nats/Dockerfile
@ -15,6 +15,6 @@
 ################################################################################

 FROM gcr.io/oss-fuzz-base/base-builder
-RUN go get github.com/nats-io/nats-server
+RUN git clone --depth 1 https://github.com/nats-io/nats-server
 COPY build.sh $SRC/
-WORKDIR $SRC/
+WORKDIR $SRC/nats-server
--- a/projects/quic-go/Dockerfile
+++ b/projects/quic-go/Dockerfile
@ -16,12 +16,12 @@

 FROM gcr.io/oss-fuzz-base/base-builder

-RUN go get -u -d github.com/marten-seemann/qpack/ && \
-  cd /root/go/src/github.com/marten-seemann/qpack && \
+RUN git clone --depth 1 https://github.com/marten-seemann/qpack/ && \
+  cd qpack && \
  go build

-RUN go get -u -d github.com/lucas-clemente/quic-go/ && \
-  cd /root/go/src/github.com/lucas-clemente/quic-go && \
+RUN git clone --depth 1 https://github.com/lucas-clemente/quic-go/ && \
+  cd quic-go && \
  go build

 COPY build.sh .
--- a/projects/quic-go/build.sh
+++ b/projects/quic-go/build.sh
@ -30,7 +30,7 @@ compile_go_fuzzer github.com/lucas-clemente/quic-go/fuzzing/tokens Fuzz token_fu
 compile_go_fuzzer github.com/lucas-clemente/quic-go/fuzzing/handshake Fuzz handshake_fuzzer

 # generate seed corpora
-go generate $GOPATH/src/github.com/lucas-clemente/quic-go/fuzzing/...
+cd quic-go && go generate fuzzing/...

 zip --quiet -r $OUT/header_fuzzer_seed_corpus.zip $GOPATH/src/github.com/lucas-clemente/quic-go/fuzzing/header/corpus
 zip --quiet -r $OUT/frame_fuzzer_seed_corpus.zip $GOPATH/src/github.com/lucas-clemente/quic-go/fuzzing/frames/corpus
--- a/projects/radon/Dockerfile
+++ b/projects/radon/Dockerfile
@ -15,6 +15,6 @@
 ################################################################################

 FROM gcr.io/oss-fuzz-base/base-builder
-RUN go get github.com/radondb/radon/src/fuzz/sqlparser
+RUN git clone --depth 1 https://github.com/radondb/radon
 COPY build.sh $SRC/
-WORKDIR $SRC/
+WORKDIR $SRC/radon
--- a/projects/syzkaller/Dockerfile
+++ b/projects/syzkaller/Dockerfile
@ -16,11 +16,7 @@

 FROM gcr.io/oss-fuzz-base/base-builder

-RUN go get -u -d github.com/google/syzkaller/prog
+RUN git clone --depth 1 https://github.com/google/syzkaller/

-# Dependency for one of the fuzz targets.
-# Note: this should not be necessary because this package is in syzkaller/vendor.
-RUN go get github.com/ianlancetaylor/demangle
-
-WORKDIR /root/go/src/github.com/google/syzkaller
+WORKDIR $SRC/syzkaller
 COPY build.sh $SRC/
--- a/projects/tidb/Dockerfile
+++ b/projects/tidb/Dockerfile
@ -17,4 +17,4 @@
 FROM gcr.io/oss-fuzz-base/base-builder
 RUN git clone --depth 1 https://github.com/pingcap/tidb
 COPY build.sh $SRC/
-WORKDIR $SRC/
+WORKDIR $SRC/tidb
--- a/projects/tidb/build.sh
+++ b/projects/tidb/build.sh
@ -18,9 +18,7 @@
 # Insert empty main function
 sed -i '23 i\func main(){}'\\n $SRC/tidb/plugin/conn_ip_example/conn_ip_example.go

-mkdir $GOPATH/src/github.com/pingcap
-mv $SRC/tidb $GOPATH/src/github.com/pingcap/
-cd $GOPATH/src/github.com/pingcap/tidb && go get ./...
+go get ./...

 compile_go_fuzzer github.com/pingcap/tidb/types FuzzMarshalJSON fuzzMarshalJSON
 compile_go_fuzzer github.com/pingcap/tidb/types FuzzNewBitLiteral fuzzNewBitLiteral
--- a/projects/vitess/Dockerfile
+++ b/projects/vitess/Dockerfile
@ -15,13 +15,6 @@
 ################################################################################

 FROM gcr.io/oss-fuzz-base/base-builder
-RUN go get 	github.com/vitessio/vitess \
-		vitess.io/vitess/go/bytes2 \
-		vitess.io/vitess/go/sqltypes \
-		vitess.io/vitess/go/vt/log \
-		vitess.io/vitess/go/vt/proto/query \
-		vitess.io/vitess/go/vt/proto/vtrpc \
-		vitess.io/vitess/go/vt/vterrors \
-		vitess.io/vitess/go/vt/vtgate/evalengine
+RUN git clone --depth 1 https://github.com/vitessio/vitess
 COPY build.sh $SRC/
-WORKDIR $SRC/
+WORKDIR $SRC/vitess