compress: initial integration (#9868)

@klauspost - I have the build working with all fuzzers except the `s2`
fuzzers which require more work. I would prefer to get the fuzzers
running first without those first and include them later.

Your fuzzing suite does some reading of files on disk at runtime which
won't work in OSS-Fuzz. I have mitigated that by creating a slice of all
the files in a new file that gets created in `./zstd` which is available
to the fuzzer at runtime. See `build.sh` for details.

Can you please leave your email address in this PR to get it finalized?

---------

Signed-off-by: AdamKorcz <adam@adalogics.com>

projects/compress/Dockerfile

@@ -0,0 +1,20 @@
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+################################################################################
+
+FROM gcr.io/oss-fuzz-base/base-builder-go
+RUN git clone --depth 1 https://github.com/klauspost/compress
+WORKDIR compress
+COPY build.sh setup_dicts.go $SRC/

projects/compress/build.sh

@@ -0,0 +1,72 @@
+#!/bin/bash -eu
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+################################################################################
+
+# In one of the Zstd fuzzers, the "dict" variable is created by reading a series of files.
+# These files are not available at runtime in the OSS-FUzz environment,
+# so we add the contents of these files to a variable and create a new file
+# that is included when we build the fuzzers in OSS-Fuzz.
+mkdir $SRC/setupdicts
+cp $SRC/setup_dicts.go $SRC/setupdicts/main.go
+cd $SRC/setupdicts
+go mod init setupdicts
+go mod tidy
+go run main.go --dict-path=$SRC/compress/zstd/testdata/dict-tests-small.zip --output-file=$SRC/compress/zstd/fuzzDicts.go
+cp $SRC/compress/zstd/fuzzDicts.go $OUT/
+# Done creating "dicts" variable.
+
+cd $SRC/compress
+
+# Temporarily use a fork of go-fuzz-headers with some improvements that has not been merged yet.
+go mod edit -replace github.com/AdaLogics/go-fuzz-headers=github.com/AdamKorcz/go-fuzz-headers-1@22e92b7968997eabd210694dd4825dd0d19b697c
+
+# Modify some files. This would be better done upstream.
+sed -i '28 a\
+	if fi == nil { return }' $SRC/compress/internal/fuzz/helpers.go
+printf "package compress\nimport _ \"github.com/AdamKorcz/go-118-fuzz-build/testing\"\n" > registerfuzzdependency.go
+sed -i 's/zr := testCreateZipReader/\/\/zr := testCreateZipReader/g' "${SRC}"/compress/zstd/fuzz_test.go
+sed -i 's/dicts = readDicts(f, zr)/dicts = fuzzDicts/g' "${SRC}"/compress/zstd/fuzz_test.go
+
+if [ "$SANITIZER" != "coverage" ]; then
+	sed -i 's/\"testing\"/\"github.com\/AdamKorcz\/go-118-fuzz-build\/testing\"/g' "${SRC}"/compress/internal/fuzz/helpers.go
+fi
+
+# OSS-Fuzz uses 'go build' to build the fuzzers, so we move the tests
+# we need into scope.
+mv $SRC/compress/zstd/decoder_test.go $SRC/compress/zstd/decoder_test_fuzz.go
+mv $SRC/compress/zstd/zstd_test.go $SRC/compress/zstd/zstd_test_fuzz.go
+mv $SRC/compress/zstd/seqdec_test.go $SRC/compress/zstd/seqdec_test_fuzz.go
+mv $SRC/compress/zstd/dict_test.go $SRC/compress/zstd/dict_test_fuzz.go
+mv $SRC/compress/s2/s2_test.go $SRC/compress/s2/s2_test_fuzz.go
+go mod tidy
+
+# Build fuzzers
+compile_native_go_fuzzer github.com/klauspost/compress/flate FuzzEncoding FuzzFlateEncoding
+compile_native_go_fuzzer github.com/klauspost/compress/zstd FuzzDecodeAll FuzzDecodeAll
+compile_native_go_fuzzer github.com/klauspost/compress/zstd FuzzDecAllNoBMI2 FuzzDecAllNoBMI2
+compile_native_go_fuzzer github.com/klauspost/compress/zstd FuzzDecoder FuzzDecoder
+compile_native_go_fuzzer github.com/klauspost/compress/zstd FuzzNoBMI2Dec FuzzNoBMI2Dec
+compile_native_go_fuzzer github.com/klauspost/compress/zstd FuzzEncoding FuzzZstdEncoding
+#compile_native_go_fuzzer github.com/klauspost/compress/s2 FuzzLZ4Block FuzzLZ4Block
+#compile_native_go_fuzzer github.com/klauspost/compress/s2 FuzzDictBlocks FuzzDictBlocks
+#compile_native_go_fuzzer github.com/klauspost/compress/s2 FuzzEncodingBlocks FuzzEncodingBlocks
+compile_native_go_fuzzer github.com/klauspost/compress/zip FuzzReader FuzzReader
+
+# Add corpora
+cp $SRC/compress/zstd/testdata/fuzz/encode-corpus-raw.zip $OUT/FuzzZstdEncoding_seed_corpus.zip
+cp $SRC/compress/zstd/testdata/fuzz/decode-corpus-raw.zip $OUT/FuzzDecodeAll_seed_corpus.zip
+cp $SRC/compress/zstd/testdata/fuzz/decode-corpus-raw.zip $OUT/FuzzDecoder_seed_corpus.zip
+cp $SRC/compress/zip/testdata/FuzzReader-raw.zip $OUT/FuzzReader_seed_corpus.zip

projects/compress/project.yaml

@@ -0,0 +1,10 @@
+homepage: "https://github.com/klauspost/compress"
+language: go
+primary_contact: "klauspost@gmail.com"
+main_repo: "https://github.com/klauspost/compress"
+auto_ccs:
+  - "adam@adalogics.com"
+fuzzing_engines:
+  - libfuzzer
+sanitizers:
+  - address

projects/compress/setup_dicts.go

@@ -0,0 +1,100 @@
+// Copyright 2023 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"bytes"
+	"flag"
+	"fmt"
+	"io"
+	"os"
+	"strings"
+	"text/template"
+
+	"github.com/klauspost/compress/zip"
+)
+
+var (
+	dictPath   = flag.String("dict-path", "", "dict path")
+	outputFile = flag.String("output-file", "", "output file")
+)
+
+func main() {
+	flag.Parse()
+	if *dictPath == "" {
+		panic("Need a dict path")
+	}
+	if *outputFile == "" {
+		panic("Need an output file")
+	}
+	dicts := getFuzzDicts(*dictPath)
+
+	t, err := template.New("todos").Parse(`
+package zstd
+var fuzzDicts = make([][]byte, 0)
+func init() {
+{{range $val := .}}
+    fuzzDicts = append(fuzzDicts, {{$val}})
+{{end}}
+}
+`)
+	if err != nil {
+		panic(err)
+	}
+	f, err := os.Create(*outputFile)
+	err = t.Execute(f, dicts)
+	if err != nil {
+		panic(err)
+	}
+	f.Close()
+}
+
+func getFuzzDicts(path string) []string {
+	data, err := os.ReadFile(path)
+	if err != nil {
+		panic(err)
+	}
+	zr, err := zip.NewReader(bytes.NewReader(data), int64(len(data)))
+	if err != nil {
+		panic(err)
+	}
+	var dicts [][]byte
+	for _, tt := range zr.File {
+		if !strings.HasSuffix(tt.Name, ".dict") {
+			continue
+		}
+		func() {
+			r, err := tt.Open()
+			if err != nil {
+				panic(err)
+			}
+			defer r.Close()
+			in, err := io.ReadAll(r)
+			if err != nil {
+				panic(err)
+			}
+			dicts = append(dicts, in)
+		}()
+	}
+	stringDicts := make([]string, 0)
+	for _, d := range dicts {
+		stringedArray := fmt.Sprintf("%v", d)
+		withComma := strings.Replace(stringedArray, " ", ", ", -1)
+		withClosingBracket := strings.Replace(withComma, "]", "}", -1)
+		withOpenBracket := strings.Replace(withClosingBracket, "[", "[]byte{", -1)
+		stringDicts = append(stringDicts, withOpenBracket)
+	}
+	return stringDicts
+}