From 7751ab5a87907ec971beace9d4d139aaab6287cf Mon Sep 17 00:00:00 2001 From: Max Moroz Date: Thu, 30 Jan 2020 15:36:44 -0800 Subject: [PATCH] [infra] Introduce "language" attribute in the project.yaml (#3297). (#3299) * [infra] Introduce "language" attribute in the project.yaml (#3297). * follow up * enable the attribute for more projects * trailing newline --- .../new-project-guide/go_lang.md | 13 ++++++++--- infra/gcb/build_and_run_coverage.py | 12 +++++----- infra/gcb/build_project.py | 1 + infra/presubmit.py | 22 ++++++++++++++++--- projects/go-attestation/project.yaml | 1 + projects/go-json-iterator/project.yaml | 2 +- projects/golang-protobuf/project.yaml | 1 + projects/golang/project.yaml | 1 + projects/gonids/project.yaml | 2 +- projects/kubernetes/project.yaml | 1 + projects/mtail/project.yaml | 1 + projects/syzkaller/project.yaml | 1 + projects/wasmtime/project.yaml | 1 + 13 files changed, 46 insertions(+), 13 deletions(-) diff --git a/docs/getting-started/new-project-guide/go_lang.md b/docs/getting-started/new-project-guide/go_lang.md index 800485ca5..939bae739 100644 --- a/docs/getting-started/new-project-guide/go_lang.md +++ b/docs/getting-started/new-project-guide/go_lang.md @@ -30,12 +30,19 @@ libFuzzer command line interface as non-Go fuzz targets. ## Project files The structure of the project directory in OSS-Fuzz repository doesn't differ for -projects written in Go. The project files have the following Go specific aspects. +projects written in Go. The project files have the following Go specific +aspects. ### project.yaml -For projects written in Go, we use only `libfuzzer` fuzzing engine and `address` -sanitizer. +The `language` attribute must be specified. + +```yaml +language: go +``` + +The only supported fuzzing engine and sanitizer are `libfuzzer` and `address`, +respectively. [Example](https://github.com/google/oss-fuzz/blob/356f2b947670b7eb33a1f535c71bc5c87a60b0d1/projects/syzkaller/project.yaml#L7): ```yaml diff --git a/infra/gcb/build_and_run_coverage.py b/infra/gcb/build_and_run_coverage.py index d7599ba7c..b94fe3558 100644 --- a/infra/gcb/build_and_run_coverage.py +++ b/infra/gcb/build_and_run_coverage.py @@ -34,8 +34,8 @@ LATEST_REPORT_INFO_URL = ('/' + COVERAGE_BUCKET_NAME + # Link where to upload code coverage report files to. UPLOAD_URL_FORMAT = 'gs://' + COVERAGE_BUCKET_NAME + '/{project}/{type}/{date}' -# TODO(#2817): Support code coverage for Go projects. -GO_FUZZ_BUILD = 'go-fuzz-build -libfuzzer' +# Languages from project.yaml that have code coverage support. +LANGUAGES_WITH_COVERAGE_SUPPORT = ['c', 'cpp'] def skip_build(message): @@ -61,9 +61,11 @@ def get_build_steps(project_dir): build_script_path = os.path.join(project_dir, 'build.sh') if os.path.exists(build_script_path): with open(build_script_path) as fh: - if GO_FUZZ_BUILD in fh.read(): - skip_build('Project "%s" uses go-fuzz, coverage is not supported yet.' % - project_name) + if project_yaml['language'] not in LANGUAGES_WITH_COVERAGE_SUPPORT: + skip_build(('Project "{project_name}" is written in "{language}", ' + 'coverage is not supported yet.').format( + project_name=project_name, + language=project_yaml['language'])) dockerfile_path = os.path.join(project_dir, 'Dockerfile') name = project_yaml['name'] diff --git a/infra/gcb/build_project.py b/infra/gcb/build_project.py index 569c7dbe4..f45b0996a 100644 --- a/infra/gcb/build_project.py +++ b/infra/gcb/build_project.py @@ -58,6 +58,7 @@ def load_project_yaml(project_dir): project_yaml.setdefault('run_tests', True) project_yaml.setdefault('coverage_extra_args', '') project_yaml.setdefault('labels', {}) + project_yaml.setdefault('language', 'cpp') return project_yaml diff --git a/infra/presubmit.py b/infra/presubmit.py index 54c0efc23..7be16a80f 100755 --- a/infra/presubmit.py +++ b/infra/presubmit.py @@ -74,7 +74,7 @@ class ProjectYamlChecker: SECTIONS_AND_CONSTANTS = { 'sanitizers': {'address', 'none', 'memory', 'undefined', 'dataflow'}, 'architectures': {'i386', 'x86_64'}, - 'fuzzing_engines': {'afl', 'libfuzzer', 'honggfuzz', 'dataflow'} + 'fuzzing_engines': {'afl', 'libfuzzer', 'honggfuzz', 'dataflow'}, } # Note: this list must be updated when we allow new sections. @@ -89,8 +89,11 @@ class ProjectYamlChecker: 'sanitizers', 'vendor_ccs', 'view_restrictions', + 'language', ] + LANGUAGES_SUPPORTED = ['c', 'cpp', 'go', 'rust', 'python'] + # Note that some projects like boost only have auto-ccs. However, forgetting # primary contact is probably a mistake. REQUIRED_SECTIONS = ['primary_contact'] @@ -108,8 +111,11 @@ class ProjectYamlChecker: return True checks = [ - self.check_project_yaml_constants, self.check_required_sections, - self.check_valid_section_names, self.check_valid_emails + self.check_project_yaml_constants, + self.check_required_sections, + self.check_valid_section_names, + self.check_valid_emails, + self.check_valid_language, ] for check_function in checks: check_function() @@ -179,6 +185,16 @@ class ProjectYamlChecker: if '@' not in email_address or '.' not in email_address: self.error(email_address + ' is an invalid email address.') + def check_valid_language(self): + """Check that the language specified is valid.""" + language = self.data.get('language') + if not language: + return + + if language not in self.LANGUAGES_SUPPORTED: + self.error('{language} is not supported ({supported}).'.format( + language=language, supported=self.LANGUAGES_SUPPORTED)) + def _check_one_project_yaml(project_yaml_filename): """Do checks on the project.yaml file.""" diff --git a/projects/go-attestation/project.yaml b/projects/go-attestation/project.yaml index ac3f56bf2..3dd47d8c1 100644 --- a/projects/go-attestation/project.yaml +++ b/projects/go-attestation/project.yaml @@ -7,3 +7,4 @@ fuzzing_engines: - libfuzzer sanitizers: - address +language: go diff --git a/projects/go-json-iterator/project.yaml b/projects/go-json-iterator/project.yaml index 2fc93ba65..101f0d44b 100644 --- a/projects/go-json-iterator/project.yaml +++ b/projects/go-json-iterator/project.yaml @@ -1,7 +1,7 @@ homepage: "https://jsoniter.com" primary_contact: "taowen@gmail.com" auto_ccs : "p.antoine@catenacyber.fr" - +language: go fuzzing_engines: - libfuzzer sanitizers: diff --git a/projects/golang-protobuf/project.yaml b/projects/golang-protobuf/project.yaml index 7c49f77e6..71ee2df12 100644 --- a/projects/golang-protobuf/project.yaml +++ b/projects/golang-protobuf/project.yaml @@ -6,3 +6,4 @@ sanitizers: - address fuzzing_engines: - libfuzzer +language: go diff --git a/projects/golang/project.yaml b/projects/golang/project.yaml index 2fe5b28dc..939f457ae 100644 --- a/projects/golang/project.yaml +++ b/projects/golang/project.yaml @@ -4,6 +4,7 @@ auto_ccs: - "golang-fuzz@googlegroups.com" - "mmoroz@chromium.org" - "josharian@gmail.com" +language: go sanitizers: - address fuzzing_engines: diff --git a/projects/gonids/project.yaml b/projects/gonids/project.yaml index 19e44019c..2b360887c 100644 --- a/projects/gonids/project.yaml +++ b/projects/gonids/project.yaml @@ -1,7 +1,7 @@ homepage: "https://github.com/google/gonids" primary_contact: "duane.security@gmail.com" auto_ccs : "p.antoine@catenacyber.fr" - +language: go fuzzing_engines: - libfuzzer sanitizers: diff --git a/projects/kubernetes/project.yaml b/projects/kubernetes/project.yaml index 695a571ec..e834026f9 100644 --- a/projects/kubernetes/project.yaml +++ b/projects/kubernetes/project.yaml @@ -4,3 +4,4 @@ fuzzing_engines: - libfuzzer sanitizers: - address +language: go diff --git a/projects/mtail/project.yaml b/projects/mtail/project.yaml index 3878cca55..af7d2d4fe 100644 --- a/projects/mtail/project.yaml +++ b/projects/mtail/project.yaml @@ -4,3 +4,4 @@ fuzzing_engines: - libfuzzer sanitizers: - address +language: go diff --git a/projects/syzkaller/project.yaml b/projects/syzkaller/project.yaml index b208801a9..ee6cd841d 100644 --- a/projects/syzkaller/project.yaml +++ b/projects/syzkaller/project.yaml @@ -4,6 +4,7 @@ auto_ccs: - "andreyknvl@google.com" - "mmoroz@chromium.org" - "syzkaller@googlegroups.com" +language: go fuzzing_engines: - libfuzzer sanitizers: diff --git a/projects/wasmtime/project.yaml b/projects/wasmtime/project.yaml index bacd1ff92..508523f57 100644 --- a/projects/wasmtime/project.yaml +++ b/projects/wasmtime/project.yaml @@ -8,3 +8,4 @@ sanitizers: - address fuzzing_engines: - libfuzzer +language: rust