From e23701f9d5f3acb60c15ec7c627e2b80b725bb90 Mon Sep 17 00:00:00 2001 From: Arthur Chan Date: Tue, 30 Aug 2022 15:20:14 +0100 Subject: [PATCH] kafka-python: intial integration (#8017) * kafka-python: intial integration Create fuzzers * Fix project initialization --- projects/kafka/Dockerfile | 29 ++++++++++++++++ projects/kafka/build.sh | 23 +++++++++++++ projects/kafka/fuzz_consumer.py | 59 ++++++++++++++++++++++++++++++++ projects/kafka/fuzz_producer.py | 60 +++++++++++++++++++++++++++++++++ projects/kafka/project.yaml | 12 +++++++ 5 files changed, 183 insertions(+) create mode 100644 projects/kafka/Dockerfile create mode 100644 projects/kafka/build.sh create mode 100644 projects/kafka/fuzz_consumer.py create mode 100644 projects/kafka/fuzz_producer.py create mode 100644 projects/kafka/project.yaml diff --git a/projects/kafka/Dockerfile b/projects/kafka/Dockerfile new file mode 100644 index 000000000..dcb29f8d9 --- /dev/null +++ b/projects/kafka/Dockerfile @@ -0,0 +1,29 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +################################################################################ + +FROM gcr.io/oss-fuzz-base/base-builder-python + +RUN apt-get update -y && apt-get install wget +RUN wget -qO - https://packages.confluent.io/deb/7.0/archive.key | apt-key add - +RUN add-apt-repository "deb https://packages.confluent.io/clients/deb $(lsb_release -cs) main" +RUN apt-get update -y && apt-get install librdkafka-dev software-properties-common lsb-release gcc make python3-dev libsasl2-modules-gssapi-mit krb5-user -y + +RUN pip3 install --upgrade pip + +RUN git clone --depth=1 https://github.com/confluentinc/confluent-kafka-python kafka +WORKDIR kafka + +COPY build.sh fuzz_*.py $SRC/ diff --git a/projects/kafka/build.sh b/projects/kafka/build.sh new file mode 100644 index 000000000..7c053d89f --- /dev/null +++ b/projects/kafka/build.sh @@ -0,0 +1,23 @@ +#!/bin/bash -eu +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +################################################################################ + +# Build and install project (using current CFLAGS, CXXFLAGS). +pip3 install . + +for fuzzer in $(find $SRC -name 'fuzz_*.py'); do + compile_python_fuzzer $fuzzer +done diff --git a/projects/kafka/fuzz_consumer.py b/projects/kafka/fuzz_consumer.py new file mode 100644 index 000000000..130355c1d --- /dev/null +++ b/projects/kafka/fuzz_consumer.py @@ -0,0 +1,59 @@ +#!/usr/bin/python3 +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import atheris +import sys +with atheris.instrument_imports(): + from confluent_kafka import Consumer, KafkaException + +def TestInput(data): + fdp = atheris.FuzzedDataProvider(data) + + def dummy_callback(err, partitions): + pass + + c = Consumer({ + 'group.id': fdp.ConsumeString(10), + 'socket.timeout.ms': fdp.ConsumeIntInRange(10,2000), + 'session.timeout.ms': fdp.ConsumeIntInRange(10,2000), + 'on_commit': dummy_callback}) + + try: + c.subscribe([fdp.ConsumeString(10)], on_assign=dummy_callback, on_revoke=dummy_callback) + c.unsubscribe() + + msg = c.poll(timeout=0.001) + msglist = c.consume(num_messages=fdp.ConsumeIntInRange(1,10), timeout=0.001) + + partitions = list(map(lambda part: TopicPartition(fdp.ConsumeString(10), part), range(0, 100, 3))) + c.assign(partitions) + c.unassign() + + c.commit(asynchronous=fdp.ConsumeBool()) + c.committed(partitions, timeout=0.001) + + c.list_topics(timeout=0.2) + c.list_topics(topic=fdp.ConsumeString(10), timeout=0.1) + except KafkaException as e: + pass + + c.close() + +def main(): + atheris.Setup(sys.argv, TestInput, enable_python_coverage=True) + atheris.Fuzz() + +if __name__ == "__main__": + main() diff --git a/projects/kafka/fuzz_producer.py b/projects/kafka/fuzz_producer.py new file mode 100644 index 000000000..8a83219f2 --- /dev/null +++ b/projects/kafka/fuzz_producer.py @@ -0,0 +1,60 @@ +#!/usr/bin/python3 +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import atheris +import sys +with atheris.instrument_imports(): + from confluent_kafka import Producer, KafkaException + +def TestInput(data): + fdp = atheris.FuzzedDataProvider(data) + + p = Producer({ + 'socket.timeout.ms': fdp.ConsumeIntInRange(10,2000), + 'message.timeout.ms': fdp.ConsumeIntInRange(10,2000) + }) + + p.produce(fdp.ConsumeString(20)) + p.produce( + fdp.ConsumeString(20), + value=fdp.ConsumeString(20), + key=fdp.ConsumeString(20) + ) + + def on_delivery(err, msg): + pass + + p.produce( + topic=fdp.ConsumeString(20), + value=fdp.ConsumeString(20), + partition=fdp.ConsumeIntInRange(1,10), + callback=on_delivery + ) + + p.poll(0.001) + p.flush(0.002) + p.flush() + + try: + p.list_topics(timeout=0.2) + except KafkaException as e: + pass + +def main(): + atheris.Setup(sys.argv, TestInput, enable_python_coverage=True) + atheris.Fuzz() + +if __name__ == "__main__": + main() diff --git a/projects/kafka/project.yaml b/projects/kafka/project.yaml new file mode 100644 index 000000000..010956622 --- /dev/null +++ b/projects/kafka/project.yaml @@ -0,0 +1,12 @@ +fuzzing_engines: +- libfuzzer +homepage: https://github.com/confluentinc/confluent-kafka-python +language: python +main_repo: https://github.com/confluentinc/confluent-kafka-python +sanitizers: +- address +- undefined +vendor_ccs: +- david@adalogics.com +- adam@adalogics.com +- arthur.chan@adalogics.com