From 276b1bc342d23142e4b2b9b9fadbf076474deec9 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Fri, 26 Mar 2021 04:12:35 +0900 Subject: [PATCH] [grpc] Support latest version of grpc PoC (#6338) * use grpcpp rather than deprecated grpc++ * grpcpp Deserialize take ByteBuffer instead of grpc_byte_buffer * grpc 1.36.0 * Fix to use grpc 1.36.0 * Fix CMakeLists to refer gRPC * add find_package(Threads) * Update bazel deps * Apply workaround about boringssl build error * Install latest cmake for osx on travis-ci Co-authored-by: Yeom Gyusun --- .travis.yml | 6 ++ CMakeLists.txt | 7 +- WORKSPACE | 17 ++-- bazel/grpc_deps_1_15_1.patch | 155 ----------------------------- bazel/grpc_deps_1_15_1_02.patch | 48 --------- bazel/protobuf_deps_3_6_1.patch | 24 ----- grpc/boringssl.patch | 13 +++ grpc/build_grpc.sh | 14 ++- grpc/samples/greeter/client.cpp | 2 +- grpc/samples/greeter/server.cpp | 2 +- grpc/src/compiler/cpp_generator.cc | 38 +++---- grpc/tests/grpctest.cpp | 3 +- include/flatbuffers/grpc.h | 5 +- tests/monster_test.grpc.fb.cc | 16 +-- tests/monster_test.grpc.fb.h | 18 ++-- 15 files changed, 83 insertions(+), 285 deletions(-) delete mode 100644 bazel/grpc_deps_1_15_1.patch delete mode 100644 bazel/grpc_deps_1_15_1_02.patch delete mode 100644 bazel/protobuf_deps_3_6_1.patch create mode 100644 grpc/boringssl.patch diff --git a/.travis.yml b/.travis.yml index c3c134651..8736f2d51 100644 --- a/.travis.yml +++ b/.travis.yml @@ -104,6 +104,7 @@ matrix: - if [ "$TRAVIS_OS_NAME" == "linux" ]; then sudo ln -s -v -f $(which gcc-$GCC_VERSION) /usr/bin/gcc; fi script: + - pip install cmake - bash .travis/check-sources.sh - bash grpc/build_grpc.sh - cmake . @@ -125,6 +126,11 @@ matrix: - BUILD_TYPE=Release script: + - pip install --user cmake + - mkdir ~/cmake_path + - ln -s $(find ~/Library/Python -name cmake -type f | head -n 1) ~/cmake_path/cmake + - ln -s $(find ~/Library/Python -name ctest -type f | head -n 1) ~/cmake_path/ctest + - export PATH=~/cmake_path:${PATH} - bash grpc/build_grpc.sh - cmake . -DCMAKE_BUILD_TYPE=$BUILD_TYPE diff --git a/CMakeLists.txt b/CMakeLists.txt index ba14f7c26..1dc6465f6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -543,10 +543,13 @@ if(FLATBUFFERS_BUILD_GRPCTEST) endif() INCLUDE_DIRECTORIES(${GRPC_INSTALL_PATH}/include) INCLUDE_DIRECTORIES(${PROTOBUF_DOWNLOAD_PATH}/src) - LINK_DIRECTORIES(${GRPC_INSTALL_PATH}/lib) + find_package(Threads REQUIRED) + list(APPEND CMAKE_PREFIX_PATH ${GRPC_INSTALL_PATH}) + find_package(protobuf CONFIG REQUIRED) + find_package(gRPC CONFIG REQUIRED) add_executable(grpctest ${FlatBuffers_GRPCTest_SRCS}) add_dependencies(grpctest generated_code) - target_link_libraries(grpctest PRIVATE grpc++_unsecure grpc_unsecure gpr pthread dl) + target_link_libraries(grpctest PRIVATE gRPC::grpc++_unsecure gRPC::grpc_unsecure gRPC::gpr pthread dl) if(FLATBUFFERS_CODE_SANITIZE AND NOT WIN32) # GRPC test has problems with alignment and will fail under ASAN/UBSAN. # add_fsanitize_to_target(grpctest ${FLATBUFFERS_CODE_SANITIZE}) diff --git a/WORKSPACE b/WORKSPACE index d856cc683..c3bfdbea2 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -18,14 +18,10 @@ go_rules_dependencies() go_register_toolchains() ##### Protobuf -_PROTOBUF_VERSION = "3.6.1" +_PROTOBUF_VERSION = "3.15.2" http_archive( name = "com_google_protobuf", - patch_args = ["-p1"], - patches = [ - "//bazel:protobuf_deps_3_6_1.patch", - ], strip_prefix = "protobuf-" + _PROTOBUF_VERSION, urls = [ "https://github.com/protocolbuffers/protobuf/archive/v" + _PROTOBUF_VERSION + ".tar.gz", @@ -33,15 +29,10 @@ http_archive( ) ##### GRPC -_GRPC_VERSION = "1.15.1" +_GRPC_VERSION = "1.36.1" http_archive( name = "com_github_grpc_grpc", - patch_args = ["-p1"], - patches = [ - "//bazel:grpc_deps_1_15_1.patch", - "//bazel:grpc_deps_1_15_1_02.patch", - ], strip_prefix = "grpc-" + _GRPC_VERSION, urls = ["https://github.com/grpc/grpc/archive/v" + _GRPC_VERSION + ".tar.gz"], ) @@ -49,3 +40,7 @@ http_archive( load("@com_github_grpc_grpc//bazel:grpc_deps.bzl", "grpc_deps") grpc_deps() + +load("@com_github_grpc_grpc//bazel:grpc_extra_deps.bzl", "grpc_extra_deps") + +grpc_extra_deps() diff --git a/bazel/grpc_deps_1_15_1.patch b/bazel/grpc_deps_1_15_1.patch deleted file mode 100644 index c5d010119..000000000 --- a/bazel/grpc_deps_1_15_1.patch +++ /dev/null @@ -1,155 +0,0 @@ -diff --git a/bazel/grpc_deps.bzl b/bazel/grpc_deps.bzl -index 4096720569..ea411b577d 100644 ---- a/bazel/grpc_deps.bzl -+++ b/bazel/grpc_deps.bzl -@@ -1,5 +1,10 @@ - """Load dependencies needed to compile and test the grpc library as a 3rd-party consumer.""" - -+# native.http_archive has been deprecated in newer bazel version. In order to work with -+# grpc 1.15.1, the version flatbuffers is currently replying on. We need to apply this patch -+# when importing the grpc dependency. -+load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") -+ - def grpc_deps(): - """Loads dependencies need to compile and test the grpc library.""" - -@@ -99,14 +104,14 @@ def grpc_deps(): - ) - - if "boringssl" not in native.existing_rules(): -- native.http_archive( -+ http_archive( - name = "boringssl", - # on the chromium-stable-with-bazel branch - url = "https://boringssl.googlesource.com/boringssl/+archive/dcd3e6e6ecddf059adb48fca45bc7346a108bdd9.tar.gz", - ) - - if "com_github_madler_zlib" not in native.existing_rules(): -- native.new_http_archive( -+ http_archive( - name = "com_github_madler_zlib", - build_file = "@com_github_grpc_grpc//third_party:zlib.BUILD", - strip_prefix = "zlib-cacf7f1d4e3d44d871b605da3b647f07d718623f", -@@ -114,14 +119,14 @@ def grpc_deps(): - ) - - if "com_google_protobuf" not in native.existing_rules(): -- native.http_archive( -+ http_archive( - name = "com_google_protobuf", - strip_prefix = "protobuf-48cb18e5c419ddd23d9badcfe4e9df7bde1979b2", - url = "https://github.com/google/protobuf/archive/48cb18e5c419ddd23d9badcfe4e9df7bde1979b2.tar.gz", - ) - - if "com_github_nanopb_nanopb" not in native.existing_rules(): -- native.new_http_archive( -+ http_archive( - name = "com_github_nanopb_nanopb", - build_file = "@com_github_grpc_grpc//third_party:nanopb.BUILD", - strip_prefix = "nanopb-f8ac463766281625ad710900479130c7fcb4d63b", -@@ -129,7 +134,7 @@ def grpc_deps(): - ) - - if "com_github_google_googletest" not in native.existing_rules(): -- native.new_http_archive( -+ http_archive( - name = "com_github_google_googletest", - build_file = "@com_github_grpc_grpc//third_party:gtest.BUILD", - strip_prefix = "googletest-ec44c6c1675c25b9827aacd08c02433cccde7780", -@@ -137,14 +142,14 @@ def grpc_deps(): - ) - - if "com_github_gflags_gflags" not in native.existing_rules(): -- native.http_archive( -+ http_archive( - name = "com_github_gflags_gflags", - strip_prefix = "gflags-30dbc81fb5ffdc98ea9b14b1918bfe4e8779b26e", - url = "https://github.com/gflags/gflags/archive/30dbc81fb5ffdc98ea9b14b1918bfe4e8779b26e.tar.gz", - ) - - if "com_github_google_benchmark" not in native.existing_rules(): -- native.new_http_archive( -+ http_archive( - name = "com_github_google_benchmark", - build_file = "@com_github_grpc_grpc//third_party:benchmark.BUILD", - strip_prefix = "benchmark-9913418d323e64a0111ca0da81388260c2bbe1e9", -@@ -152,7 +157,7 @@ def grpc_deps(): - ) - - if "com_github_cares_cares" not in native.existing_rules(): -- native.new_http_archive( -+ http_archive( - name = "com_github_cares_cares", - build_file = "@com_github_grpc_grpc//third_party:cares/cares.BUILD", - strip_prefix = "c-ares-3be1924221e1326df520f8498d704a5c4c8d0cce", -@@ -160,14 +165,14 @@ def grpc_deps(): - ) - - if "com_google_absl" not in native.existing_rules(): -- native.http_archive( -+ http_archive( - name = "com_google_absl", - strip_prefix = "abseil-cpp-cd95e71df6eaf8f2a282b1da556c2cf1c9b09207", - url = "https://github.com/abseil/abseil-cpp/archive/cd95e71df6eaf8f2a282b1da556c2cf1c9b09207.tar.gz", - ) - - if "com_github_bazelbuild_bazeltoolchains" not in native.existing_rules(): -- native.http_archive( -+ http_archive( - name = "com_github_bazelbuild_bazeltoolchains", - strip_prefix = "bazel-toolchains-cdea5b8675914d0a354d89f108de5d28e54e0edc", - urls = [ -@@ -178,7 +183,7 @@ def grpc_deps(): - ) - - if "io_opencensus_cpp" not in native.existing_rules(): -- native.http_archive( -+ http_archive( - name = "io_opencensus_cpp", - strip_prefix = "opencensus-cpp-fdf0f308b1631bb4a942e32ba5d22536a6170274", - url = "https://github.com/census-instrumentation/opencensus-cpp/archive/fdf0f308b1631bb4a942e32ba5d22536a6170274.tar.gz", -@@ -200,7 +205,7 @@ def grpc_test_only_deps(): - ) - - if "com_github_twisted_twisted" not in native.existing_rules(): -- native.new_http_archive( -+ http_archive( - name = "com_github_twisted_twisted", - strip_prefix = "twisted-twisted-17.5.0", - url = "https://github.com/twisted/twisted/archive/twisted-17.5.0.zip", -@@ -208,7 +213,7 @@ def grpc_test_only_deps(): - ) - - if "com_github_yaml_pyyaml" not in native.existing_rules(): -- native.new_http_archive( -+ http_archive( - name = "com_github_yaml_pyyaml", - strip_prefix = "pyyaml-3.12", - url = "https://github.com/yaml/pyyaml/archive/3.12.zip", -@@ -216,7 +221,7 @@ def grpc_test_only_deps(): - ) - - if "com_github_twisted_incremental" not in native.existing_rules(): -- native.new_http_archive( -+ http_archive( - name = "com_github_twisted_incremental", - strip_prefix = "incremental-incremental-17.5.0", - url = "https://github.com/twisted/incremental/archive/incremental-17.5.0.zip", -@@ -224,7 +229,7 @@ def grpc_test_only_deps(): - ) - - if "com_github_zopefoundation_zope_interface" not in native.existing_rules(): -- native.new_http_archive( -+ http_archive( - name = "com_github_zopefoundation_zope_interface", - strip_prefix = "zope.interface-4.4.3", - url = "https://github.com/zopefoundation/zope.interface/archive/4.4.3.zip", -@@ -232,7 +237,7 @@ def grpc_test_only_deps(): - ) - - if "com_github_twisted_constantly" not in native.existing_rules(): -- native.new_http_archive( -+ http_archive( - name = "com_github_twisted_constantly", - strip_prefix = "constantly-15.1.0", - url = "https://github.com/twisted/constantly/archive/15.1.0.zip", diff --git a/bazel/grpc_deps_1_15_1_02.patch b/bazel/grpc_deps_1_15_1_02.patch deleted file mode 100644 index 328134ea8..000000000 --- a/bazel/grpc_deps_1_15_1_02.patch +++ /dev/null @@ -1,48 +0,0 @@ -diff --git a/bazel/cc_grpc_library.bzl b/bazel/cc_grpc_library.bzl -index 3288565714..6bfcd653f5 100644 ---- a/bazel/cc_grpc_library.bzl -+++ b/bazel/cc_grpc_library.bzl -@@ -1,6 +1,6 @@ - """Generates and compiles C++ grpc stubs from proto_library rules.""" - --load("//:bazel/generate_cc.bzl", "generate_cc") -+load("//bazel:generate_cc.bzl", "generate_cc") - - def cc_grpc_library(name, srcs, deps, proto_only, well_known_protos, generate_mocks = False, use_external = False, **kwargs): - """Generates C++ grpc classes from a .proto file. -diff --git a/bazel/generate_cc.bzl b/bazel/generate_cc.bzl -index ae747aa42c..2f14071f92 100644 ---- a/bazel/generate_cc.bzl -+++ b/bazel/generate_cc.bzl -@@ -83,7 +83,7 @@ _generate_cc = rule( - attrs = { - "srcs": attr.label_list( - mandatory = True, -- non_empty = True, -+ allow_empty = False, - providers = ["proto"], - ), - "plugin": attr.label( -diff --git a/bazel/grpc_build_system.bzl b/bazel/grpc_build_system.bzl -index 73147bf3ac..e8c392aa93 100644 ---- a/bazel/grpc_build_system.bzl -+++ b/bazel/grpc_build_system.bzl -@@ -24,6 +24,9 @@ - # - - # The set of pollers to test against if a test exercises polling -+ -+load("//bazel:cc_grpc_library.bzl", "cc_grpc_library") -+ - POLLERS = ['epollex', 'epollsig', 'epoll1', 'poll', 'poll-cv'] - - def if_not_windows(a): -@@ -95,8 +98,6 @@ def grpc_proto_plugin(name, srcs = [], deps = []): - deps = deps, - ) - --load("//:bazel/cc_grpc_library.bzl", "cc_grpc_library") -- - def grpc_proto_library(name, srcs = [], deps = [], well_known_protos = False, - has_services = True, use_external = False, generate_mocks = False): - cc_grpc_library( diff --git a/bazel/protobuf_deps_3_6_1.patch b/bazel/protobuf_deps_3_6_1.patch deleted file mode 100644 index 1c85062d2..000000000 --- a/bazel/protobuf_deps_3_6_1.patch +++ /dev/null @@ -1,24 +0,0 @@ -diff --git a/protobuf.bzl b/protobuf.bzl -index 78f19c621..3b6d183b9 100644 ---- a/protobuf.bzl -+++ b/protobuf.bzl -@@ -130,7 +130,7 @@ proto_gen = rule( - "protoc": attr.label( - cfg = "host", - executable = True, -- single_file = True, -+ allow_single_file = True, - mandatory = True, - ), - "plugin": attr.label( -@@ -266,8 +266,8 @@ def internal_gen_well_known_protos_java(srcs): - Args: - srcs: the well known protos - """ -- root = Label("%s//protobuf_java" % (REPOSITORY_NAME)).workspace_root -- pkg = PACKAGE_NAME + "/" if PACKAGE_NAME else "" -+ root = Label("%s//protobuf_java" % (native.repository_name())).workspace_root -+ pkg = native.package_name() + "/" if native.package_name() else "" - if root == "": - include = " -I%ssrc " % pkg - else: diff --git a/grpc/boringssl.patch b/grpc/boringssl.patch new file mode 100644 index 000000000..632499e28 --- /dev/null +++ b/grpc/boringssl.patch @@ -0,0 +1,13 @@ +diff --git a/CMakeLists.txt b/CMakeLists.txt +index 1645a264a..12f8ca999 100644 +--- a/CMakeLists.txt ++++ b/CMakeLists.txt +@@ -635,6 +635,8 @@ add_library( + src/ssl/tls_record.cc + ) + ++target_link_libraries(ssl crypto) ++ + add_executable( + bssl + diff --git a/grpc/build_grpc.sh b/grpc/build_grpc.sh index 3c0c27e1a..190a1ef96 100755 --- a/grpc/build_grpc.sh +++ b/grpc/build_grpc.sh @@ -1,14 +1,20 @@ #!/bin/bash -grpc_1_15_1_githash=1a60e6971f428323245a930031ad267bb3142ba4 +grpc_1_36_0_githash=736e3758351ced3cd842bad3ba4e2540f01bbc48 function build_grpc () { git clone https://github.com/grpc/grpc.git google/grpc cd google/grpc - git checkout ${grpc_1_15_1_githash} + git checkout ${grpc_1_36_0_githash} git submodule update --init - make - make install prefix=`pwd`/install + # Apply boringssl build patch + cd third_party/boringssl-with-bazel + git apply ../../../../grpc/boringssl.patch + cd ../.. + mkdir ../grpc_build + cd ../grpc_build + cmake ../grpc -DgRPC_INSTALL=ON -DgRPC_BUILD_TESTS=OFF -DBUILD_SHARED_LIBS=ON -DCMAKE_INSTALL_PREFIX=`pwd`/../grpc/install + cmake --build . --target install ${JOBS:+-j$JOBS} if [ ! -f ${GRPC_INSTALL_PATH}/lib/libgrpc++_unsecure.so.1 ]; then ln -s ${GRPC_INSTALL_PATH}/lib/libgrpc++_unsecure.so.6 ${GRPC_INSTALL_PATH}/lib/libgrpc++_unsecure.so.1 fi diff --git a/grpc/samples/greeter/client.cpp b/grpc/samples/greeter/client.cpp index 2e42f8fda..0e4f34814 100644 --- a/grpc/samples/greeter/client.cpp +++ b/grpc/samples/greeter/client.cpp @@ -1,7 +1,7 @@ #include "greeter.grpc.fb.h" #include "greeter_generated.h" -#include +#include #include #include diff --git a/grpc/samples/greeter/server.cpp b/grpc/samples/greeter/server.cpp index db442593d..ac38fa969 100644 --- a/grpc/samples/greeter/server.cpp +++ b/grpc/samples/greeter/server.cpp @@ -1,7 +1,7 @@ #include "greeter.grpc.fb.h" #include "greeter_generated.h" -#include +#include #include #include diff --git a/grpc/src/compiler/cpp_generator.cc b/grpc/src/compiler/cpp_generator.cc index 6cfd22e39..8dd408830 100644 --- a/grpc/src/compiler/cpp_generator.cc +++ b/grpc/src/compiler/cpp_generator.cc @@ -144,15 +144,15 @@ grpc::string GetHeaderIncludes(grpc_generator::File *file, std::map vars; static const char *headers_strs[] = { - "grpc++/impl/codegen/async_stream.h", - "grpc++/impl/codegen/async_unary_call.h", - "grpc++/impl/codegen/method_handler_impl.h", - "grpc++/impl/codegen/proto_utils.h", - "grpc++/impl/codegen/rpc_method.h", - "grpc++/impl/codegen/service_type.h", - "grpc++/impl/codegen/status.h", - "grpc++/impl/codegen/stub_options.h", - "grpc++/impl/codegen/sync_stream.h"}; + "grpcpp/impl/codegen/async_stream.h", + "grpcpp/impl/codegen/async_unary_call.h", + "grpcpp/impl/codegen/method_handler.h", + "grpcpp/impl/codegen/proto_utils.h", + "grpcpp/impl/codegen/rpc_method.h", + "grpcpp/impl/codegen/service_type.h", + "grpcpp/impl/codegen/status.h", + "grpcpp/impl/codegen/stub_options.h", + "grpcpp/impl/codegen/sync_stream.h"}; std::vector headers(headers_strs, array_end(headers_strs)); PrintIncludes(printer.get(), headers, params); printer->Print(vars, "\n"); @@ -1179,14 +1179,14 @@ grpc::string GetSourceIncludes(grpc_generator::File *file, std::map vars; static const char *headers_strs[] = { - "grpc++/impl/codegen/async_stream.h", - "grpc++/impl/codegen/async_unary_call.h", - "grpc++/impl/codegen/channel_interface.h", - "grpc++/impl/codegen/client_unary_call.h", - "grpc++/impl/codegen/method_handler_impl.h", - "grpc++/impl/codegen/rpc_service_method.h", - "grpc++/impl/codegen/service_type.h", - "grpc++/impl/codegen/sync_stream.h"}; + "grpcpp/impl/codegen/async_stream.h", + "grpcpp/impl/codegen/async_unary_call.h", + "grpcpp/impl/codegen/channel_interface.h", + "grpcpp/impl/codegen/client_unary_call.h", + "grpcpp/impl/codegen/method_handler.h", + "grpcpp/impl/codegen/rpc_service_method.h", + "grpcpp/impl/codegen/service_type.h", + "grpcpp/impl/codegen/sync_stream.h"}; std::vector headers(headers_strs, array_end(headers_strs)); PrintIncludes(printer.get(), headers, params); @@ -1604,8 +1604,8 @@ grpc::string GetMockIncludes(grpc_generator::File *file, std::map vars; static const char *headers_strs[] = { - "grpc++/impl/codegen/async_stream.h", - "grpc++/impl/codegen/sync_stream.h", + "grpcpp/impl/codegen/async_stream.h", + "grpcpp/impl/codegen/sync_stream.h", "gmock/gmock.h", }; std::vector headers(headers_strs, array_end(headers_strs)); diff --git a/grpc/tests/grpctest.cpp b/grpc/tests/grpctest.cpp index decf5e513..6991b7ebb 100644 --- a/grpc/tests/grpctest.cpp +++ b/grpc/tests/grpctest.cpp @@ -14,8 +14,9 @@ * limitations under the License. */ -#include +#include +#include #include #include "monster_test.grpc.fb.h" diff --git a/include/flatbuffers/grpc.h b/include/flatbuffers/grpc.h index bd24c501b..b7935551c 100644 --- a/include/flatbuffers/grpc.h +++ b/include/flatbuffers/grpc.h @@ -20,8 +20,8 @@ // Helper functionality to glue FlatBuffers and GRPC. #include "flatbuffers/flatbuffers.h" -#include "grpc++/support/byte_buffer.h" #include "grpc/byte_buffer_reader.h" +#include "grpcpp/support/byte_buffer.h" namespace flatbuffers { namespace grpc { @@ -287,8 +287,9 @@ template class SerializationTraits> { } // Deserialize by pulling the - static grpc::Status Deserialize(grpc_byte_buffer *buffer, + static grpc::Status Deserialize(ByteBuffer *buf, flatbuffers::grpc::Message *msg) { + grpc_byte_buffer *buffer = *reinterpret_cast(buf); if (!buffer) { return ::grpc::Status(::grpc::StatusCode::INTERNAL, "No payload"); } diff --git a/tests/monster_test.grpc.fb.cc b/tests/monster_test.grpc.fb.cc index f83e60481..0b07cc782 100644 --- a/tests/monster_test.grpc.fb.cc +++ b/tests/monster_test.grpc.fb.cc @@ -5,14 +5,14 @@ #include "monster_test_generated.h" #include "monster_test.grpc.fb.h" -#include -#include -#include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include +#include +#include +#include namespace MyGame { namespace Example { diff --git a/tests/monster_test.grpc.fb.h b/tests/monster_test.grpc.fb.h index 72402ecc3..1b3384722 100644 --- a/tests/monster_test.grpc.fb.h +++ b/tests/monster_test.grpc.fb.h @@ -7,15 +7,15 @@ #include "monster_test_generated.h" #include "flatbuffers/grpc.h" -#include -#include -#include -#include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include +#include +#include +#include +#include namespace grpc { class CompletionQueue;