Add support for brotli compression (#406)

This commit is contained in:
An Tao 2020-04-04 18:15:15 +08:00 committed by GitHub
parent 2fb0f845f6
commit c43ba9e514
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
20 changed files with 460 additions and 15 deletions

View File

@ -28,6 +28,7 @@ addons:
- build-essential
- cmake
- boost1.67
- libbrotli-dev
homebrew:
packages:
- jsoncpp

View File

@ -109,6 +109,13 @@ if(NOT WIN32)
endif()
endif(NOT WIN32)
find_package(Brotli)
if(Brotli_FOUND)
message(STATUS "Brotli found")
add_definitions(-DUSE_BROTLI)
target_link_libraries(${PROJECT_NAME} PRIVATE Brotli_lib)
endif(Brotli_FOUND)
set(DROGON_SOURCES
lib/src/AOPAdvice.cc
lib/src/CacheFile.cc
@ -430,6 +437,7 @@ install(
"${CMAKE_CURRENT_SOURCE_DIR}/cmake_modules/FindSQLite3.cmake"
"${CMAKE_CURRENT_SOURCE_DIR}/cmake_modules/FindMySQL.cmake"
"${CMAKE_CURRENT_SOURCE_DIR}/cmake_modules/Findpg.cmake"
"${CMAKE_CURRENT_SOURCE_DIR}/cmake_modules/FindBrotli.cmake"
DESTINATION "${INSTALL_DROGON_CMAKE_DIR}"
COMPONENT dev)

View File

@ -25,7 +25,7 @@ Drogon is a cross-platform framework, It supports Linux, Mac OS, FreeBSD and Win
* Support WebSocket (server side and client side);
* Support JSON format request and response, very friendly to the Restful API application development;
* Support file download and upload;
* Support gzip compression transmission;
* Support gzip, brotli compression transmission;
* Support pipelining;
* Provide a lightweight command line tool, drogon_ctl, to simplify the creation of various classes in Drogon and the generation of view code;
* Support non-blocking I/O based asynchronously reading and writing database (PostgreSQL and MySQL(MariaDB) database);

View File

@ -24,7 +24,7 @@ Drogon是一个跨平台框架它支持Linux也支持Mac OS、FreeBSD
* 支持websocket(server端和client端);
* 支持Json格式请求和应答, 对Restful API应用开发非常友好;
* 支持文件下载和上传,支持sendfile系统调用
* 支持gzip压缩传输
* 支持gzip/brotli压缩传输;
* 支持pipelining
* 提供一个轻量的命令行工具drogon_ctl帮助简化各种类的创建和视图代码的生成过程
* 基于非阻塞IO实现的异步数据库读写目前支持PostgreSQL和MySQL(MariaDB)数据库;
@ -34,4 +34,154 @@ Drogon是一个跨平台框架它支持Linux也支持Mac OS、FreeBSD
* 支持插件,可通过配置文件在加载期动态拆装;
* 支持内建插入点的AOP
### 更多详情请浏览 [wiki](https://github.com/an-tao/drogon/wiki/01-概述)
## 一个非常简单的例子
不像大多数C++框架那样drogon的主程序可以保持非常简单。 Drogon使用了一些小技巧是主程序和控制器解耦合. 控制器的路径路由设置可以在控制器类定义中或者配置文件中完成.
下面是一个典型的主程序的样子:
```c++
#include <drogon/drogon.h>
using namespace drogon;
int main()
{
app().setLogPath("./")
.setLogLevel(trantor::Logger::kWarn)
.addListener("0.0.0.0", 80)
.setThreadNum(16)
.enableRunAsDaemon()
.run();
}
```
如果使用配置文件,可以进一步简化成如下的样子:
```c++
#include <drogon/drogon.h>
using namespace drogon;
int main()
{
app().loadConfigFile("./config.json").run();
}
```
当然Drogon也提供了一些接口使用户可以在main()函数中直接添加控制器逻辑比如用户可以注册一个lambda处理器到drogon框架中如下所示
```c++
app.registerHandler("/test?username={name}",
[](const HttpRequestPtr& req,
std::function<void (const HttpResponsePtr &)> &&callback,
const std::string &name)
{
Json::Value json;
json["result"]="ok";
json["message"]=std::string("hello,")+name;
auto resp=HttpResponse::newHttpJsonResponse(json);
callback(resp);
},
{Get,"LoginFilter"});
```
这看起来是很方便但是这并不适用于复杂的应用试想假如有数十个或者数百个处理函数要注册进框架main()函数将膨胀到不可读的程度。显然让每个包含处理函数的类在自己的定义中完成注册是更好的选择。所以除非你的应用逻辑非常简单我们不推荐使用上述接口更好的实践是我们可以创建一个HttpSimpleController对象如下
```c++
/// The TestCtrl.h file
#pragma once
#include <drogon/HttpSimpleController.h>
using namespace drogon;
class TestCtrl:public drogon::HttpSimpleController<TestCtrl>
{
public:
virtual void asyncHandleHttpRequest(const HttpRequestPtr& req, std::function<void (const HttpResponsePtr &)> &&callback) override;
PATH_LIST_BEGIN
PATH_ADD("/test",Get);
PATH_LIST_END
};
/// The TestCtrl.cc file
#include "TestCtrl.h"
void TestCtrl::asyncHandleHttpRequest(const HttpRequestPtr& req,
std::function<void (const HttpResponsePtr &)> &&callback)
{
//write your application logic here
auto resp = HttpResponse::newHttpResponse();
resp->setBody("<p>Hello, world!</p>");
resp->setExpiredTime(0);
callback(resp);
}
```
**上面程序的大部分代码都可以由`drogon_ctl`命令创建**(这个命令是`drogon_ctl create controller TestCtr`。用户所需做的就是添加自己的业务逻辑。在这个例子中当客户端访问URL`http://ip/test`时,控制器简单的返回了一个`Hello, world!`页面。
对于JSON格式的响应我们可以像下面这样创建控制器
```c++
/// The header file
#pragma once
#include <drogon/HttpSimpleController.h>
using namespace drogon;
class JsonCtrl : public drogon::HttpSimpleController<JsonCtrl>
{
public:
virtual void asyncHandleHttpRequest(const HttpRequestPtr &req, std::function<void(const HttpResponsePtr &)> &&callback) override;
PATH_LIST_BEGIN
//list path definitions here;
PATH_ADD("/json", Get);
PATH_LIST_END
};
/// The source file
#include "JsonCtrl.h"
void JsonCtrl::asyncHandleHttpRequest(const HttpRequestPtr &req,
std::function<void(const HttpResponsePtr &)> &&callback)
{
Json::Value ret;
ret["message"] = "Hello, World!";
auto resp = HttpResponse::newHttpJsonResponse(ret);
callback(resp);
}
```
让我们更进一步通过HttpController类创建一个RESTful API的例子如下所示忽略了实现文件
```c++
/// The header file
#pragma once
#include <drogon/HttpController.h>
using namespace drogon;
namespace api
{
namespace v1
{
class User : public drogon::HttpController<User>
{
public:
METHOD_LIST_BEGIN
//use METHOD_ADD to add your custom processing function here;
METHOD_ADD(User::getInfo, "/{id}", Get); //path is /api/v1/User/{arg1}
METHOD_ADD(User::getDetailInfo, "/{id}/detailinfo", Get); //path is /api/v1/User/{arg1}/detailinfo
METHOD_ADD(User::newUser, "/{name}", Post); //path is /api/v1/User/{arg1}
METHOD_LIST_END
//your declaration of processing function maybe like this:
void getInfo(const HttpRequestPtr &req, std::function<void(const HttpResponsePtr &)> &&callback, int userId) const;
void getDetailInfo(const HttpRequestPtr &req, std::function<void(const HttpResponsePtr &)> &&callback, int userId) const;
void newUser(const HttpRequestPtr &req, std::function<void(const HttpResponsePtr &)> &&callback, std::string &&userName);
public:
User()
{
LOG_DEBUG << "User constructor!";
}
};
} // namespace v1
} // namespace api
```
如你所见,通过`HttpController`类用户可以同时映射路径和路径参数这对RESTful API应用来说非常方便。
另外,你可以发现前面所有的处理函数接口都是异步的,处理器的响应是通过回调对象返回的。这种设计是出于对高性能的考虑,因为在异步模式下,可以使用少量的线程(比如和处理器核心数相等的线程)处理大量的并发请求。
After compiling all of the above source files, we get a very simple web application. This is a good start. **for more information, please visit the [wiki](https://github.com/an-tao/drogon/wiki/01-Overview) or the [doxiz](https://doxiz.com/drogon/master/overview/)**
编译上述的所有源文件后我们得到了一个非常简单的web应用程序这是一个不错的开始。**请访问[wiki](https://github.com/an-tao/drogon/wiki/01-Overview)或者[doxiz](https://doxiz.com/drogon/master/overview/)以获取更多的信息**

View File

@ -30,6 +30,9 @@ endif()
if(@Boost_FOUND@)
find_dependency(Boost)
endif()
if(@Brotli_FOUND@)
find_dependency(Brotli)
endif()
# Our library dependencies (contains definitions for IMPORTED targets)

View File

@ -0,0 +1,50 @@
# ***************************************************************************
# _ _ ____ _
# Project ___| | | | _ \| |
# / __| | | | |_) | |
# | (__| |_| | _ <| |___
# \___|\___/|_| \_\_____|
#
# Copyright (C) 1998 - 2020, Daniel Stenberg, <daniel@haxx.se>, et al.
#
# This software is licensed as described in the file COPYING, which you should
# have received as part of this distribution. The terms are also available at
# https://curl.haxx.se/docs/copyright.html.
#
# You may opt to use, copy, modify, merge, publish, distribute and/or sell
# copies of the Software, and permit persons to whom the Software is furnished
# to do so, under the terms of the COPYING file.
#
# This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
# KIND, either express or implied.
#
# ##############################################################################
include(FindPackageHandleStandardArgs)
find_path(BROTLI_INCLUDE_DIR "brotli/decode.h")
find_library(BROTLICOMMON_LIBRARY NAMES brotlicommon)
find_library(BROTLIDEC_LIBRARY NAMES brotlidec)
find_library(BROTLIENC_LIBRARY NAMES brotlienc)
find_package_handle_standard_args(Brotli
REQUIRED_VARS
BROTLIDEC_LIBRARY
BROTLIENC_LIBRARY
BROTLICOMMON_LIBRARY
BROTLI_INCLUDE_DIR
FAIL_MESSAGE
"Could NOT find BROTLI")
set(BROTLI_INCLUDE_DIRS ${BROTLI_INCLUDE_DIR})
set(BROTLI_LIBRARIES ${BROTLICOMMON_LIBRARY} ${BROTLIDEC_LIBRARY}
${BROTLIENC_LIBRARY})
if(Brotli_FOUND)
add_library(Brotli_lib INTERFACE IMPORTED)
set_target_properties(Brotli_lib
PROPERTIES INTERFACE_INCLUDE_DIRECTORIES
"${BROTLI_INCLUDE_DIRS}"
INTERFACE_LINK_LIBRARIES
"${BROTLI_LIBRARIES}")
endif(Brotli_FOUND)

View File

@ -154,6 +154,8 @@
"use_sendfile": true,
//use_gzip: True by default, use gzip to compress the response body's content;
"use_gzip": true,
//use_brotli: False by default, use brotli to compress the response body's content;
"use_brotli": false,
//static_files_cache_time: 5 (seconds) by default, the time in which the static file response is cached,
//0 means cache forever, the negative value means no cache
"static_files_cache_time": 5,

View File

@ -154,6 +154,8 @@
"use_sendfile": true,
//use_gzip: True by default, use gzip to compress the response body's content;
"use_gzip": true,
//use_brotli: False by default, use brotli to compress the response body's content;
"use_brotli": false,
//static_files_cache_time: 5 (seconds) by default, the time in which the static file response is cached,
//0 means cache forever, the negative value means no cache
"static_files_cache_time": 5,

View File

@ -163,6 +163,34 @@ void doTest(const HttpClientPtr &client,
exit(1);
}
});
/// Test brotli
#ifdef USE_BROTLI
req = HttpRequest::newHttpRequest();
req->setMethod(drogon::Get);
req->addHeader("accept-encoding", "br");
req->setPath("/api/v1/apitest/get/111");
client->sendRequest(req,
[=](ReqResult result, const HttpResponsePtr &resp) {
if (result == ReqResult::Ok)
{
if (resp->getBody().length() == 4994)
{
outputGood(req, isHttps);
}
else
{
LOG_DEBUG << resp->getBody().length();
LOG_ERROR << "Error!";
exit(1);
}
}
else
{
LOG_ERROR << "Error!";
exit(1);
}
});
#endif
/// Post json
Json::Value json;
json["request"] = "json";

View File

@ -822,6 +822,23 @@ class HttpAppFramework : public trantor::NonCopyable
/// Return true if gzip is enabled.
virtual bool isGzipEnabled() const = 0;
/// Enable brotli compression.
/**
* @param useBrotli if the parameter is true, use brotli to compress the
* response body's content;
* The default value is true.
*
* @note
* This operation can be performed by an option in the configuration file.
* After brotli is enabled, brotli is used under the following conditions:
* 1. The content type of response is not a binary type.
* 2. The content length is bigger than 1024 bytes.
*/
virtual HttpAppFramework &enableBrotli(bool useBrotli) = 0;
/// Return true if brotli is enabled.
virtual bool isBrotliEnabled() const = 0;
/// Set the time in which the static file response is cached in memory.
/**
* @param cacheTime in seconds. 0 means always cached, negative means no

View File

@ -82,14 +82,20 @@ std::string urlEncodeComponent(const std::string &);
/// Commpress or decompress data using gzip lib.
/**
* @param data Data before compressing or after decompressing
* @param ndata Data length before compressing or after decompressing
* @param zdata Data after compressing or before decompressing
* @param nzdata Data length after compressing or before decompressing
* @param data the input data
* @param ndata the input data length
*/
std::string gzipCompress(const char *data, const size_t ndata);
std::string gzipDecompress(const char *data, const size_t ndata);
/// Commpress or decompress data using brotli lib.
/**
* @param data the input data
* @param ndata the input data length
*/
std::string brotliCompress(const char *data, const size_t ndata);
std::string brotliDecompress(const char *data, const size_t ndata);
/// Get the http full date string
/**
* rfc2616-3.3.1

View File

@ -354,6 +354,8 @@ static void loadApp(const Json::Value &app)
drogon::app().enableSendfile(useSendfile);
auto useGzip = app.get("use_gzip", true).asBool();
drogon::app().enableGzip(useGzip);
auto useBr = app.get("use_brotli", false).asBool();
drogon::app().enableBrotli(useBr);
auto staticFilesCacheTime = app.get("static_files_cache_time", 5).asInt();
drogon::app().setStaticFilesCacheTime(staticFilesCacheTime);
loadControllers(app["simple_controllers_map"]);

View File

@ -254,6 +254,15 @@ class HttpAppFrameworkImpl : public HttpAppFramework
{
return useGzip_;
}
virtual HttpAppFramework &enableBrotli(bool useBrotli) override
{
useBrotli_ = useBrotli;
return *this;
}
virtual bool isBrotliEnabled() const override
{
return useBrotli_;
}
virtual HttpAppFramework &setStaticFilesCacheTime(int cacheTime) override;
virtual int staticFilesCacheTime() const override;
virtual HttpAppFramework &setIdleConnectionTimeout(size_t timeout) override
@ -496,6 +505,7 @@ class HttpAppFrameworkImpl : public HttpAppFramework
size_t pipeliningRequestsNumber_{0};
bool useSendfile_{true};
bool useGzip_{true};
bool useBrotli_{false};
size_t clientMaxBodySize_{1024 * 1024};
size_t clientMaxMemoryBodySize_{64 * 1024};
size_t clientMaxWebSocketMessageSize_{128 * 1024};

View File

@ -400,10 +400,17 @@ void HttpClientImpl::onRecvMessage(const trantor::TcpConnectionPtr &connPtr,
responseParser->reset();
assert(!pipeliningCallbacks_.empty());
auto &type = resp->getHeaderBy("content-type");
if (resp->getHeaderBy("content-encoding") == "gzip")
auto &coding = resp->getHeaderBy("content-encoding");
if (coding == "gzip")
{
resp->gunzip();
}
#ifdef USE_BROTLI
else if (coding == "br")
{
resp->brDecompress();
}
#endif
if (type.find("application/json") != std::string::npos)
{
resp->parseJson();

View File

@ -310,6 +310,19 @@ class HttpResponseImpl : public HttpResponse
std::make_shared<HttpMessageStringBody>(move(gunzipBody));
}
}
#ifdef USE_BROTLI
void brDecompress()
{
if (bodyPtr_)
{
auto gunzipBody =
utils::brotliDecompress(bodyPtr_->data(), bodyPtr_->length());
removeHeader("content-encoding");
bodyPtr_ =
std::make_shared<HttpMessageStringBody>(move(gunzipBody));
}
}
#endif
~HttpResponseImpl();
protected:

View File

@ -36,13 +36,43 @@ static HttpResponsePtr getCompressedResponse(const HttpRequestImplPtr &req,
LOG_TRACE << "Use gzip to compress the body";
auto &sendfileName =
static_cast<HttpResponseImpl *>(response.get())->sendfileName();
if (app().isGzipEnabled() && sendfileName.empty() &&
req->getHeaderBy("accept-encoding").find("gzip") != std::string::npos &&
static_cast<HttpResponseImpl *>(response.get())
->getHeaderBy("content-encoding")
.empty() &&
response->getContentType() < CT_APPLICATION_OCTET_STREAM &&
response->getBody().length() > 1024 && !isHeadMethod)
if (!sendfileName.empty() ||
response->getContentType() >= CT_APPLICATION_OCTET_STREAM ||
response->getBody().length() < 1024 || isHeadMethod ||
!(static_cast<HttpResponseImpl *>(response.get())
->getHeaderBy("content-encoding")
.empty()))
{
return response;
}
#ifdef USE_BROTLI
if (app().isBrotliEnabled() &&
req->getHeaderBy("accept-encoding").find("br") != std::string::npos)
{
auto newResp = response;
auto strCompress = utils::brotliCompress(response->getBody().data(),
response->getBody().length());
if (!strCompress.empty())
{
if (response->expiredTime() >= 0)
{
// cached response,we need to make a clone
newResp = std::make_shared<HttpResponseImpl>(
*static_cast<HttpResponseImpl *>(response.get()));
newResp->setExpiredTime(-1);
}
newResp->setBody(std::move(strCompress));
newResp->addHeader("Content-Encoding", "br");
}
else
{
LOG_ERROR << "brotli got 0 length result";
}
return newResp;
}
#endif
if (app().isGzipEnabled() &&
req->getHeaderBy("accept-encoding").find("gzip") != std::string::npos)
{
auto newResp = response;
auto strCompress = utils::gzipCompress(response->getBody().data(),

View File

@ -14,6 +14,10 @@
#include <drogon/utils/Utilities.h>
#include <trantor/utils/Logger.h>
#ifdef USE_BROTLI
#include <brotli/decode.h>
#include <brotli/encode.h>
#endif
#ifdef _WIN32
#include <Rpc.h>
#include <direct.h>
@ -998,6 +1002,79 @@ int createPath(const std::string &path)
}
return 0;
}
#ifdef USE_BROTLI
std::string brotliCompress(const char *data, const size_t ndata)
{
std::string ret;
if (ndata == 0)
return ret;
ret.resize(BrotliEncoderMaxCompressedSize(ndata));
size_t encodedSize{ret.size()};
auto r = BrotliEncoderCompress(5,
BROTLI_DEFAULT_WINDOW,
BROTLI_DEFAULT_MODE,
ndata,
(const uint8_t *)(data),
&encodedSize,
(uint8_t *)(ret.data()));
if (r == BROTLI_FALSE)
ret.resize(0);
else
ret.resize(encodedSize);
return ret;
}
std::string brotliDecompress(const char *data, const size_t ndata)
{
if (ndata == 0)
return std::string(data, ndata);
size_t availableIn = ndata;
auto nextIn = (const uint8_t *)(data);
auto decompressed = std::string(availableIn * 3, 0);
size_t availableOut = decompressed.size();
auto nextOut = (uint8_t *)(decompressed.data());
size_t totalOut{0};
bool done = false;
auto s = BrotliDecoderCreateInstance(nullptr, nullptr, nullptr);
while (!done)
{
auto result = BrotliDecoderDecompressStream(
s, &availableIn, &nextIn, &availableOut, &nextOut, &totalOut);
if (result == BROTLI_DECODER_RESULT_SUCCESS)
{
decompressed.resize(totalOut);
done = true;
}
else if (result == BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT)
{
assert(totalOut == decompressed.size());
decompressed.resize(totalOut * 2);
nextOut = (uint8_t *)(decompressed.data() + totalOut);
availableOut = totalOut;
}
else
{
decompressed.resize(0);
done = true;
}
}
BrotliDecoderDestroyInstance(s);
return decompressed;
}
#else
std::string brotliCompress(const char *data, const size_t ndata)
{
LOG_ERROR << "If you do not have the brotli package installed, you cannot "
"use brotliCompress()";
abort();
}
std::string brotliDecompress(const char *data, const size_t ndata)
{
LOG_ERROR << "If you do not have the brotli package installed, you cannot "
"use brotliDecompress()";
abort();
}
#endif
} // namespace utils
} // namespace drogon

View File

@ -33,6 +33,7 @@ fi
sed -i -e "s/\"run_as_daemon.*$/\"run_as_daemon\": true\,/" config.example.json
sed -i -e "s/\"relaunch_on_error.*$/\"relaunch_on_error\": true\,/" config.example.json
sed -i -e "s/\"threads_num.*$/\"threads_num\": 0\,/" config.example.json
sed -i -e "s/\"use_brotli.*$/\"use_brotli\": true\,/" config.example.json
if [ ! -f "webapp" ]; then
echo "Build failed"

View File

@ -0,0 +1,32 @@
#include <drogon/utils/Utilities.h>
#include <gtest/gtest.h>
#include <string>
#include <iostream>
using namespace drogon::utils;
TEST(BrotliTest, shortText)
{
std::string source{"123中文顶替要枯械"};
auto compressed = brotliCompress(source.data(), source.length());
auto decompressed =
brotliDecompress(compressed.data(), compressed.length());
EXPECT_EQ(source, decompressed);
}
TEST(BrotliTest, longText)
{
std::string source;
for (size_t i = 0; i < 100000; i++)
{
source.append(std::to_string(i));
}
auto compressed = brotliCompress(source.data(), source.length());
auto decompressed =
brotliDecompress(compressed.data(), compressed.length());
EXPECT_EQ(source, decompressed);
}
int main(int argc, char **argv)
{
testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}

View File

@ -4,6 +4,9 @@ add_executable(gzip_unittest GzipUnittest.cpp)
add_executable(md5_unittest MD5Unittest.cpp ../lib/src/ssl_funcs/Md5.cc)
add_executable(sha1_unittest SHA1Unittest.cpp ../lib/src/ssl_funcs/Sha1.cc)
add_executable(ostringstream_unittest OStringStreamUnitttest.cpp)
if(Brotli_FOUND)
add_executable(brotli_unittest BrotliUnittest.cpp)
endif()
set(UNITTEST_TARGETS
drogon_msgbuffer_unittest
@ -12,6 +15,9 @@ set(UNITTEST_TARGETS
md5_unittest
sha1_unittest
ostringstream_unittest)
if(Brotli_FOUND)
set(UNITTEST_TARGETS ${UNITTEST_TARGETS} brotli_unittest)
endif()
set_property(TARGET ${UNITTEST_TARGETS}
PROPERTY CXX_STANDARD ${DROGON_CXX_STANDARD})