Rooba
/
drogon
mirror of https://github.com/drogonframework/drogon.git
1
1
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #3 from an-tao/cache_static_file 

Cache static files for several seconds to promote the performance
This commit is contained in:
an-tao 2018-10-15 16:21:20 +08:00 committed by GitHub
parent 30ab9e09b1 5362de9a58
commit 3bca1cdb0b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 265 additions and 136 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
config.example.json
View File

@ -86,6 +86,9 @@
//use sendfile() system-call to send static file to client;
"use_sendfile": true,
//use_gzip:true by default,use gzip to compress the response body's content;
"use_gzip": true
"use_gzip": true,
//static_files_cache_time:5 by default,the time in which static file response is cached,
//0 means cache forever,the negative value means no cache
"static_files_cache_time":5
}
}

23
drogon_ctl/CMakeLists.txt
View File

@ -1,5 +1,22 @@
link_libraries(drogon)
link_libraries(drogon trantor uuid pthread jsoncpp dl z)
if(OpenSSL_FOUND)
link_libraries(ssl crypto)
endif()
AUX_SOURCE_DIRECTORY(. SRC_DIR)
add_executable(drogon_ctl ${SRC_DIR})
add_dependencies(drogon_ctl trantor makeVersion)
add_executable(_drogon_ctl ${SRC_DIR})
FILE(GLOB SCP_LIST ${CMAKE_CURRENT_SOURCE_DIR}/templates/*.csp)
foreach(cspFile ${SCP_LIST})
message(STATUS "cspFile:" ${cspFile})
EXEC_PROGRAM(basename ARGS "-s .csp ${cspFile}" OUTPUT_VARIABLE classname)
message(STATUS "view classname:" ${classname})
add_custom_command(OUTPUT ${classname}.h ${classname}.cc
COMMAND _drogon_ctl
ARGS create view ${cspFile}
DEPENDS ${cspFile}
VERBATIM )
set(TEMPL_SRC ${TEMPL_SRC} ${classname}.cc)
endforeach()
add_executable(drogon_ctl ${SRC_DIR} ${TEMPL_SRC})
add_dependencies(drogon_ctl trantor makeVersion _drogon_ctl)
install(TARGETS drogon_ctl DESTINATION bin)

78
drogon_ctl/create_project.cc
View File

@ -1,4 +1,5 @@
#include "create_project.h"
#include <drogon/HttpResponse.h>
#include <iostream>
#include <sys/stat.h>
#include <sys/types.h>
@ -201,81 +202,8 @@ static void newJsonFindFile(std::ofstream &jsonFile)
static void newConfigFile(std::ofstream &configFile)
{
configFile << "/* This is a JSON format configuration file\n"
"*/\n"
"{\n"
" //ssl:the global ssl files setting\n"
" /*\"ssl\": {\n"
" \"cert\": \"../../trantor/trantor/tests/server.pem\",\n"
" \"key\": \"../../trantor/trantor/tests/server.pem\"\n"
" },\n"
" \"listeners\": [\n"
" {\n"
" //address:ip address,0.0.0.0 by default\n"
" \"address\": \"0.0.0.0\",\n"
" //port:port number\n"
" \"port\": 80,\n"
" //https:if use https for security,false by default\n"
" \"https\": false\n"
" },\n"
" {\n"
" \"address\": \"0.0.0.0\",\n"
" \"port\": 443,\n"
" \"https\": true,\n"
" //cert,key:cert file path and key file path,empty by default,\n"
" //if empty,use global setting\n"
" \"cert\": \"\",\n"
" \"key\": \"\"\n"
" }\n"
" ],*/\n"
" \"app\": {\n"
" //threads_num:num of threads,1 by default\n"
" \"threads_num\": 16,\n"
" //enable_session:false by default\n"
" \"enable_session\": false,\n"
" \"session_timeout\": 0,\n"
" //document_root:Root path of HTTP document,defaut path is ./\n"
" \"document_root\": \"./\",\n"
" /* file_types:\n"
" * HTTP download file types,The file types supported by drogon\n"
" * by default are \"html\", \"js\", \"css\", \"xml\", \"xsl\", \"txt\", \"svg\",\n"
" * \"ttf\", \"otf\", \"woff2\", \"woff\" , \"eot\", \"png\", \"jpg\", \"jpeg\",\n"
" * \"gif\", \"bmp\", \"ico\", \"icns\", etc. */\n"
" \"file_types\": [\"gif\",\"png\",\"jpg\",\"js\",\"css\",\"html\",\"ico\",\"swf\",\"xap\",\"apk\",\"cur\",\"xml\"],\n"
" //max_connections:max connections number,100000 by default\n"
" \"max_connections\": 100000,\n"
" //Load_dynamic_views: false by default, when set to true, drogon will\n"
" //compile and load dynamically \"CSP View Files\" in directories defined\n"
" //by \"dynamic_views_path\"\n"
" \"load_dynamic_views\": true,\n"
" //dynamic_views_path: if the path isn't prefixed with / or ./,\n"
" //it will be relative path of document_root path\n"
" \"dynamic_views_path\": [\"./views\"],\n"
" //log:set log output,drogon output logs to stdout by default\n"
" \"log\": {\n"
" //log_path:log file path,empty by default,in which case,log will output to the stdout\n"
" \"log_path\": \"./\",\n"
" //logfile_base_name:log file base name,empty by default which means 'drogon' will name logfile as\n"
" //drogon.log ...\n"
" \"logfile_base_name\": \"\",\n"
" //log_size_limit:100000000 bytes by default,\n"
" //When the log file size reaches \"log_size_limit\", the log file is switched.\n"
" \"log_size_limit\": 100000000,\n"
" //log_level:\"DEBUG\" by default,options:\"TRACE\",\"DEBUG\",\"INFO\",\"WARN\"\n"
" //The TRACE level is only valid when built in DEBUG mode.\n"
" \"log_level\": \"DEBUG\"\n"
" },\n"
" //run_as_daemon:false by default\n"
" \"run_as_daemon\": false,\n"
" //relaunch_on_error:false by default,if true,the program will be restart by parent after exit;\n"
" \"relaunch_on_error\": false,\n"
" //use_sendfile:true by default,if ture,the program will\n"
" //use sendfile() system-call to send static file to client;\n"
" \"use_sendfile\": true,\n"
" //use_gzip:true by default,use gzip to compress the response body's content;\n"
" \"use_gzip\": true\n"
" }\n"
"}\n";
auto resp=HttpResponse::newHttpViewResponse("config",drogon::HttpViewData());
configFile << resp->getBody();
}
void create_project::createProject(const std::string &projectName)
{

94
drogon_ctl/templates/config.csp Normal file
View File

@ -0,0 +1,94 @@
/* This is a JSON format configuration file
*/
{
//ssl:the global ssl files setting
/*
"ssl": {
"cert": "../../trantor/trantor/tests/server.pem",
"key": "../../trantor/trantor/tests/server.pem"
},
"listeners": [
{
//address:ip address,0.0.0.0 by default
"address": "0.0.0.0",
//port:port number
"port": 80,
//https:if use https for security,false by default
"https": false
},
{
"address": "0.0.0.0",
"port": 443,
"https": true,
//cert,key:cert file path and key file path,empty by default,
//if empty,use global setting
"cert": "",
"key": ""
}
],*/
"app": {
//threads_num:num of threads,1 by default
"threads_num": 16,
//enable_session:false by default
"enable_session": false,
"session_timeout": 0,
//document_root:Root path of HTTP document,defaut path is ./
"document_root": "./",
/* file_types:
* HTTP download file types,The file types supported by drogon
* by default are "html", "js", "css", "xml", "xsl", "txt", "svg",
* "ttf", "otf", "woff2", "woff" , "eot", "png", "jpg", "jpeg",
* "gif", "bmp", "ico", "icns", etc. */
"file_types": [
"gif",
"png",
"jpg",
"js",
"css",
"html",
"ico",
"swf",
"xap",
"apk",
"cur",
"xml"
],
//max_connections:max connections number,100000 by default
"max_connections": 100000,
//max_connections_per_ip:max connections number per clinet,0 by default which means no limit
"max_connections_per_ip": 0,
//Load_dynamic_views: false by default, when set to true, drogon will
//compile and load dynamically "CSP View Files" in directories defined
//by "dynamic_views_path"
//"load_dynamic_views":true,
//dynamic_views_path: if the path isn't prefixed with / or ./,
//it will be relative path of document_root path
//"dynamic_views_path":["./views"],
//log:set log output,drogon output logs to stdout by default
"log": {
//log_path:log file path,empty by default,in which case,log will output to the stdout
"log_path": "./",
//logfile_base_name:log file base name,empty by default which means drogon will name logfile as
//drogon.log ...
"logfile_base_name": "",
//log_size_limit:100000000 bytes by default,
//When the log file size reaches "log_size_limit", the log file will be switched.
"log_size_limit": 100000000,
//log_level:"DEBUG" by default,options:"TRACE","DEBUG","INFO","WARN"
//The TRACE level is only valid when built in DEBUG mode.
"log_level": "DEBUG"
},
//run_as_daemon:false by default
"run_as_daemon": false,
//relaunch_on_error:false by default,if true,the program will be restart by parent after exit;
"relaunch_on_error": false,
//use_sendfile:true by default,if ture,the program will
//use sendfile() system-call to send static file to client;
"use_sendfile": true,
//use_gzip:true by default,use gzip to compress the response body's content;
"use_gzip": true,
//static_files_cache_time:5 (seconds) by default,the time in which static file response is cached,
//0 means cache forever,the negative value means no cache
"static_files_cache_time":5
}
}

2
lib/inc/drogon/HttpAppFramework.h
View File

@ -109,6 +109,8 @@ class HttpAppFramework : public trantor::NonCopyable
virtual void enableSendfile(bool sendFile) = 0;
virtual void enableGzip(bool useGzip) = 0;
virtual bool useGzip() const = 0;
virtual void setStaticFilesCacheTime(int cacheTime) = 0;
virtual int staticFilesCacheTime() const = 0;
private:
virtual void registerHttpApiController(const std::string &pathPattern,

2
lib/src/ConfigLoader.cc
View File

@ -160,6 +160,8 @@ static void loadApp(const Json::Value &app)
HttpAppFramework::instance().enableSendfile(useSendfile);
auto useGzip = app.get("use_gzip", true).asBool();
HttpAppFramework::instance().enableGzip(useGzip);
auto staticFilesCacheTime=app.get("static_files_cache_time",5).asInt();
HttpAppFramework::instance().setStaticFilesCacheTime(staticFilesCacheTime);
}
static void loadListeners(const Json::Value &listeners)
{

156
lib/src/HttpAppFrameworkImpl.cc
View File

@ -707,12 +707,86 @@ void HttpAppFrameworkImpl::onAsyncRequest(const HttpRequestPtr &req, const std::
transform(filetype.begin(), filetype.end(), filetype.begin(), tolower);
if (_fileTypeSet.find(filetype) != _fileTypeSet.end())
{
LOG_INFO << "file query!";
//LOG_INFO << "file query!";
std::string filePath = _rootPath + path;
std::shared_ptr<HttpResponseImpl> resp = std::make_shared<HttpResponseImpl>();
//find cached response
HttpResponsePtr cachedResp;
{
std::lock_guard<std::mutex> guard(_staticFilesCacheMutex);
if (_staticFilesCache.find(filePath) != _staticFilesCache.end())
{
cachedResp = _staticFilesCache[filePath].lock();
if (!cachedResp)
{
_staticFilesCache.erase(filePath);
}
}
}
if (needSetJsessionid)
resp->addCookie("JSESSIONID", session_id);
//check last modified time,rfc2616-14.25
//If-Modified-Since: Mon, 15 Oct 2018 06:26:33 GMT
if (_enableLastModify)
{
if (cachedResp)
{
if (cachedResp->getHeader("Last-Modified") == req->getHeader("If-Modified-Since"))
{
resp->setStatusCode(HttpResponse::k304NotModified);
if (needSetJsessionid)
{
resp->addCookie("JSESSIONID", session_id);
}
callback(resp);
return;
}
}
else
{
struct stat fileStat;
LOG_TRACE << "enabled LastModify";
if (stat(filePath.c_str(), &fileStat) >= 0)
{
LOG_TRACE << "last modify time:" << fileStat.st_mtime;
struct tm tm1;
gmtime_r(&fileStat.st_mtime, &tm1);
std::string timeStr;
timeStr.resize(64);
auto len = strftime((char *)timeStr.data(), timeStr.size(), "%a, %d %b %Y %T GMT", &tm1);
timeStr.resize(len);
std::string modiStr = req->getHeader("If-Modified-Since");
if (modiStr == timeStr && !modiStr.empty())
{
LOG_TRACE << "not Modified!";
resp->setStatusCode(HttpResponse::k304NotModified);
if (needSetJsessionid)
{
resp->addCookie("JSESSIONID", session_id);
}
callback(resp);
return;
}
resp->addHeader("Last-Modified", timeStr);
resp->addHeader("Expires", "Thu, 01 Jan 1970 00:00:00 GMT");
}
}
}
if (cachedResp)
{
if (needSetJsessionid)
{
//make a copy
auto newCachedResp = std::make_shared<HttpResponseImpl>(*std::dynamic_pointer_cast<HttpResponseImpl>(cachedResp));
newCachedResp->addCookie("JSESSIONID", session_id);
newCachedResp->setExpiredTime(-1);
callback(newCachedResp);
return;
}
callback(cachedResp);
return;
}
// pick a Content-Type for the file
if (filetype == "html")
@ -757,6 +831,20 @@ void HttpAppFrameworkImpl::onAsyncRequest(const HttpRequestPtr &req, const std::
resp->setContentTypeCode(CT_APPLICATION_OCTET_STREAM);
readSendFile(filePath, req, resp);
if (needSetJsessionid)
{
auto newCachedResp = resp;
if (resp->expiredTime() >= 0)
{
//make a copy
newCachedResp = std::make_shared<HttpResponseImpl>(*std::dynamic_pointer_cast<HttpResponseImpl>(resp));
newCachedResp->setExpiredTime(-1);
}
newCachedResp->addCookie("JSESSIONID", session_id);
callback(newCachedResp);
return;
}
callback(resp);
return;
}
@ -836,7 +924,6 @@ void HttpAppFrameworkImpl::onAsyncRequest(const HttpRequestPtr &req, const std::
}
else
{
LOG_ERROR << "can't find controller " << ctrlName;
}
});
@ -969,11 +1056,10 @@ void HttpAppFrameworkImpl::onAsyncRequest(const HttpRequestPtr &req, const std::
}
}
void HttpAppFrameworkImpl::readSendFile(const std::string &filePath, const HttpRequestPtr &req, const HttpResponsePtr resp)
void HttpAppFrameworkImpl::readSendFile(const std::string &filePath, const HttpRequestPtr &req, const HttpResponsePtr &resp)
{
//If-Modified-Since: Wed Jun 15 08:57:30 2016 GMT
std::ifstream infile(filePath, std::ifstream::binary);
LOG_INFO << "send http file:" << filePath;
LOG_TRACE << "send http file:" << filePath;
if (!infile)
{
@ -982,54 +1068,42 @@ void HttpAppFrameworkImpl::readSendFile(const std::string &filePath, const HttpR
return;
}
if (_enableLastModify)
{
struct stat fileStat;
LOG_TRACE << "enabled LastModify";
if (stat(filePath.c_str(), &fileStat) >= 0)
{
LOG_TRACE << "last modify time:" << fileStat.st_mtime;
struct tm tm1;
gmtime_r(&fileStat.st_mtime, &tm1);
char timeBuf[64];
asctime_r(&tm1, timeBuf);
std::string timeStr(timeBuf);
std::string::size_type len = timeStr.length();
std::string lastModified = timeStr.substr(0, len - 1) + " GMT";
std::string modiStr = req->getHeader("If-Modified-Since");
if (modiStr != "" && modiStr == lastModified)
{
LOG_TRACE << "not Modified!";
resp->setStatusCode(HttpResponse::k304NotModified);
return;
}
resp->addHeader("Last-Modified", lastModified);
resp->addHeader("Expires", "Thu, 01 Jan 1970 00:00:00 GMT");
}
}
std::streambuf *pbuf = infile.rdbuf();
std::streamsize filesize = pbuf->pubseekoff(0, infile.end);
pbuf->pubseekoff(0, infile.beg); // rewind
if (_useSendfile &&
(req->getHeader("Accept-Encoding").find("gzip") == std::string::npos ||
resp->getContentTypeCode() >= CT_APPLICATION_OCTET_STREAM))
resp->getContentTypeCode() >= CT_APPLICATION_OCTET_STREAM) &&
filesize > 1024 * 100)
{
//binary file or no gzip supported by client
std::dynamic_pointer_cast<HttpResponseImpl>(resp)->setSendfile(filePath);
}
else
{
std::streambuf *pbuf = infile.rdbuf();
std::streamsize size = pbuf->pubseekoff(0, infile.end);
pbuf->pubseekoff(0, infile.beg); // rewind
std::string str;
str.resize(size);
pbuf->sgetn(&str[0], size);
LOG_INFO << "file len:" << str.length();
str.resize(filesize);
pbuf->sgetn(&str[0], filesize);
resp->setBody(std::move(str));
}
resp->setStatusCode(HttpResponse::k200OK);
//cache the response for 5 seconds by default
if (_staticFilesCacheTime >= 0)
{
resp->setExpiredTime(_staticFilesCacheTime);
_responseCacheMap->insert(filePath, resp, resp->expiredTime(), [=]() {
std::lock_guard<std::mutex> guard(_staticFilesCacheMutex);
_staticFilesCache.erase(filePath);
});
{
std::lock_guard<std::mutex> guard(_staticFilesCacheMutex);
_staticFilesCache[filePath] = resp;
}
}
}
trantor::EventLoop *HttpAppFrameworkImpl::loop()

8
lib/src/HttpAppFrameworkImpl.h
View File

@ -25,6 +25,7 @@
#include <string>
#include <vector>
#include <memory>
#include <mutex>
#include <regex>
namespace drogon
@ -73,6 +74,8 @@ class HttpAppFrameworkImpl : public HttpAppFramework
virtual void enableSendfile(bool sendFile) override { _useSendfile = sendFile; }
virtual void enableGzip(bool useGzip) override { _useGzip = useGzip; }
virtual bool useGzip() const override { return _useGzip; }
virtual void setStaticFilesCacheTime(int cacheTime) override { _staticFilesCacheTime = cacheTime; }
virtual int staticFilesCacheTime() const override { return _staticFilesCacheTime; }
virtual ~HttpAppFrameworkImpl()
{
//Destroy the following objects before _loop destruction
@ -95,7 +98,7 @@ class HttpAppFrameworkImpl : public HttpAppFramework
void onWebsockMessage(const WebSocketConnectionPtr &wsConnPtr, trantor::MsgBuffer *buffer);
void onWebsockDisconnect(const WebSocketConnectionPtr &wsConnPtr);
void onConnection(const TcpConnectionPtr &conn);
void readSendFile(const std::string &filePath, const HttpRequestPtr &req, const HttpResponsePtr resp);
void readSendFile(const std::string &filePath, const HttpRequestPtr &req, const HttpResponsePtr &resp);
void addApiPath(const std::string &path,
const HttpApiBinderBasePtr &binder,
const std::vector<std::string> &filters);
@ -190,5 +193,8 @@ class HttpAppFrameworkImpl : public HttpAppFramework
size_t _logfileSize = 100000000;
bool _useSendfile = true;
bool _useGzip = true;
int _staticFilesCacheTime = 5;
std::unordered_map<std::string, std::weak_ptr<HttpResponse>> _staticFilesCache;
std::mutex _staticFilesCacheMutex;
};
} // namespace drogon

8
lib/src/HttpContext.cc
View File

@ -339,12 +339,12 @@ bool HttpContext::parseResponse(MsgBuffer *buf)
if (response_._left_body_length >= buf->readableBytes())
{
response_._left_body_length -= buf->readableBytes();
response_._body += std::string(buf->peek(), buf->readableBytes());
response_._bodyPtr->append(std::string(buf->peek(), buf->readableBytes()));
buf->retrieveAll();
}
else
{
response_._body += std::string(buf->peek(), response_._left_body_length);
response_._bodyPtr->append(std::string(buf->peek(), response_._left_body_length));
buf->retrieve(request_->contentLen);
response_._left_body_length = 0;
}
@ -359,7 +359,7 @@ bool HttpContext::parseResponse(MsgBuffer *buf)
}
else if (res_state_ == HttpResponseParseState::kExpectClose)
{
response_._body += std::string(buf->peek(), buf->readableBytes());
response_._bodyPtr->append(std::string(buf->peek(), buf->readableBytes()));
buf->retrieveAll();
break;
}
@ -396,7 +396,7 @@ bool HttpContext::parseResponse(MsgBuffer *buf)
if (*(buf->peek() + response_._current_chunk_length) == '\r' &&
*(buf->peek() + response_._current_chunk_length + 1) == '\n')
{
response_._body += std::string(buf->peek(), response_._current_chunk_length);
response_._bodyPtr->append(std::string(buf->peek(), response_._current_chunk_length));
buf->retrieve(response_._current_chunk_length + 2);
response_._current_chunk_length = 0;
res_state_ = HttpResponseParseState::kExpectChunkLen;

4
lib/src/HttpResponseImpl.cc
View File

@ -324,7 +324,7 @@ void HttpResponseImpl::makeHeaderString(MsgBuffer *output) const
output->append("\r\n");
if (_sendfileName.empty())
{
snprintf(buf, sizeof buf, "Content-Length: %lu\r\n", _body.size());
snprintf(buf, sizeof buf, "Content-Length: %lu\r\n", _bodyPtr->size());
}
else
{
@ -397,5 +397,5 @@ void HttpResponseImpl::appendToBuffer(MsgBuffer *output) const
output->append("\r\n\r\n");
LOG_TRACE << "reponse(no body):" << output->peek();
output->append(_body);
output->append(*_bodyPtr);
}

21
lib/src/HttpResponseImpl.h
View File

@ -47,7 +47,8 @@ class HttpResponseImpl : public HttpResponse
: _statusCode(kUnknown),
_closeConnection(false),
_left_body_length(0),
_current_chunk_length(0)
_current_chunk_length(0),
_bodyPtr(new std::string())
{
}
virtual HttpStatusCode statusCode() override
@ -205,11 +206,11 @@ class HttpResponseImpl : public HttpResponse
virtual void setBody(const std::string &body) override
{
_body = body;
_bodyPtr = std::make_shared<std::string>(body);
}
virtual void setBody(std::string &&body) override
{
_body = std::move(body);
_bodyPtr = std::make_shared<std::string>(std::move(body));
}
virtual void redirect(const std::string &url) override
@ -226,7 +227,7 @@ class HttpResponseImpl : public HttpResponse
_fullHeaderString.reset();
_headers.clear();
_cookies.clear();
_body.clear();
_bodyPtr.reset(new std::string());
_left_body_length = 0;
_current_chunk_length = 0;
}
@ -245,11 +246,11 @@ class HttpResponseImpl : public HttpResponse
virtual const std::string &getBody() const override
{
return _body;
return *_bodyPtr;
}
virtual std::string &getBody() override
{
return _body;
return *_bodyPtr;
}
void swap(HttpResponseImpl &that)
{
@ -259,7 +260,7 @@ class HttpResponseImpl : public HttpResponse
std::swap(_v, that._v);
_statusMessage.swap(that._statusMessage);
std::swap(_closeConnection, that._closeConnection);
_body.swap(that._body);
_bodyPtr.swap(that._bodyPtr);
std::swap(_left_body_length, that._left_body_length);
std::swap(_current_chunk_length, that._current_chunk_length);
std::swap(_contentType, that._contentType);
@ -274,7 +275,7 @@ class HttpResponseImpl : public HttpResponse
builder["collectComments"] = false;
JSONCPP_STRING errs;
std::unique_ptr<Json::CharReader> reader(builder.newCharReader());
if (!reader->parse(_body.data(), _body.data() + _body.size(), _jsonPtr.get(), &errs))
if (!reader->parse(_bodyPtr->data(), _bodyPtr->data() + _bodyPtr->size(), _jsonPtr.get(), &errs))
{
LOG_ERROR << errs;
_jsonPtr.reset();
@ -320,9 +321,11 @@ class HttpResponseImpl : public HttpResponse
Version _v;
std::string _statusMessage;
bool _closeConnection;
std::string _body;
size_t _left_body_length;
size_t _current_chunk_length;
std::shared_ptr<std::string> _bodyPtr;
uint8_t _contentType = CT_TEXT_HTML;
ssize_t _expriedTime = -1;