diff --git a/checkin_notes b/checkin_notes index 094cb8804d..df5ca8c194 100755 --- a/checkin_notes +++ b/checkin_notes @@ -10181,3 +10181,34 @@ Rom 9 Aug 2005 lib/ gui_rpc_client.h gui_rpc_client_ops.C + +David 9 August 2005 + - Initial checkin for Curl-based core client. + This will allow the client to use HTTPS for any purpose + (main page, scheduler URL, file upload/download), + and it will eventually eliminate the funky/complex + HTTP and proxy code in the core client. + + For now, this should have no effect on standard compiles; + everything is #ifdef'd to use the old HTTP code. + (from Carl Christensen) + If you want to build a Curl-based version: + build the libcurl library + Unix: in client/ rename Makefile_curl.am and build. + Win: rename boinc*_curl.vcproj and build + + client/ + client_state.C,h + file_xfer.h + gui_http.h + main.C + net_stats.h + next_xfer_curl.C,h (new) + Makefile_curl.am (new) + scheduler_op.h + curl-7.14.0/* (new) + lib/ + util.C + win_build/ + boinc_cli_curl.vcproj (new) + boincmgr_curl.vcproj (new) diff --git a/client/Makefile.curl.am b/client/Makefile.curl.am new file mode 100644 index 0000000000..3c22562804 --- /dev/null +++ b/client/Makefile.curl.am @@ -0,0 +1,85 @@ +## -*- mode: make; tab-width: 4 -*- +## $Id$ + +include $(top_srcdir)/Makefile.incl + +# (for a while we used "-static -static-libgcc" on linux, but this is obsolete +# now) +#STATIC_FLAGS=@STATIC_FLAGS@ + +client-bin: @CLIENT_BIN_FILENAME@ + +LIBS += @CLIENTLIBS@ + +bin_PROGRAMS = boinc_client + +EXTRA_PROGRAMS = cpu_benchmark + +boinc_client_SOURCES = \ + acct_mgr.C \ + acct_setup.C \ + app.C \ + app_control.C \ + app_graphics.C \ + app_start.C \ + check_state.C \ + client_msgs.C \ + client_state.C \ + client_types.C \ + cs_account.C \ + cs_apps.C \ + cs_benchmark.C \ + cs_cmdline.C \ + cs_data.C \ + cs_files.C \ + cs_prefs.C \ + cs_scheduler.C \ + cs_statefile.C \ + cs_trickle.C \ + dhrystone.C \ + dhrystone2.C \ + file_names.C \ + file_xfer.C \ + gui_http.C \ + gui_rpc_server.C \ + gui_rpc_server_ops.C \ + hostinfo_network.C \ + hostinfo_unix.C \ + http_curl.C \ + log_flags.C \ + main.C \ + net_stats.C \ + net_xfer_curl.C \ + pers_file_xfer.C \ + scheduler_op.C \ + ss_logic.C \ + time_stats.C \ + whetstone.C + +boinc_client_DEPENDENCIES = $(LIBRSA) +boinc_client_CPPFLAGS = -D_USE_CURL -I../../curl-7.14.0/include -I $(srcdir)/win $(AM_CPPFLAGS) -O3 +boinc_client_LDFLAGS = -static-libgcc +boinc_client_LDADD = -L/usr/local/ssl/lib -lssl -L../../curl-7.14.0/lib -lcurl -L../lib -lboinc $(RSA_LIBS) $(PTHREAD_LIBS) +#boinc_client_LDFLAGS = $(STATIC_FLAGS) + +# the following don't do anything +cpu_benchmark_SOURCES = whetstone.C dhrystone.C +cpu_benchmark_CFLAGS = -O3 $(AM_CFLAGS) + +all-local: client-bin + +# make a hard link to the client name. +@CLIENT_BIN_FILENAME@: boinc_client + rm -f $@ + @LN@ $? $@ + @STRIP@ $@ + +## these source files need to be specified because no rule uses them. + +EXTRA_DIST = *.h \ + mac \ + translation \ + win + +clean-local: + rm -f @CLIENT_BIN_FILENAME@ diff --git a/client/client_state.C b/client/client_state.C index 9afcddb3bd..9e167c236b 100644 --- a/client/client_state.C +++ b/client/client_state.C @@ -56,7 +56,11 @@ #include "hostinfo.h" #include "hostinfo_network.h" #include "network.h" +#ifdef _USE_CURL +#include "http_curl.h" +#else #include "http.h" +#endif #include "log_flags.h" #include "client_msgs.h" #include "client_state.h" diff --git a/client/client_state.h b/client/client_state.h index c4ec94dbc8..1c00b8dc92 100644 --- a/client/client_state.h +++ b/client/client_state.h @@ -33,17 +33,23 @@ #include "gui_rpc_server.h" #include "gui_http.h" #include "hostinfo.h" -#include "http.h" #include "language.h" #include "miofile.h" #include "net_stats.h" -#include "net_xfer.h" #include "pers_file_xfer.h" #include "prefs.h" #include "scheduler_op.h" #include "ss_logic.h" #include "time_stats.h" +#ifdef _USE_CURL +#include "http_curl.h" +#include "net_xfer_curl.h" +#else +#include "http.h" +#include "net_xfer.h" +#endif + #define USER_RUN_REQUEST_ALWAYS 1 #define USER_RUN_REQUEST_AUTO 2 #define USER_RUN_REQUEST_NEVER 3 diff --git a/client/file_xfer.h b/client/file_xfer.h index 263756e74b..e3de7029fc 100644 --- a/client/file_xfer.h +++ b/client/file_xfer.h @@ -30,7 +30,11 @@ // - upload authentication #include "client_types.h" +#ifdef _USE_CURL +#include "http_curl.h" +#else #include "http.h" +#endif class FILE_XFER : public HTTP_OP { public: diff --git a/client/gui_http.h b/client/gui_http.h index fda74d9d2a..1afe51a514 100644 --- a/client/gui_http.h +++ b/client/gui_http.h @@ -26,7 +26,11 @@ using std::string; +#ifdef _USE_CURL +#include "http_curl.h" +#else #include "http.h" +#endif // base class for various types of ops // diff --git a/client/http_curl.C b/client/http_curl.C new file mode 100644 index 0000000000..41f0fd5279 --- /dev/null +++ b/client/http_curl.C @@ -0,0 +1,733 @@ +// Berkeley Open Infrastructure for Network Computing +// http://boinc.berkeley.edu +// Copyright (C) 2005 University of California +// +// This is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; +// either version 2.1 of the License, or (at your option) any later version. +// +// This software is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +// See the GNU Lesser General Public License for more details. +// +// To view the GNU Lesser General Public License visit +// http://www.gnu.org/copyleft/lesser.html +// or write to the Free Software Foundation, Inc., +// 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +#ifdef _USE_CURL // SSL clients will use https.C + +#include "cpp.h" + +#ifdef _WIN32 +#include "boinc_win.h" +#else +#include +#include +#include +#include +//#include +#include +#include +#endif + +#include "error_numbers.h" +#include "filesys.h" +#include "util.h" + +//#include "network.h" +#include "client_msgs.h" +#include "base64.h" +#include "http_curl.h" + +#define CONNECTED_STATE_NOT_CONNECTED 0 +#define CONNECTED_STATE_CONNECTED 1 +#define CONNECTED_STATE_UNKNOWN 2 + +#define HTTP_BLOCKSIZE 16384 + +using std::string; +using std::istringstream; +using std::vector; +using std::getline; + +extern CURLM* g_curlMulti; // global curl multi handle for http/s +static char g_user_agent_string[256] = {""}; +//static const char g_content_type[] = {"Content-Type: application/octet-stream"}; +// CMC Note: old BOINC used the above, but libcurl seems to like the following: +static const char g_content_type[] = {"Content-Type: application/x-www-form-urlencoded"}; + + +/** as an example of usage of http_op -- the scheduler typically requires + just the following: + +http_op + +http_op_state flag +http_op_retval + +set_proxy (g_state.proxy_info) +init_get() +init_post() + +http_ops + -> insert + remove + +*/ + +// Breaks a HTTP URL down into its server, port and file components +// format of url: +// [http[s]://]host.dom.dom[:port][/dir/file] +// +void parse_url(const char* url, char* host, int &port, char* file) { + char* p; + char buf[256]; + bool bSSL = false; + + //const short csiLen = bSSL ? 8 : 7; // get right size of http/s string comparator + + // strip off http:// if present + // + //if (strncmp(url, (bSSL ? "https://" : "http://"), csiLen) == 0) { + if (strncmp(url, "http://", 7) == 0) { + safe_strcpy(buf, url+7); + } else { + // wait, may be https:// + if (strncmp(url, "https://", 8) == 0) { + safe_strcpy(buf, url+8); + bSSL = true; // retain the fact that this was a secure http url + } + else { // no http:// or https:// prepended on url + safe_strcpy(buf, url); + } + } + + // parse and strip off file part if present + // + p = strchr(buf, '/'); + if (p) { + strcpy(file, p+1); + *p = 0; + } else { + strcpy(file, ""); + } + + // parse and strip off port if present + // + p = strchr(buf,':'); + if (p) { + port = atol(p+1); + *p = 0; + } else { + // CMC note: if they didn't pass in a port #, + // but the url starts with https://, assume they + // want a secure port (HTTPS, port 443) + port = (bSSL ? 443 : 80); + } + + // what remains is the host + // + strcpy(host, buf); +} + +void get_user_agent_string() { +/* + sprintf(g_user_agent_string, "User-Agent: BOINC client (%s %d.%02d)", + HOSTTYPE, BOINC_MAJOR_VERSION, BOINC_MINOR_VERSION + ); + */ + + sprintf(g_user_agent_string, "BOINC client (%s %d.%02d)", + HOSTTYPE, BOINC_MAJOR_VERSION, BOINC_MINOR_VERSION + ); + +} + +HTTP_OP::HTTP_OP() { + strcpy(m_url, ""); // CMC added this to "preserve" the url for libcurl + strcpy(url_hostname, ""); + strcpy(filename, ""); + content_length = 0; + file_offset = 0; + strcpy(request_header, ""); + http_op_state = HTTP_STATE_IDLE; + http_op_type = HTTP_OP_NONE; + http_op_retval = 0; +} + +HTTP_OP::~HTTP_OP() { +} + +// Initialize HTTP GET operation +// +int HTTP_OP::init_get( + const char* url, const char* out, bool del_old_file, double off +) { + //char proxy_buf[256]; + + if (del_old_file) { + unlink(out); + } + req1 = NULL; // not using req1, but init_post2 uses it + file_offset = off; + safe_strcpy(m_url, url); + parse_url(url, url_hostname, port, filename); + NET_XFER::init(url_hostname, port, HTTP_BLOCKSIZE); + // usually have an outfile on a get + if (off != 0) { + bytes_xferred = off; + } + http_op_type = HTTP_OP_GET; + http_op_state = HTTP_STATE_CONNECTING; + return HTTP_OP::libcurl_exec(url, NULL, out, off, false); +} + +// Initialize HTTP POST operation +// +int HTTP_OP::init_post( + const char* url, const char* in, const char* out +) { + int retval; + double size; + + SCOPE_MSG_LOG scope_messages(log_messages, CLIENT_MSG_LOG::DEBUG_HTTP); + req1 = NULL; // not using req1, but init_post2 uses it + + strcpy(m_url, url); + parse_url(url, url_hostname, port, filename); + + if (in) { // we should pretty much always have an in file for _post, optional in _post2 + strcpy(infile, in); + retval = file_size(infile, size); + if (retval) return retval; + content_length = (int)size; + } + //PROXY::init(url_hostname, port); + NET_XFER::init(url_hostname,port, HTTP_BLOCKSIZE); + http_op_type = HTTP_OP_POST; + http_op_state = HTTP_STATE_CONNECTING; + scope_messages.printf("HTTP_OP::init_post(): %p io_done %d\n", this, io_done); + return HTTP_OP::libcurl_exec(url, in, out, 0.0, true); +} + +// the following will do an HTTP GET or POST using libcurl, polling at the net_xfer level +int HTTP_OP::libcurl_exec(const char* url, const char* in, const char* out, + double offset, bool bPost) +{ + CURLMcode curlMErr; + CURLcode curlErr; + //char proxy_buf[256]; + char strTmp[128]; + + // get user agent string + if (g_user_agent_string[0] == 0x00) + get_user_agent_string(); + + if (in) + strcpy(infile, in); + if (out) { + bTempOutfile = false; + strcpy(outfile, out); + } + else + { //CMC -- I always want an outfile for the server response, delete when op done + bTempOutfile = true; + memset(outfile, 0x00, _MAX_PATH); +#ifdef _WIN32 + char* ptrName; + ptrName = _tempnam("./", "blc"); + if (ptrName) { + strcpy(outfile, ptrName); + free(ptrName); + } +#else // use mkstemp on Mac & Linux due to security issues + strcpy(outfile, "blcXXXXXX"); // a template for the mkstemp + mkstemp(outfile); +#endif + if (outfile[0] == 0x00) { + // oh well we're desparate, use tmpnam! + tmpnam(outfile); + } + } + + // setup libcurl handle + SCOPE_MSG_LOG scope_messages(log_messages, CLIENT_MSG_LOG::DEBUG_NET_XFER); + + // CMC -- init the curlEasy handle and setup options + // the polling will do the actual start of the HTTP/S transaction + + curlEasy = curl_easy_init(); // get a curl_easy handle to use + if (!curlEasy) + { + msg_printf(0, MSG_ERROR, "%s\n", "Couldn't create curlEasy handle\n"); + return 1; // returns 0 (CURLM_OK) on successful handle creation + } + + // OK, we have a handle, now open an asynchronous libcurl connection + + // set the URL to use + curlErr = curl_easy_setopt(curlEasy, CURLOPT_URL, m_url); + + /* + CURLOPT_SSL_VERIFYHOST + +Pass a long as parameter. + +This option determines whether curl verifies that the server claims to be who you want it to be. + +When negotiating an SSL connection, the server sends a certificate indicating its identity. + +When CURLOPT_SSL_VERIFYHOST is 2, that certificate must indicate that the server is the server to which you meant to connect, or the connection fails. + +Curl considers the server the intended one when the Common Name field or a Subject Alternate Name field in the certificate matches the host name in the URL to which you told Curl to connect. + +When the value is 1, the certificate must contain a Common Name field, but it doesn't matter what name it says. (This is not ordinarily a useful setting). + +When the value is 0, the connection succeeds regardless of the names in the certificate. + +The default, since 7.10, is 2. + +The checking this option controls is of the identity that the server claims. The server could be lying. To control lying, see CURLOPT_SSL_VERIFYPEER. + */ + curlErr = curl_easy_setopt(curlEasy, CURLOPT_SSL_VERIFYHOST, 2L); + + // disable "tough" certificate checking (i.e. self-signed is OK) + // if zero below, will accept self-signed certificates (cert not 3rd party trusted) + // if non-zero below, you need a valid 3rd party CA (i.e. Verisign, Thawte) + curlErr = curl_easy_setopt(curlEasy, CURLOPT_SSL_VERIFYPEER, 0L); + + // set the user agent as this boinc client & version + curlErr = curl_easy_setopt(curlEasy, CURLOPT_USERAGENT, g_user_agent_string); + + // note: custom headers here means EVERYTHING needs to be set here, + // including form content types. Probably best just to set the user agent above + // and let libcurl handle the headers. + + // create custom header for BOINC + /* + + pcurlList = curl_slist_append(pcurlList, "Pragma: no-cache"); + pcurlList = curl_slist_append(pcurlList, "Cache-Control: no-cache"); + + pcurlList = curl_slist_append(pcurlList, g_user_agent_string); + + //sprintf(strTmp, "Host: %s:%d", url_hostname, port); + //pcurlList = curl_slist_append(pcurlList, strTmp); + if (offset>0.0f) { + file_offset = offset; + sprintf(strTmp, "Range: bytes=%.0f-", offset); + pcurlList = curl_slist_append(pcurlList, strTmp); + } + pcurlList = curl_slist_append(pcurlList, "Connection: close"); + pcurlList = curl_slist_append(pcurlList, "Accept: *\/*"); + + // "Proxy-Authorization: Basic %s\015\012" + if (pcurlList) { // send custom headers if required + curlErr = curl_easy_setopt(curlEasy, CURLOPT_HTTPHEADER, pcurlList); + } + + */ + + // bypass any signal handlers that curl may want to install + curlErr = curl_easy_setopt(curlEasy, CURLOPT_NOSIGNAL, 1L); + // bypass progress meter + curlErr = curl_easy_setopt(curlEasy, CURLOPT_NOPROGRESS, 1L); + + // force curl to use HTTP/1.0 (which the old BOINC http libraries did) + //curlErr = curl_easy_setopt(curlEasy, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_0); + curlErr = curl_easy_setopt(curlEasy, CURLOPT_MAXREDIRS, 5L); + curlErr = curl_easy_setopt(curlEasy, CURLOPT_AUTOREFERER, 1L); + curlErr = curl_easy_setopt(curlEasy, CURLOPT_FOLLOWLOCATION, 1L); + + // setup any proxy they may need + setupProxyCurl(); + + // set the content type in the header (defined at the top -- app/octect-stream) + pcurlList = curl_slist_append(pcurlList, g_content_type); + + // we'll need to setup an output file for the reply + if (outfile && strlen(outfile)>0) + { + fileOut = boinc_fopen(outfile, "wb"); + if (!fileOut) { + msg_printf(NULL, MSG_ERROR, + "HTTP_CURL:libcurl_exec(): Can't setup HTTP response output file %s\n", outfile); + io_done = true; + http_op_retval = ERR_FOPEN; + http_op_state = HTTP_STATE_DONE; + return -3; + } + // CMC Note: we can make the libcurl_write "fancier" in the future, + // for now it just fwrite's to the file request, which is sufficient + curlErr = curl_easy_setopt(curlEasy, CURLOPT_WRITEFUNCTION, libcurl_write); + // note that in my lib_write I'm sending in a pointer to this instance of HTTP_OP + curlErr = curl_easy_setopt(curlEasy, CURLOPT_WRITEDATA, this); + } + + if (bPost) + { // POST + want_upload = true; + want_download = false; + if (infile && strlen(infile)>0) { + fileIn = boinc_fopen(infile, "rb"); + if (!fileIn) { + msg_printf(NULL, MSG_ERROR, "HTTP_CURL:libcurl_exec(): no input file %s\n", infile); + io_done = true; + http_op_retval = ERR_FOPEN; + http_op_state = HTTP_STATE_DONE; + return -1; + } + } + // send offset range if needed + if (offset>0.0f) { + file_offset = offset; + sprintf(strTmp, "Range: bytes=%.0f-", offset); + pcurlList = curl_slist_append(pcurlList, strTmp); + } + if (pcurlList) { // send custom headers if required + curlErr = curl_easy_setopt(curlEasy, CURLOPT_HTTPHEADER, pcurlList); + } + + // set the data file info to read for the PUT/POST + // note the use of this curl typedef for large filesizes + + /* HTTP PUT method + curl_off_t fs = (curl_off_t) content_length; + curlErr = curl_easy_setopt(curlEasy, CURLOPT_POSTFIELDS, NULL); + curlErr = curl_easy_setopt(curlEasy, CURLOPT_INFILESIZE, content_length); + curlErr = curl_easy_setopt(curlEasy, CURLOPT_READDATA, fileIn); + curlErr = curl_easy_setopt(curlEasy, CURLOPT_INFILESIZE_LARGE, fs); + curlErr = curl_easy_setopt(curlEasy, CURLOPT_PUT, 1L); + */ + + // HTTP POST method + // set the multipart form for the file -- boinc just has the one section (file) + + /* CMC -- if we ever want to do POST as multipart forms someday + // (many seem to prefer it that way, i.e. libcurl) + + pcurlFormStart = pcurlFormEnd = NULL; + curl_formadd(&pcurlFormStart, &pcurlFormEnd, + CURLFORM_FILECONTENT, infile, + CURLFORM_CONTENTSLENGTH, content_length, + CURLFORM_CONTENTTYPE, g_content_type, + CURLFORM_END); + curl_formadd(&post, &last, + CURLFORM_COPYNAME, "logotype-image", + CURLFORM_FILECONTENT, "curl.png", CURLFORM_END); + */ + + //curlErr = curl_easy_setopt(curlEasy, CURLOPT_HTTPPOST, pcurlFormStart); + curl_off_t fs = (curl_off_t) content_length; + + pByte = NULL; + lSeek = 0; // initialize the vars we're going to use for byte transfers + + // CMC Note: we can make the libcurl_read "fancier" in the future, + // for now it just fwrite's to the file request, which is sufficient + curlErr = curl_easy_setopt(curlEasy, CURLOPT_POSTFIELDS, NULL); + curlErr = curl_easy_setopt(curlEasy, CURLOPT_POSTFIELDSIZE_LARGE, fs); + curlErr = curl_easy_setopt(curlEasy, CURLOPT_READFUNCTION, libcurl_read); + // note that in my lib_write I'm sending in a pointer to this instance of HTTP_OP + curlErr = curl_easy_setopt(curlEasy, CURLOPT_READDATA, this); + + curlErr = curl_easy_setopt(curlEasy, CURLOPT_POST, 1L); + } + else + { // GET + want_upload = false; + want_download = true; + + // now write the header, pcurlList gets freed in net_xfer_curl + if (pcurlList) { // send custom headers if required + curlErr = curl_easy_setopt(curlEasy, CURLOPT_HTTPHEADER, pcurlList); + } + + // setup the GET! + curlErr = curl_easy_setopt(curlEasy, CURLOPT_HTTPGET, 1L); + } + + // last but not least, add this to the curl_multi + curlMErr = curl_multi_add_handle(g_curlMulti, curlEasy); + if (curlMErr != CURLM_OK && curlMErr != CURLM_CALL_MULTI_PERFORM) + { // bad error, couldn't attach easy curl handle + msg_printf(0, MSG_ERROR, "%s\n", "Couldn't add curlEasy handle to curlMulti\n"); + return curlMErr; // returns 0 (CURLM_OK) on successful handle creation + } + + // that should about do it, the net_xfer_set polling will actually start the transaction + // for this HTTP_OP + return 0; +} + + +// Initialize HTTP POST operation +// +int HTTP_OP::init_post2( + const char* url, char* r1, const char* in, double offset +) { + int retval; + double size; + //char proxy_buf[256]; + + strcpy(m_url, url); + parse_url(url, url_hostname, port, filename); + //PROXY::init(url_hostname, port); + //NET_XFER::init(get_proxy_server_name(url_hostname),get_proxy_port(port), HTTP_BLOCKSIZE); + NET_XFER::init(url_hostname,port, HTTP_BLOCKSIZE); + req1 = r1; + if (in) { + safe_strcpy(infile, in); + file_offset = offset; + retval = file_size(infile, size); + if (retval) { + printf("HTTP::init_post2: couldn't get file size\n"); + return retval; + } + content_length = (int)size - (int)offset; + } + content_length += strlen(req1); + http_op_type = HTTP_OP_POST2; + http_op_state = HTTP_STATE_CONNECTING; + return HTTP_OP::libcurl_exec(url, in, NULL, offset, true); +} + +// Returns true if the HTTP operation is complete +// +bool HTTP_OP::http_op_done() { + return (http_op_state == HTTP_STATE_DONE); +} + +HTTP_OP_SET::HTTP_OP_SET(NET_XFER_SET* p) { + net_xfers = p; +} + +// Adds an HTTP_OP to the set +// +int HTTP_OP_SET::insert(HTTP_OP* ho) { + int retval; + retval = net_xfers->insert(ho); + if (retval) return retval; + http_ops.push_back(ho); + return 0; +} + +bool HTTP_OP_SET::poll() +{ + // with libcurl you really just hand it up to the net_xfer_set level + //return net_xfers->poll(); + return true; +} + +// Remove an HTTP_OP from the set +// +int HTTP_OP_SET::remove(HTTP_OP* p) { + vector::iterator iter; + + net_xfers->remove(p); + + iter = http_ops.begin(); + while (iter != http_ops.end()) { + if (*iter == p) { + http_ops.erase(iter); + return 0; + } + iter++; + } + msg_printf(NULL, MSG_ERROR, "HTTP_OP_SET::remove(): not found\n"); + return ERR_NOT_FOUND; +} + +int HTTP_OP_SET::nops() { + return http_ops.size(); +} + + +int HTTP_OP::set_proxy(PROXY_INFO *new_pi) { + pi.use_http_proxy = new_pi->use_http_proxy; + strcpy(pi.http_user_name, new_pi->http_user_name); + strcpy(pi.http_user_passwd, new_pi->http_user_passwd); + strcpy(pi.http_server_name, new_pi->http_server_name); + pi.http_server_port = new_pi->http_server_port; + pi.use_http_auth = new_pi->use_http_auth; + + pi.use_socks_proxy = new_pi->use_socks_proxy; + strcpy(pi.socks5_user_name, new_pi->socks5_user_name); + strcpy(pi.socks5_user_passwd, new_pi->socks5_user_passwd); + strcpy(pi.socks_server_name, new_pi->socks_server_name); + pi.socks_server_port = new_pi->socks_server_port; + pi.socks_version = new_pi->socks_version; + + return 0; +} + +size_t libcurl_write(void *ptr, size_t size, size_t nmemb, HTTP_OP* phop) +{ + // take the stream param as a FILE* and write to disk + //CMC TODO: maybe assert stRead == size*nmemb, add exception handling on phop members + size_t stWrite = fwrite(ptr, size, nmemb, (FILE*) phop->fileOut); + phop->bytes_xferred += (double)(stWrite); + //if (phop->bytes_xferred == (int) phop->content_length) + //{ // that's all we need! + //} + phop->update_speed(); // this should update the transfer speed + return stWrite; +} + +size_t libcurl_read( void *ptr, size_t size, size_t nmemb, HTTP_OP* phop) +{ + // OK here's the deal -- phop points to the calling object, + // which has already pre-opened the file. we'll want to + // use pByte as a pointer for fseek calls into the file, and + // write out size*nmemb # of bytes to ptr + + // take the stream param as a FILE* and write to disk + //if (pByte) delete [] pByte; + //pByte = new unsigned char[content_length]; + //memset(pByte, 0x00, content_length); // may as will initialize it! + + // note that fileIn was opened earlier, go to lSeek from the top and read from there + size_t stSend = size * nmemb; + size_t stRead = 0; + +// if (phop->http_op_type == HTTP_OP_POST2) { + if (phop->req1 && ! phop->bSentHeader) { // need to send headers first, then data file + // uck -- the way 'post2' is done, you have to read the + // header bytes, and then cram on the file upload bytes + + // so requests from 0 to strlen(req1)-1 are from memory, + // and from strlen(req1) to content_length are from the file + + // just send the headers from htp->req1 if needed + if (phop->lSeek < (long) strlen(phop->req1)) { + // need to read header, either just starting to read (i.e. + // this is the first time in this function for this phop) + // or the last read didn't ask for the entire header + + stRead = strlen(phop->req1) - phop->lSeek; // how much of header left to read + + // only memcpy if request isn't out of bounds + if (stRead < 0) { + stRead = 0; + } + else { + memcpy(ptr, (void*)(phop->req1 + phop->lSeek), stRead); + } + phop->lSeek += (long) stRead; // increment lSeek to new position + phop->bytes_xferred += (double)(stRead); + // see if we're done with headers + phop->bSentHeader = (bool)(phop->lSeek >= (long) strlen(phop->req1)); + // reset lSeek if done to make it easier for file operations + if (phop->bSentHeader) phop->lSeek = 0; + return stRead; // + } + } + // now for file to read in (if any), also don't bother if this request + // was just enough for the header (which was taken care of above) + if (phop->fileIn) { + // note we'll have to fudge lSeek a little if there was + // also a header, just use a temp var + //size_t stOld = stRead; // we'll want to save the ptr location of last stRead + + // keep a separate pointer to "bump ahead" the pointer for the file data + // as ptr may have been used above for the header info + //unsigned char *ptr2; + // get the file seek offset, both from the offset requested (added) + // as well as the size of the header above discounted + // - ((phop->req1 && stRead>0) ? stRead : + // (phop->req1 ? strlen(phop->req1) : 0L)) + long lFileSeek = phop->lSeek + + (long) phop->file_offset; + fseek(phop->fileIn, lFileSeek, SEEK_SET); + if (!feof(phop->fileIn)) { // CMC TODO: better error checking for size*nmemb + // i.e. that we don't go overbounds of the file etc, we can check against + // content_length (which includes the strlen(req1) also) + // note the use of stOld to get to the right position in case the header was read in above + //ptr2 = (unsigned char*)ptr +(int)stOld; + stRead = fread(ptr, 1, stSend, phop->fileIn); + } + phop->lSeek += (long) stRead; // increment lSeek to new position + phop->bytes_xferred += (double)(stRead); + } + phop->update_speed(); // this should update the transfer speed + return stRead; +} + +void HTTP_OP::setupProxyCurl() +{ + // CMC: use the libcurl proxy routines with this object's proxy information struct + /* PROXY_INFO pi useful members: + pi.http_server_name + pi.http_server_port + pi.http_user_name + pi.http_user_passwd + pi.socks5_user_name + pi.socks5_user_passwd + pi.socks_server_name + pi.socks_server_port + pi.socks_version + pi.use_http_auth + pi.use_http_proxy + pi.use_socks_proxy + + Curl self-explanatory setopt params for proxies: + CURLOPT_HTTPPROXYTUNNEL + CURLOPT_PROXYTYPE (pass in CURLPROXY_HTTP or CURLPROXY_SOCKS5) + CURLOPT_PROXYPORT -- a long port # + CURLOPT_PROXY - pass in char* of the proxy url + CURLOPT_PROXYUSERPWD -- a char* in the format username:password + CURLOPT_HTTPAUTH -- pass in one of CURLAUTH_BASIC, CURLAUTH_DIGEST, + CURLAUTH_GSSNEGOTIATE, CURLAUTH_NTLM, CURLAUTH_ANY, CURLAUTH_ANYSAFE + CURLOPT_PROXYAUTH -- "or" | the above bitmasks -- only basic, digest, ntlm work + + */ + + CURLcode curlErr; + + // CMC Note: the string szCurlProxyUserPwd must remain in memory + // outside of this method (libcurl relies on it later when it makes + // the proxy connection), so it has been placed as a member data for HTTP_OP + memset(szCurlProxyUserPwd,0x00,128); + + if (pi.use_http_proxy) + { // setup a basic http proxy + curlErr = curl_easy_setopt(curlEasy, CURLOPT_PROXYTYPE, CURLPROXY_HTTP); + curlErr = curl_easy_setopt(curlEasy, CURLOPT_PROXYPORT, (long) pi.http_server_port); + curlErr = curl_easy_setopt(curlEasy, CURLOPT_PROXY, (char*) pi.http_server_name); + + if (pi.use_http_auth) + { +/* testing! + fprintf(stdout, "Using httpauth for proxy: %s:%d %s:%s\n", + pi.http_server_name, pi.http_server_port, + pi.http_user_name, pi.http_user_passwd); +*/ + sprintf(szCurlProxyUserPwd, "%s:%s", pi.http_user_name, pi.http_user_passwd); + curlErr = curl_easy_setopt(curlEasy, CURLOPT_PROXYAUTH, CURLAUTH_ANY); + curlErr = curl_easy_setopt(curlEasy, CURLOPT_PROXYUSERPWD, szCurlProxyUserPwd); + } + } + else { + if (pi.use_socks_proxy) + { + //pi.socks_version -- picks between socks5 & socks4 -- but libcurl only socks5! + curlErr = curl_easy_setopt(curlEasy, CURLOPT_PROXYTYPE, CURLPROXY_SOCKS5); + curlErr = curl_easy_setopt(curlEasy, CURLOPT_PROXYPORT, (long) pi.socks_server_port); + curlErr = curl_easy_setopt(curlEasy, CURLOPT_PROXY, (char*) pi.socks_server_name); + + if (strlen(pi.socks5_user_passwd)>0 || strlen(pi.socks5_user_name)>0) + { + sprintf(szCurlProxyUserPwd, "%s:%s", pi.socks5_user_name, pi.socks5_user_passwd); + curlErr = curl_easy_setopt(curlEasy, CURLOPT_PROXYAUTH, CURLAUTH_ANY); + curlErr = curl_easy_setopt(curlEasy, CURLOPT_PROXYUSERPWD, szCurlProxyUserPwd); + } + } + } +} + +const char *BOINC_RCSID_57f273bb60 = "$Id$"; + +#endif // not _USE_CURL diff --git a/client/http_curl.h b/client/http_curl.h new file mode 100644 index 0000000000..2837af7e61 --- /dev/null +++ b/client/http_curl.h @@ -0,0 +1,156 @@ +// Berkeley Open Infrastructure for Network Computing +// http://boinc.berkeley.edu +// Copyright (C) 2005 University of California +// +// This is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; +// either version 2.1 of the License, or (at your option) any later version. +// +// This software is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +// See the GNU Lesser General Public License for more details. +// +// To view the GNU Lesser General Public License visit +// http://www.gnu.org/copyleft/lesser.html +// or write to the Free Software Foundation, Inc., +// 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +// HTTP_OP represents an HTTP operation. +// There are variants for GET and POST, +// and for the data source/sink (see below). + +// CMC Note: This was redone to use libcurl ref: http://curl.haxx.se/libcurl +// to allow ease of use for SSL/HTTPS etc + +// -D_USE_CURL in your C flags will build a "curl" BOINC client +// note the ifdef _USE_CURL in the *_curl.C/.h files to bypass +// this code (similarly the ifndef _USE_CURL in http/net_xfer/proxy.C/.h) + +#ifdef _USE_CURL // only adds this file if user wants to link against libcurl + +#ifndef _HTTP_ +#define _HTTP_ + +// SOCKS #defines +#define SOCKS_VERSION_4 0x04 +#define SOCKS_VERSION_5 0x05 + +// now include the curl library: originally from http://curl.haxx.se/libcurl +#include + +#include "proxy_info.h" +#include "net_xfer_curl.h" + +// official HTTP status codes +#define HTTP_STATUS_OK 200 +#define HTTP_STATUS_PARTIAL_CONTENT 206 +#define HTTP_STATUS_RANGE_REQUEST_ERROR 416 +#define HTTP_STATUS_MOVED_PERM 301 +#define HTTP_STATUS_MOVED_TEMP 302 +#define HTTP_STATUS_NOT_FOUND 404 +#define HTTP_STATUS_PROXY_AUTH_REQ 407 +#define HTTP_STATUS_INTERNAL_SERVER_ERROR 500 +#define HTTP_STATUS_SERVICE_UNAVAILABLE 503 + +#define HTTP_OP_NONE 0 +#define HTTP_OP_GET 1 + // data sink is a file (used for file download) +#define HTTP_OP_POST 2 + // data source and sink are files (used for scheduler op) +#define HTTP_OP_HEAD 4 + // no data (used for file upload) +#define HTTP_OP_POST2 5 + // a POST operation where the request comes from a combination + // of a string and a file w/offset, + // and the reply goes into a memory buffer. + // Used for file upload + +class HTTP_OP : public NET_XFER +{ +public: + HTTP_OP(); + ~HTTP_OP(); + + // proxy info + PROXY_INFO pi; + + int port; + char filename[256]; + char url_hostname[256]; + + // CMC added this to "preserve" the url for libcurl use, + // as we don't really need to do all the parsing stuff + char m_url[256]; + char szCurlProxyUserPwd[128]; // string needed for proxy username/password + + // the hostname part of the URL. + // May not be the host we connect to (if using proxy) + int content_length; + double file_offset; + char request_header[4096]; + //HTTP_REPLY_HEADER hrh; + // move these to net_xfer + /* + int http_op_state; // values below + int http_op_type; + int http_op_retval; + */ + // zero if success, or a BOINC error code, or an HTTP status code +//// bool proxy_auth_done; + + //int init_head(const char* url); + int init_get(const char* url, const char* outfile, bool del_old_file, double offset=0); + int init_post(const char* url, const char* infile, const char* outfile); + int init_post2( + const char* url, + char* req1, // first part of request. ALSO USED FOR REPLY + const char* infile, double offset // infile is NULL if no file sent + ); + bool http_op_done(); + int set_proxy(PROXY_INFO *new_pi); + void setupProxyCurl(); + +private: + // internal use in the class -- takes an init_get/post/post2 and turns it into + // an appropriate libcurl request + int libcurl_exec(const char* url, const char* in = NULL, const char* out = NULL, + double offset = 0.0f, bool bPost = true); + +}; + +// global function used by libcurl to write http replies to disk +size_t libcurl_write(void *ptr, size_t size, size_t nmemb, HTTP_OP* phop); +size_t libcurl_read( void *ptr, size_t size, size_t nmemb, HTTP_OP* phop); + +// represents a set of HTTP requests in progress +// +class HTTP_OP_SET { + std::vector http_ops; + NET_XFER_SET* net_xfers; +public: + HTTP_OP_SET(NET_XFER_SET*); + bool poll(); + int insert(HTTP_OP*); + int remove(HTTP_OP*); + int nops(); +}; + +#define HTTP_STATE_IDLE 0 +#define HTTP_STATE_CONNECTING 1 +#define HTTP_STATE_SOCKS_CONNECT 2 +#define HTTP_STATE_REQUEST_HEADER 3 +#define HTTP_STATE_REQUEST_BODY1 4 + // sending the string part of a POST2 operation +#define HTTP_STATE_REQUEST_BODY 5 +#define HTTP_STATE_REPLY_HEADER 6 +#define HTTP_STATE_REPLY_BODY 7 +#define HTTP_STATE_DONE 8 + +// default bSSL is false for compatibility with the uploader stuff, which will remain non-SSL +extern void parse_url(const char* url, char* host, int &port, char* file); + +#endif //__HTTP_H + +#endif // _USE_CURL \ No newline at end of file diff --git a/client/main.C b/client/main.C index 086005e8e5..ab3c3b6e20 100644 --- a/client/main.C +++ b/client/main.C @@ -58,8 +58,10 @@ typedef void (CALLBACK* IdleTrackerTerm)(); #include "client_msgs.h" #include "main.h" - - +#ifdef _USE_CURL // CMC do the curl initialization here in main +extern int curl_init(void); +extern int curl_cleanup(void); +#endif // Display a message to the user. // Depending on the priority, the message may be more or less obtrusive @@ -332,6 +334,10 @@ int main(int argc, char** argv) { init_core_client(argc, argv); +#ifdef _USE_CURL + curl_init(); +#endif + #ifdef _WIN32 // Initialize WinSock if ( WinsockInitialize() != 0 ) { @@ -425,6 +431,10 @@ int main(int argc, char** argv) { return ERR_IO; } #endif + +#ifdef _USE_CURL + curl_cleanup(); +#endif return retval; } diff --git a/client/net_stats.h b/client/net_stats.h index f7d828c85b..44b6f93075 100644 --- a/client/net_stats.h +++ b/client/net_stats.h @@ -28,7 +28,11 @@ #include #endif +#ifdef _USE_CURL +#include "net_xfer_curl.h" +#else #include "net_xfer.h" +#endif #include "miofile.h" // there's one of these each for upload and download diff --git a/client/net_xfer_curl.C b/client/net_xfer_curl.C new file mode 100644 index 0000000000..3950e15641 --- /dev/null +++ b/client/net_xfer_curl.C @@ -0,0 +1,577 @@ +// Berkeley Open Infrastructure for Network Computing +// http://boinc.berkeley.edu +// Copyright (C) 2005 University of California +// +// This is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; +// either version 2.1 of the License, or (at your option) any later version. +// +// This software is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +// See the GNU Lesser General Public License for more details. +// +// To view the GNU Lesser General Public License visit +// http://www.gnu.org/copyleft/lesser.html +// or write to the Free Software Foundation, Inc., +// 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +#ifdef _USE_CURL + +#include "cpp.h" + +#ifdef _WIN32 +#ifndef _CONSOLE +#include "stdafx.h" +#include "wingui_mainwindow.h" +#else +#include "boinc_win.h" +#endif + +#include "win_util.h" + +#endif + +#ifndef _WIN32 +#include +#include +#include +#include +#include +#include +#include +#include +#include +#endif + +#include "error_numbers.h" +#include "net_xfer_curl.h" +#include "util.h" +#include "network.h" +#include "filesys.h" + +#include "client_types.h" +#include "client_state.h" +#include "client_msgs.h" + +using std::vector; + +// if an active transfer doesn't get any activity +// in this many seconds, error out +// +#define NET_XFER_TIMEOUT 600 + +CURLM* g_curlMulti = NULL; // global curl for this module, can handle http & https + +// the file descriptor sets need to be global so libcurl has access always +fd_set read_fds, write_fds, error_fds; + +// call these once at the start of the program and once at the end (init & cleanup of course) +extern int curl_init() +{ + curl_global_init(CURL_GLOBAL_ALL); + g_curlMulti = curl_multi_init(); + return (int)(g_curlMulti == NULL); +} + +extern int curl_cleanup() +{ + if (g_curlMulti) { + curl_multi_cleanup(g_curlMulti); + } + return 0; +} + +void NET_XFER::reset() { + req1 = NULL; + strcpy(infile, ""); + strcpy(outfile, ""); + bTempOutfile = true; + is_connected = false; + want_download = false; + want_upload = false; + do_file_io = true; // CMC Note: should I default to true, i.e. this does all i/o? + io_done = false; + fileIn = NULL; + fileOut = NULL; + io_ready = true; // don't allow higher levels to do i/o? + error = 0; + file_read_buf_offset = 0; + file_read_buf_len = 0; + bytes_xferred = 0; + xfer_speed = 0; + bSentHeader = false; + close_socket(); +} + +NET_XFER::NET_XFER() { + pcurlList = NULL; // these have to be NULL, just in constructor + curlEasy = NULL; + pcurlFormStart = NULL; + pcurlFormEnd = NULL; + pByte = NULL; + lSeek = 0; + reset(); +} + +NET_XFER::~NET_XFER() { + close_socket(); + close_file(); +} + +void NET_XFER::close_socket() { + // CMC: this just cleans up the curlEasy, and "spoofs" the old close_socket + if (pcurlList) { + curl_slist_free_all(pcurlList); + pcurlList = NULL; + } + if (curlEasy && pcurlFormStart) + { + curl_formfree(pcurlFormStart); + curl_formfree(pcurlFormEnd); + pcurlFormStart = pcurlFormEnd = NULL; + } + if (curlEasy && g_curlMulti) + { // release this handle + curl_multi_remove_handle(g_curlMulti, curlEasy); + curl_easy_cleanup(curlEasy); + curlEasy = NULL; + } +} + +void NET_XFER::close_file() { + if (fileIn) { + fclose(fileIn); + fileIn = NULL; + } + if (fileOut) { + fclose(fileOut); + fileOut = NULL; + } + if (pByte) { //free any read memory used + delete [] pByte; + pByte = NULL; + } +} + +void NET_XFER::init(char* host, int p, int b) { + reset(); + safe_strcpy(hostname, host); + port = p; + blocksize = (b > MAX_BLOCKSIZE ? MAX_BLOCKSIZE : b); + start_time = gstate.now; + reset_timeout(); +} + +bool NET_XFER::check_timeout(bool time_passed) { + if (seconds_until_timeout == 0) { + io_done = true; + error = ERR_TIMEOUT; + return true; + } + if (time_passed) { + seconds_until_timeout--; + } + return false; +} + +void NET_XFER::reset_timeout() { + seconds_until_timeout = NET_XFER_TIMEOUT; +} + +char* NET_XFER::get_hostname() { + return hostname; +} + +NET_XFER_SET::NET_XFER_SET() { + max_bytes_sec_up = 0; + max_bytes_sec_down = 0; + bytes_left_up = 0; + bytes_left_down = 0; + bytes_up = 0; + bytes_down = 0; + up_active = false; + down_active = false; +} + +// Connect to a server, +// and if successful insert the NET_XFER object into the set +// +int NET_XFER_SET::insert(NET_XFER* nxp) { + //int retval = nxp->open_server(); + //if (retval) return retval; + net_xfers.push_back(nxp); + return 0; +} + +// Remove a NET_XFER object from the set +// +int NET_XFER_SET::remove(NET_XFER* nxp) { + vector::iterator iter; + + iter = net_xfers.begin(); + while (iter != net_xfers.end()) { + if (*iter == nxp) { + net_xfers.erase(iter); + return 0; + } + iter++; + } + msg_printf(NULL, MSG_ERROR, "NET_XFER_SET::remove(): not found\n"); + return ERR_NOT_FOUND; +} + +// Transfer data to/from active sockets. +// Keep doing I/O until would block, or we hit rate limits, +// or .5 second goes by +// +bool NET_XFER_SET::poll() { + double bytes_xferred =0.0f; + int retval; + bool action = false; + // CMC probably don't need this loop in libcurl, the regular polling interval should suffice? +// while (1) { + retval = do_select(bytes_xferred, 0.5); + action = (retval == 0); +// if (retval) break; +// if (bytes_xferred == 0) break; +// action = true; +// if ((dtime() - gstate.now) > 0.5) break; +// } + //bInHere = false; + return action; +} + +static void double_to_timeval(double x, timeval& t) { + t.tv_sec = (int)x; + t.tv_usec = (int)(1000000*(x - (int)x)); +} + +// Wait at most x seconds for network I/O to become possible, +// then do up to about .5 seconds of I/O. +// +int NET_XFER_SET::net_sleep(double x) { + // CMC -- this seems to be the culprit for the race condition + // as we're polling from client_state::do_something() as well + // as from main->client_state::net_sleep() + // need to ensure that the do_select isn't instantaneous -- + // i.e. it will boinc_sleep if no network transactions + + int retval; + double bytes_xferred; + + retval = do_select(bytes_xferred, x); + if (retval) return retval; + if (bytes_xferred) { + return poll(); + } + return 0; +} + +// do a select with the given timeout, +// then do I/O on as many sockets as possible, subject to rate limits +// Transfer at most one block per socket. +// +int NET_XFER_SET::do_select(double& bytes_transferred, double timeout) { + int iNumMsg; + NET_XFER* nxf = NULL; + struct timeval tv; + bool time_passed = false; + CURLMsg *pcurlMsg = NULL; + + SCOPE_MSG_LOG scope_messages(log_messages, CLIENT_MSG_LOG::DEBUG_NET_XFER); + bytes_transferred = 0; + + if (!net_xfers.size()) { + boinc_sleep(timeout); // sleep a little, don't just return + return 1; // no pending or running net transactions + } + + // if a second has gone by, do rate-limit accounting + // + time_t t = time(0); + if (t != last_time) { + time_passed = true; + last_time = (int)t; + if (bytes_left_up < max_bytes_sec_up) { + bytes_left_up += max_bytes_sec_up; + } + if (bytes_left_down < max_bytes_sec_down) { + bytes_left_down += max_bytes_sec_down; + } + } + + int max_fd = 0, iRunning = 0; // curl flags for max # of fds & # running queries + CURLMcode curlMErr; + CURLcode curlErr; + double_to_timeval(timeout, tv); + + // get the data waiting for transfer in or out + // note that I use timeout value so that we don't "hog" in this loop + while ( (curlMErr = curl_multi_perform(g_curlMulti, &iRunning)) == + CURLM_CALL_MULTI_PERFORM + && ((double) time(0) - (double) t) < timeout); + + // read messages from curl that may have come in from the above loop + while ( (pcurlMsg = curl_multi_info_read(g_curlMulti, &iNumMsg)) ) + { + // if we have a msg, then somebody finished + // can check also with pcurlMsg->msg == CURLMSG_DONE + if ((nxf = lookup_curl(pcurlMsg->easy_handle)) ) { + // we have a message from one of our http_ops + // get the response code for this request + curlErr = curl_easy_getinfo(nxf->curlEasy, + CURLINFO_RESPONSE_CODE, &nxf->response); + curlErr = curl_easy_getinfo(nxf->curlEasy, + CURLINFO_OS_ERRNO, &nxf->error); + + nxf->io_done = true; + nxf->io_ready = false; + + if (nxf->want_download) { + bytes_down += nxf->bytes_xferred; + // get xfer speed (don't mess with bytes_xferred, that's in write function + curlErr = curl_easy_getinfo(nxf->curlEasy, + CURLINFO_SPEED_DOWNLOAD, &nxf->xfer_speed); + } + if (nxf->want_upload) { + bytes_up += nxf->bytes_xferred; + // get xfer speed (don't mess with bytes_xferred, that's in write function + curlErr = curl_easy_getinfo(nxf->curlEasy, + CURLINFO_SPEED_UPLOAD, &nxf->xfer_speed); + } + + // the op is done if curl_multi_msg_read gave us a msg for this http_op + nxf->http_op_state = HTTP_STATE_DONE; + + // 200 is a good HTTP response code + // It may not mean the data received is "good" + // (the calling program will have to check/parse that) + // but it at least means that the server operation + // went through fine + nxf->http_op_retval = nxf->response - 200; + + if (!nxf->http_op_retval && nxf->http_op_type == HTTP_OP_POST2) { + // for a successfully completed request on a "post2" -- + // read in the temp file into req1 memory + fclose(nxf->fileOut); + double dSize = 0.0f; + file_size(nxf->outfile, dSize); + nxf->fileOut = boinc_fopen(nxf->outfile, "rb"); + if (!nxf->fileOut) + { // ack, can't open back up! + nxf->response = 1; // flag as a bad response for a possible retry later + } + fseek(nxf->fileOut, 0, SEEK_SET); + // CMC Note: req1 is a pointer to "header" which is 4096 + memset(nxf->req1, 0x00, 4096); + fread(nxf->req1, 1, (size_t) dSize, nxf->fileOut); + } + + // close files and "sockets" (i.e. libcurl handles) + nxf->close_file(); + nxf->close_socket(); + + // finally remove the tmpfile if not explicitly set + if (nxf->bTempOutfile) + boinc_delete_file(nxf->outfile); + } + } + + // reset and get curl fds + FD_ZERO(&read_fds); + FD_ZERO(&write_fds); + FD_ZERO(&error_fds); + + // "prime" the fdset for the next libcurl multi op + curlMErr = curl_multi_fdset(g_curlMulti, &read_fds, &write_fds, &error_fds, &max_fd); + + /* CMC - this is dumb, if data coming in, don't sleep on it! + // need to sleep any leftover time + double dSleepItOff = timeout - (double)(time(0) - t); + if (dSleepItOff > 0.0f) + boinc_sleep(dSleepItOff); + */ + + return 0; + + /* CMC obsolete + if (curlMErr != CURLM_OK) { + return ERR_SELECT; // something wrong + } + else if (max_fd == -1) { // no fd's, nothing to do! + up_active = down_active = false; + return retval; + } + // do a select on all libcurl handles + // http://curl.haxx.se/libcurl/c/curl_multi_fdset.html + timeval tv_curl; + tv_curl.tv_sec = 5; + tv_curl.tv_usec = 0; // libcurl docs say use a small (single digit) # of seconds + n = select(max_fd+1, &read_fds, &write_fds, &error_fds, &tv_curl); + + switch (n) { + case -1: // error on select + return ERR_SELECT; + break; + case 0: + return 1; + break; + default: + break; + } + + // at this point, libcurl should have done all the pending transfers + // now go interate net_xfers -- examine each one for status, errors, etc + return 1; + */ +} + +// Return the NET_XFER object whose socket matches fd +NET_XFER* NET_XFER_SET::lookup_curl(CURL* pcurl) +{ + for (unsigned int i=0; icurlEasy == pcurl) { + return net_xfers[i]; + } + } + return 0; +} + +/* CMC not needed for libcurl + +// transfer up to a block of data; return #bytes transferred +// +int NET_XFER::do_xfer(int& nbytes_transferred) { + // Leave these as signed ints so recv/send can return errors + int n, m, nleft; + bool would_block; + char buf[MAX_BLOCKSIZE]; + + nbytes_transferred = 0; + + SCOPE_MSG_LOG scope_messages(log_messages, CLIENT_MSG_LOG::DEBUG_NET_XFER); + + if (want_download) { +#ifdef _WIN32 + n = recv(socket, buf, blocksize, 0); +#else + n = read(socket, buf, blocksize); +#endif + scope_messages.printf("NET_XFER::do_xfer(): read %d bytes from socket %d\n", n, socket); + if (n == 0) { + io_done = true; + want_download = false; + } else if (n < 0) { + io_done = true; + error = ERR_READ; + } else { + nbytes_transferred += n; + bytes_xferred += n; + m = fwrite(buf, 1, n, file); + if (n != m) { + fprintf(stdout, "Error: incomplete disk write\n"); + io_done = true; + error = ERR_FWRITE; + } + } + } else if (want_upload) { + // If we've sent the current contents of the buffer, + // read the next block + // + if (file_read_buf_len == file_read_buf_offset) { + m = fread(file_read_buf, 1, blocksize, file); + if (m == 0) { + want_upload = false; + io_done = true; + return 0; + } else if (m < 0) { + io_done = true; + error = ERR_FREAD; + return 0; + } + file_read_buf_len = m; + file_read_buf_offset = 0; + } + nleft = file_read_buf_len - file_read_buf_offset; + while (nleft) { +#ifdef WIN32 + n = send(socket, file_read_buf+file_read_buf_offset, nleft, 0); + would_block = (WSAGetLastError() == WSAEWOULDBLOCK); +#else + n = write(socket, file_read_buf+file_read_buf_offset, nleft); + would_block = (errno == EAGAIN); +#endif + if (would_block && n < 0) n = 0; + scope_messages.printf( + "NET_XFER::do_xfer(): wrote %d bytes to socket %d%s\n", + n, socket, (would_block?", would have blocked":"") + ); + if (n < 0 && !would_block) { + error = ERR_WRITE; + io_done = true; + break; + } + + file_read_buf_offset += n; + nbytes_transferred += n; + bytes_xferred += n; + + if (n < nleft || would_block) { + break; + } + + nleft -= n; + } + } + return 0; +} +*/ + +// Update the transfer speed for this NET_XFER +// called on every I/O +// +void NET_XFER::update_speed() { + double delta_t = dtime() - start_time; + if (delta_t > 0) { + xfer_speed = bytes_xferred / delta_t; +#if 0 + // TODO: figure out what to do here + } else if (xfer_speed == 0) { + xfer_speed = 999999999; +#endif + } +} + +void NET_XFER::got_error() { + // + error = ERR_IO; + io_done = true; + log_messages.printf( + CLIENT_MSG_LOG::DEBUG_NET_XFER, "IO error on socket %d\n", socket + ); +} + +// return true if an upload is currently in progress +// or has been since the last call to this. +// Similar for download. +void NET_XFER_SET::check_active(bool& up, bool& down) { + unsigned int i; + NET_XFER* nxp; + + up = up_active; + down = down_active; + for (i=0; iis_connected && nxp->do_file_io) { + nxp->want_download?down=true:up=true; + } + } + up_active = false; + down_active = false; +} + +const char *BOINC_RCSID_e0a7088e04 = "$Id$"; + +#endif //_USE_CURL \ No newline at end of file diff --git a/client/net_xfer_curl.h b/client/net_xfer_curl.h new file mode 100644 index 0000000000..8a163b8cf9 --- /dev/null +++ b/client/net_xfer_curl.h @@ -0,0 +1,147 @@ +// Berkeley Open Infrastructure for Network Computing +// http://boinc.berkeley.edu +// Copyright (C) 2005 University of California +// +// This is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; +// either version 2.1 of the License, or (at your option) any later version. +// +// This software is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +// See the GNU Lesser General Public License for more details. +// +// To view the GNU Lesser General Public License visit +// http://www.gnu.org/copyleft/lesser.html +// or write to the Free Software Foundation, Inc., +// 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +#ifdef _USE_CURL + +#ifndef _NET_XFER_ +#define _NET_XFER_ + +#ifndef _WIN32 +#include +#include +#include +#endif + +// now include the curl library: originally from http://curl.haxx.se/libcurl +#include + +extern CURLM* g_curlMulti; // the global libcurl multi handle + +// The following classes implement polling (non-blocking) I/O +// between a disk file (or memory block) and a socket + +#define MAX_BLOCKSIZE 16384 + +// global functions for starting & stopping libcurl +extern int curl_init(); +extern int curl_cleanup(); + +// represents a network connection, either being accessed directly +// or being transferred to/from a file +// +class NET_XFER { +public: + FILE* fileIn; + FILE* fileOut; // CMC need an output file for POST responses + CURL* curlEasy; // the "easy curl" handle for this net_xfer request + struct curl_slist *pcurlList; // curl slist for http headers + struct curl_httppost *pcurlFormStart; // a pointer to a form item for POST + struct curl_httppost *pcurlFormEnd; // a pointer to a form item for POST + unsigned char* pByte; // pointer to bytes for reading via libcurl_read function + long lSeek; // this is a pointer within the file we're reading + char infile[_MAX_PATH]; + char outfile[_MAX_PATH]; + bool bTempOutfile; // CMC -- flag that outfile is really a tempfile we should delete + char* req1; + bool bSentHeader; // CMC -- a flag that I already sent the header + + // int socket; // CMC -- deprecated, net_xfer's via curlEasy handle above + char hostname[256]; // The host we're connecting to (possibly a proxy) + bool is_connected; + bool want_download; // at most one should be true + bool want_upload; + bool do_file_io; + // If true: poll() should transfer data to/from file + // (in which case "file" and blocksize are relevant) + // If false: set io_ready (higher layers will do I/O) + bool io_done; + // set to true when the current transfer is over: + // - the transfer timed out (not activity for a long time) + // - network connect failed + // - got EOF on socket read (0 bytes, select indicated I/O ready) + // - error on disk write (e.g. volume full) + // - reached end of disk file on upload + // - got file read error on upload + // - write to socket failed on upload + bool io_ready; + // Signals higher layers that they can read or write socket now + // (used if !do_file_io) + long error; + long response; + int port; + int blocksize; + double start_time; + double xfer_speed; + double bytes_xferred; // bytes transferred in this session + double content_length; + char file_read_buf[MAX_BLOCKSIZE]; + int file_read_buf_offset, file_read_buf_len; + int seconds_until_timeout; + + // CMC - moved from http_op + int http_op_state; // values below + int http_op_type; + int http_op_retval; + + NET_XFER(); + ~NET_XFER(); + void reset(); + void init(char* host, int port, int blocksize); + int get_ip_addr(int &ip_addr); + //int open_server(); + void close_socket(); + void close_file(); + // int do_xfer(int&); // CMC not needed for libcurl + void update_speed(); + void got_error(); + char* get_hostname(); + bool check_timeout(bool); + void reset_timeout(); +}; + +// bandwidth limitation is implemented at this level, as follows: +// There are limits max_bytes_sec_up and max_bytes_sec_down. +// We keep track of the last time and bytes_left_up and bytes_left_down; +// Each second we reset these to zero. + +class NET_XFER_SET { + std::vector net_xfers; +public: + NET_XFER_SET(); + double max_bytes_sec_up, max_bytes_sec_down; + // user-specified limits on throughput + double bytes_left_up, bytes_left_down; + // bytes left to transfer in the current second + double bytes_up, bytes_down; + // total bytes transferred + bool up_active, down_active; + // has there been transfer activity since last call to check_active()? + int last_time; + int insert(NET_XFER*); + int remove(NET_XFER*); + bool poll(); + int net_sleep(double); + int do_select(double& bytes_transferred, double timeout); + NET_XFER* lookup_curl(CURL* pcurl); // lookup by easycurl handle + void check_active(bool&, bool&); +}; + +#endif // _H + +#endif // _USE_CURL \ No newline at end of file diff --git a/client/scheduler_op.h b/client/scheduler_op.h index a4f805b614..c4a0a2e7b4 100644 --- a/client/scheduler_op.h +++ b/client/scheduler_op.h @@ -23,7 +23,11 @@ #include #include "client_types.h" +#ifdef _USE_CURL +#include "http_curl.h" +#else #include "http.h" +#endif #include "prefs.h" // SCHEDULER_OP encapsulates the mechanism for diff --git a/lib/util.C b/lib/util.C index 895fbc2606..d1486f7c25 100755 --- a/lib/util.C +++ b/lib/util.C @@ -447,6 +447,7 @@ void escape_url_readable(char *in, char* out) { out[y] = 0; } + // Canonicalize a master url. // - Convert the first part of a URL (before the "://") to http://, // or prepend it @@ -456,10 +457,12 @@ void escape_url_readable(char *in, char* out) { void canonicalize_master_url(char* url) { char buf[1024]; size_t n; + bool bSSL = false; // keep track if they sent in https:// char *p = strstr(url, "://"); if (p) { - strcpy(buf, p+3); + bSSL = (bool) (p == url + 5); + strcpy(buf, p+3); } else { strcpy(buf, url); } @@ -472,7 +475,7 @@ void canonicalize_master_url(char* url) { if (buf[n-1] != '/') { strcat(buf, "/"); } - sprintf(url, "http://%s", buf); + sprintf(url, "http%s://%s", (bSSL ? "s" : ""), buf); // CMC Here -- add SSL if needed } // is the string a valid master URL, in canonical form? @@ -480,10 +483,19 @@ void canonicalize_master_url(char* url) { bool valid_master_url(char* buf) { char* p, *q; size_t n; + bool bSSL = false; p = strstr(buf, "http://"); - if (p != buf) return false; - q = p+strlen("http://"); + if (p != buf) { + // allow https + p = strstr(buf, "https://"); + if (p == buf) { + bSSL = true; + } else { + return false; // no http or https, it's bad! + } + } + q = p+strlen(bSSL ? "https://" : "http://"); p = strstr(q, "."); if (!p) return false; if (p == q) return false; diff --git a/win_build/boinc_cli_curl.vcproj b/win_build/boinc_cli_curl.vcproj new file mode 100644 index 0000000000..5417481571 --- /dev/null +++ b/win_build/boinc_cli_curl.vcproj @@ -0,0 +1,890 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/win_build/boincmgr_curl.vcproj b/win_build/boincmgr_curl.vcproj new file mode 100644 index 0000000000..7488b37b75 --- /dev/null +++ b/win_build/boincmgr_curl.vcproj @@ -0,0 +1,589 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +