Browse Source

evhttpd implementation

- *Replace usage of boost::asio with [libevent2](http://libevent.org/)*.
boost::asio is not part of C++11, so unlike other boost there is no
forwards-compatibility reason to stick with it. Together with #4738 (convert
json_spirit to UniValue), this rids Bitcoin Core of the worst offenders with
regard to compile-time slowness.

- *Replace spit-and-duct-tape http server with evhttp*. Front-end http handling
is handled by libevent, a work queue (with configurable depth and parallelism)
is used to handle application requests.

- *Wrap HTTP request in C++ class*; this makes the application code mostly
HTTP-server-neutral

- *Refactor RPC to move all http-specific code to a separate file*.
Theoreticaly this can allow building without HTTP server but with another RPC
backend, e.g. Qt's debug console (currently not implemented) or future RPC
mechanisms people may want to use.

- *HTTP dispatch mechanism*; services (e.g., RPC, REST) register which URL
paths they want to handle.

By using a proven, high-performance asynchronous networking library (also used
by Tor) and HTTP server, problems such as #5674, #5655, #344 should be avoided.

What works? bitcoind, bitcoin-cli, bitcoin-qt. Unit tests and RPC/REST tests
pass. The aim for now is everything but SSL support.

Configuration options:

- `-rpcthreads`: repurposed as "number of  work handler threads". Still
defaults to 4.

- `-rpcworkqueue`: maximum depth of work queue. When this is reached, new
requests will return a 500 Internal Error.

- `-rpctimeout`: inactivity time, in seconds, after which to disconnect a
client.

- `-debug=http`: low-level http activity logging
pull/1/head
Wladimir J. van der Laan 8 years ago
parent
commit
40b556d374
  1. 4
      src/Makefile.am
  2. 133
      src/bitcoin-cli.cpp
  3. 9
      src/bitcoind.cpp
  4. 201
      src/httprpc.cpp
  5. 37
      src/httprpc.h
  6. 586
      src/httpserver.cpp
  7. 138
      src/httpserver.h
  8. 44
      src/init.cpp
  9. 2
      src/init.h
  10. 9
      src/qt/bitcoin.cpp
  11. 243
      src/rest.cpp
  12. 229
      src/rpcprotocol.cpp
  13. 87
      src/rpcprotocol.h
  14. 554
      src/rpcserver.cpp
  15. 72
      src/rpcserver.h

4
src/Makefile.am

@ -98,6 +98,8 @@ BITCOIN_CORE_H = \ @@ -98,6 +98,8 @@ BITCOIN_CORE_H = \
eccryptoverify.h \
ecwrapper.h \
hash.h \
httprpc.h \
httpserver.h \
init.h \
key.h \
keystore.h \
@ -170,6 +172,8 @@ libbitcoin_server_a_SOURCES = \ @@ -170,6 +172,8 @@ libbitcoin_server_a_SOURCES = \
bloom.cpp \
chain.cpp \
checkpoints.cpp \
httprpc.cpp \
httpserver.cpp \
init.cpp \
leveldbwrapper.cpp \
main.cpp \

133
src/bitcoin-cli.cpp

@ -11,6 +11,12 @@ @@ -11,6 +11,12 @@
#include "utilstrencodings.h"
#include <boost/filesystem/operations.hpp>
#include <stdio.h>
#include <event2/event.h>
#include <event2/http.h>
#include <event2/buffer.h>
#include <event2/keyvalq_struct.h>
#include "univalue/univalue.h"
@ -32,9 +38,6 @@ std::string HelpMessageCli() @@ -32,9 +38,6 @@ std::string HelpMessageCli()
strUsage += HelpMessageOpt("-rpcuser=<user>", _("Username for JSON-RPC connections"));
strUsage += HelpMessageOpt("-rpcpassword=<pw>", _("Password for JSON-RPC connections"));
strUsage += HelpMessageGroup(_("SSL options: (see the Bitcoin Wiki for SSL setup instructions)"));
strUsage += HelpMessageOpt("-rpcssl", _("Use OpenSSL (https) for JSON-RPC connections"));
return strUsage;
}
@ -92,32 +95,75 @@ static bool AppInitRPC(int argc, char* argv[]) @@ -92,32 +95,75 @@ static bool AppInitRPC(int argc, char* argv[])
fprintf(stderr, "Error: Invalid combination of -regtest and -testnet.\n");
return false;
}
if (GetBoolArg("-rpcssl", false))
{
fprintf(stderr, "Error: SSL mode for RPC (-rpcssl) is no longer supported.\n");
return false;
}
return true;
}
UniValue CallRPC(const string& strMethod, const UniValue& params)
/** Reply structure for request_done to fill in */
struct HTTPReply
{
// Connect to localhost
bool fUseSSL = GetBoolArg("-rpcssl", false);
boost::asio::io_service io_service;
boost::asio::ssl::context context(io_service, boost::asio::ssl::context::sslv23);
context.set_options(boost::asio::ssl::context::no_sslv2 | boost::asio::ssl::context::no_sslv3);
boost::asio::ssl::stream<boost::asio::ip::tcp::socket> sslStream(io_service, context);
SSLIOStreamDevice<boost::asio::ip::tcp> d(sslStream, fUseSSL);
boost::iostreams::stream< SSLIOStreamDevice<boost::asio::ip::tcp> > stream(d);
const bool fConnected = d.connect(GetArg("-rpcconnect", "127.0.0.1"), GetArg("-rpcport", itostr(BaseParams().RPCPort())));
if (!fConnected)
throw CConnectionFailed("couldn't connect to server");
int status;
std::string body;
};
static void http_request_done(struct evhttp_request *req, void *ctx)
{
HTTPReply *reply = static_cast<HTTPReply*>(ctx);
if (req == NULL) {
/* If req is NULL, it means an error occurred while connecting, but
* I'm not sure how to find out which one. We also don't really care.
*/
reply->status = 0;
return;
}
// Find credentials to use
reply->status = evhttp_request_get_response_code(req);
struct evbuffer *buf = evhttp_request_get_input_buffer(req);
if (buf)
{
size_t size = evbuffer_get_length(buf);
const char *data = (const char*)evbuffer_pullup(buf, size);
if (data)
reply->body = std::string(data, size);
evbuffer_drain(buf, size);
}
}
UniValue CallRPC(const string& strMethod, const UniValue& params)
{
std::string host = GetArg("-rpcconnect", "127.0.0.1");
int port = GetArg("-rpcport", BaseParams().RPCPort());
// Create event base
struct event_base *base = event_base_new(); // TODO RAII
if (!base)
throw runtime_error("cannot create event_base");
// Synchronously look up hostname
struct evhttp_connection *evcon = evhttp_connection_base_new(base, NULL, host.c_str(), port); // TODO RAII
if (evcon == NULL)
throw runtime_error("create connection failed");
evhttp_connection_set_timeout(evcon, GetArg("-rpctimeout", 30));
HTTPReply response;
struct evhttp_request *req = evhttp_request_new(http_request_done, (void*)&response); // TODO RAII
if (req == NULL)
throw runtime_error("create http request failed");
// Get credentials
std::string strRPCUserColonPass;
if (mapArgs["-rpcpassword"] == "") {
// Try fall back to cookie-based authentication if no password is provided
if (!GetAuthCookie(&strRPCUserColonPass)) {
throw runtime_error(strprintf(
_("You must set rpcpassword=<password> in the configuration file:\n%s\n"
"If the file does not exist, create it with owner-readable-only file permissions."),
_("Could not locate RPC credentials. No authentication cookie could be found, and no rpcpassword is set in the configuration file (%s)"),
GetConfigFile().string().c_str()));
}
@ -125,34 +171,41 @@ UniValue CallRPC(const string& strMethod, const UniValue& params) @@ -125,34 +171,41 @@ UniValue CallRPC(const string& strMethod, const UniValue& params)
strRPCUserColonPass = mapArgs["-rpcuser"] + ":" + mapArgs["-rpcpassword"];
}
// HTTP basic authentication
map<string, string> mapRequestHeaders;
mapRequestHeaders["Authorization"] = string("Basic ") + EncodeBase64(strRPCUserColonPass);
// Send request
string strRequest = JSONRPCRequest(strMethod, params, 1);
string strPost = HTTPPost(strRequest, mapRequestHeaders);
stream << strPost << std::flush;
// Receive HTTP reply status
int nProto = 0;
int nStatus = ReadHTTPStatus(stream, nProto);
struct evkeyvalq *output_headers = evhttp_request_get_output_headers(req);
assert(output_headers);
evhttp_add_header(output_headers, "Host", host.c_str());
evhttp_add_header(output_headers, "Connection", "close");
evhttp_add_header(output_headers, "Authorization", (std::string("Basic ") + EncodeBase64(strRPCUserColonPass)).c_str());
// Attach request data
std::string strRequest = JSONRPCRequest(strMethod, params, 1);
struct evbuffer * output_buffer = evhttp_request_get_output_buffer(req);
assert(output_buffer);
evbuffer_add(output_buffer, strRequest.data(), strRequest.size());
int r = evhttp_make_request(evcon, req, EVHTTP_REQ_POST, "/");
if (r != 0) {
evhttp_connection_free(evcon);
event_base_free(base);
throw CConnectionFailed("send http request failed");
}
// Receive HTTP reply message headers and body
map<string, string> mapHeaders;
string strReply;
ReadHTTPMessage(stream, mapHeaders, strReply, nProto, std::numeric_limits<size_t>::max());
event_base_dispatch(base);
evhttp_connection_free(evcon);
event_base_free(base);
if (nStatus == HTTP_UNAUTHORIZED)
if (response.status == 0)
throw CConnectionFailed("couldn't connect to server");
else if (response.status == HTTP_UNAUTHORIZED)
throw runtime_error("incorrect rpcuser or rpcpassword (authorization failed)");
else if (nStatus >= 400 && nStatus != HTTP_BAD_REQUEST && nStatus != HTTP_NOT_FOUND && nStatus != HTTP_INTERNAL_SERVER_ERROR)
throw runtime_error(strprintf("server returned HTTP error %d", nStatus));
else if (strReply.empty())
else if (response.status >= 400 && response.status != HTTP_BAD_REQUEST && response.status != HTTP_NOT_FOUND && response.status != HTTP_INTERNAL_SERVER_ERROR)
throw runtime_error(strprintf("server returned HTTP error %d", response.status));
else if (response.body.empty())
throw runtime_error("no response from server");
// Parse reply
UniValue valReply(UniValue::VSTR);
if (!valReply.read(strReply))
if (!valReply.read(response.body))
throw runtime_error("couldn't parse reply from server");
const UniValue& reply = valReply.get_obj();
if (reply.empty())

9
src/bitcoind.cpp

@ -10,11 +10,16 @@ @@ -10,11 +10,16 @@
#include "noui.h"
#include "scheduler.h"
#include "util.h"
#include "httpserver.h"
#include "httprpc.h"
#include "rpcserver.h"
#include <boost/algorithm/string/predicate.hpp>
#include <boost/filesystem.hpp>
#include <boost/thread.hpp>
#include <stdio.h>
/* Introduction text for doxygen: */
/*! \mainpage Developer documentation
@ -44,7 +49,7 @@ void WaitForShutdown(boost::thread_group* threadGroup) @@ -44,7 +49,7 @@ void WaitForShutdown(boost::thread_group* threadGroup)
}
if (threadGroup)
{
threadGroup->interrupt_all();
Interrupt(*threadGroup);
threadGroup->join_all();
}
}
@ -154,7 +159,7 @@ bool AppInit(int argc, char* argv[]) @@ -154,7 +159,7 @@ bool AppInit(int argc, char* argv[])
if (!fRet)
{
threadGroup.interrupt_all();
Interrupt(threadGroup);
// threadGroup.join_all(); was left out intentionally here, because we didn't re-test all of
// the startup-failure cases to make sure they don't result in a hang due to some
// thread-blocking-waiting-for-another-thread-during-startup case

201
src/httprpc.cpp

@ -0,0 +1,201 @@ @@ -0,0 +1,201 @@
#include "httprpc.h"
#include "base58.h"
#include "chainparams.h"
#include "httpserver.h"
#include "rpcprotocol.h"
#include "rpcserver.h"
#include "random.h"
#include "sync.h"
#include "util.h"
#include "utilstrencodings.h"
#include "ui_interface.h"
#include <boost/algorithm/string.hpp> // boost::trim
/** Simple one-shot callback timer to be used by the RPC mechanism to e.g.
* re-lock the wellet.
*/
class HTTPRPCTimer : public RPCTimerBase
{
public:
HTTPRPCTimer(struct event_base* eventBase, boost::function<void(void)>& func, int64_t seconds) : ev(eventBase, false, new Handler(func))
{
struct timeval tv = {seconds, 0};
ev.trigger(&tv);
}
private:
HTTPEvent ev;
class Handler : public HTTPClosure
{
public:
Handler(const boost::function<void(void)>& func) : func(func)
{
}
private:
boost::function<void(void)> func;
void operator()() { func(); }
};
};
class HTTPRPCTimerInterface : public RPCTimerInterface
{
public:
HTTPRPCTimerInterface(struct event_base* base) : base(base)
{
}
const char* Name()
{
return "HTTP";
}
RPCTimerBase* NewTimer(boost::function<void(void)>& func, int64_t seconds)
{
return new HTTPRPCTimer(base, func, seconds);
}
private:
struct event_base* base;
};
/* Pre-base64-encoded authentication token */
static std::string strRPCUserColonPass;
/* Stored RPC timer interface (for unregistration) */
static HTTPRPCTimerInterface* httpRPCTimerInterface = 0;
static void JSONErrorReply(HTTPRequest* req, const UniValue& objError, const UniValue& id)
{
// Send error reply from json-rpc error object
int nStatus = HTTP_INTERNAL_SERVER_ERROR;
int code = find_value(objError, "code").get_int();
if (code == RPC_INVALID_REQUEST)
nStatus = HTTP_BAD_REQUEST;
else if (code == RPC_METHOD_NOT_FOUND)
nStatus = HTTP_NOT_FOUND;
std::string strReply = JSONRPCReply(NullUniValue, objError, id);
req->WriteHeader("Content-Type", "application/json");
req->WriteReply(nStatus, strReply);
}
static bool RPCAuthorized(const std::string& strAuth)
{
if (strRPCUserColonPass.empty()) // Belt-and-suspenders measure if InitRPCAuthentication was not called
return false;
if (strAuth.substr(0, 6) != "Basic ")
return false;
std::string strUserPass64 = strAuth.substr(6);
boost::trim(strUserPass64);
std::string strUserPass = DecodeBase64(strUserPass64);
return TimingResistantEqual(strUserPass, strRPCUserColonPass);
}
static bool HTTPReq_JSONRPC(HTTPRequest* req, const std::string &)
{
// JSONRPC handles only POST
if (req->GetRequestMethod() != HTTPRequest::POST) {
req->WriteReply(HTTP_BAD_METHOD, "JSONRPC server handles only POST requests");
return false;
}
// Check authorization
std::pair<bool, std::string> authHeader = req->GetHeader("authorization");
if (!authHeader.first) {
req->WriteReply(HTTP_UNAUTHORIZED);
return false;
}
if (!RPCAuthorized(authHeader.second)) {
LogPrintf("ThreadRPCServer incorrect password attempt from %s\n", req->GetPeer().ToString());
/* Deter brute-forcing
If this results in a DoS the user really
shouldn't have their RPC port exposed. */
MilliSleep(250);
req->WriteReply(HTTP_UNAUTHORIZED);
return false;
}
JSONRequest jreq;
try {
// Parse request
UniValue valRequest;
if (!valRequest.read(req->ReadBody()))
throw JSONRPCError(RPC_PARSE_ERROR, "Parse error");
std::string strReply;
// singleton request
if (valRequest.isObject()) {
jreq.parse(valRequest);
UniValue result = tableRPC.execute(jreq.strMethod, jreq.params);
// Send reply
strReply = JSONRPCReply(result, NullUniValue, jreq.id);
// array of requests
} else if (valRequest.isArray())
strReply = JSONRPCExecBatch(valRequest.get_array());
else
throw JSONRPCError(RPC_PARSE_ERROR, "Top-level object parse error");
req->WriteHeader("Content-Type", "application/json");
req->WriteReply(HTTP_OK, strReply);
} catch (const UniValue& objError) {
JSONErrorReply(req, objError, jreq.id);
return false;
} catch (const std::exception& e) {
JSONErrorReply(req, JSONRPCError(RPC_PARSE_ERROR, e.what()), jreq.id);
return false;
}
return true;
}
static bool InitRPCAuthentication()
{
if (mapArgs["-rpcpassword"] == "")
{
LogPrintf("No rpcpassword set - using random cookie authentication\n");
if (!GenerateAuthCookie(&strRPCUserColonPass)) {
uiInterface.ThreadSafeMessageBox(
_("Error: A fatal internal error occurred, see debug.log for details"), // Same message as AbortNode
"", CClientUIInterface::MSG_ERROR);
return false;
}
} else {
strRPCUserColonPass = mapArgs["-rpcuser"] + ":" + mapArgs["-rpcpassword"];
}
return true;
}
bool StartHTTPRPC()
{
LogPrint("rpc", "Starting HTTP RPC server\n");
if (!InitRPCAuthentication())
return false;
RegisterHTTPHandler("/", true, HTTPReq_JSONRPC);
assert(EventBase());
httpRPCTimerInterface = new HTTPRPCTimerInterface(EventBase());
RPCRegisterTimerInterface(httpRPCTimerInterface);
return true;
}
void InterruptHTTPRPC()
{
LogPrint("rpc", "Interrupting HTTP RPC server\n");
}
void StopHTTPRPC()
{
LogPrint("rpc", "Stopping HTTP RPC server\n");
UnregisterHTTPHandler("/", true);
if (httpRPCTimerInterface) {
RPCUnregisterTimerInterface(httpRPCTimerInterface);
delete httpRPCTimerInterface;
httpRPCTimerInterface = 0;
}
}

37
src/httprpc.h

@ -0,0 +1,37 @@ @@ -0,0 +1,37 @@
// Copyright (c) 2015 The Bitcoin Core developers
// Distributed under the MIT software license, see the accompanying
// file COPYING or http://www.opensource.org/licenses/mit-license.php.
#ifndef BITCOIN_HTTPRPC_H
#define BITCOIN_HTTPRPC_H
#include <string>
#include <map>
class HTTPRequest;
/** Start HTTP RPC subsystem.
* Precondition; HTTP and RPC has been started.
*/
bool StartHTTPRPC();
/** Interrupt HTTP RPC subsystem.
*/
void InterruptHTTPRPC();
/** Stop HTTP RPC subsystem.
* Precondition; HTTP and RPC has been stopped.
*/
void StopHTTPRPC();
/** Start HTTP REST subsystem.
* Precondition; HTTP and RPC has been started.
*/
bool StartREST();
/** Interrupt RPC REST subsystem.
*/
void InterruptREST();
/** Stop HTTP REST subsystem.
* Precondition; HTTP and RPC has been stopped.
*/
void StopREST();
#endif

586
src/httpserver.cpp

@ -0,0 +1,586 @@ @@ -0,0 +1,586 @@
// Copyright (c) 2015 The Bitcoin Core developers
// Distributed under the MIT software license, see the accompanying
// file COPYING or http://www.opensource.org/licenses/mit-license.php.
#include "httpserver.h"
#include "chainparamsbase.h"
#include "compat.h"
#include "util.h"
#include "netbase.h"
#include "rpcprotocol.h" // For HTTP status codes
#include "sync.h"
#include "ui_interface.h"
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <signal.h>
#include <event2/event.h>
#include <event2/http.h>
#include <event2/thread.h>
#include <event2/buffer.h>
#include <event2/util.h>
#include <event2/keyvalq_struct.h>
#ifdef EVENT__HAVE_NETINET_IN_H
#include <netinet/in.h>
#ifdef _XOPEN_SOURCE_EXTENDED
#include <arpa/inet.h>
#endif
#endif
#include <boost/algorithm/string/case_conv.hpp> // for to_lower()
#include <boost/foreach.hpp>
#include <boost/scoped_ptr.hpp>
/** HTTP request work item */
class HTTPWorkItem : public HTTPClosure
{
public:
HTTPWorkItem(HTTPRequest* req, const std::string &path, const HTTPRequestHandler& func):
req(req), path(path), func(func)
{
}
void operator()()
{
func(req.get(), path);
}
boost::scoped_ptr<HTTPRequest> req;
private:
std::string path;
HTTPRequestHandler func;
};
/** Simple work queue for distributing work over multiple threads.
* Work items are simply callable objects.
*/
template <typename WorkItem>
class WorkQueue
{
private:
/** Mutex protects entire object */
CWaitableCriticalSection cs;
CConditionVariable cond;
/* XXX in C++11 we can use std::unique_ptr here and avoid manual cleanup */
std::deque<WorkItem*> queue;
bool running;
size_t maxDepth;
public:
WorkQueue(size_t maxDepth) : running(true),
maxDepth(maxDepth)
{
}
/* Precondition: worker threads have all stopped */
~WorkQueue()
{
while (!queue.empty()) {
delete queue.front();
queue.pop_front();
}
}
/** Enqueue a work item */
bool Enqueue(WorkItem* item)
{
boost::unique_lock<boost::mutex> lock(cs);
if (queue.size() >= maxDepth) {
return false;
}
queue.push_back(item);
cond.notify_one();
return true;
}
/** Thread function */
void Run()
{
while (running) {
WorkItem* i = 0;
{
boost::unique_lock<boost::mutex> lock(cs);
while (running && queue.empty())
cond.wait(lock);
if (!running)
break;
i = queue.front();
queue.pop_front();
}
(*i)();
delete i;
}
}
/** Interrupt and exit loops */
void Interrupt()
{
boost::unique_lock<boost::mutex> lock(cs);
running = false;
cond.notify_all();
}
/** Return current depth of queue */
size_t Depth()
{
boost::unique_lock<boost::mutex> lock(cs);
return queue.size();
}
};
struct HTTPPathHandler
{
HTTPPathHandler() {}
HTTPPathHandler(std::string prefix, bool exactMatch, HTTPRequestHandler handler):
prefix(prefix), exactMatch(exactMatch), handler(handler)
{
}
std::string prefix;
bool exactMatch;
HTTPRequestHandler handler;
};
/** HTTP module state */
//! libevent event loop
static struct event_base* eventBase = 0;
//! HTTP server
struct evhttp* eventHTTP = 0;
//! List of subnets to allow RPC connections from
static std::vector<CSubNet> rpc_allow_subnets;
//! Work queue for handling longer requests off the event loop thread
static WorkQueue<HTTPClosure>* workQueue = 0;
//! Handlers for (sub)paths
std::vector<HTTPPathHandler> pathHandlers;
/** Check if a network address is allowed to access the HTTP server */
static bool ClientAllowed(const CNetAddr& netaddr)
{
if (!netaddr.IsValid())
return false;
BOOST_FOREACH (const CSubNet& subnet, rpc_allow_subnets)
if (subnet.Match(netaddr))
return true;
return false;
}
/** Initialize ACL list for HTTP server */
static bool InitHTTPAllowList()
{
rpc_allow_subnets.clear();
rpc_allow_subnets.push_back(CSubNet("127.0.0.0/8")); // always allow IPv4 local subnet
rpc_allow_subnets.push_back(CSubNet("::1")); // always allow IPv6 localhost
if (mapMultiArgs.count("-rpcallowip")) {
const std::vector<std::string>& vAllow = mapMultiArgs["-rpcallowip"];
BOOST_FOREACH (std::string strAllow, vAllow) {
CSubNet subnet(strAllow);
if (!subnet.IsValid()) {
uiInterface.ThreadSafeMessageBox(
strprintf("Invalid -rpcallowip subnet specification: %s. Valid are a single IP (e.g. 1.2.3.4), a network/netmask (e.g. 1.2.3.4/255.255.255.0) or a network/CIDR (e.g. 1.2.3.4/24).", strAllow),
"", CClientUIInterface::MSG_ERROR);
return false;
}
rpc_allow_subnets.push_back(subnet);
}
}
std::string strAllowed;
BOOST_FOREACH (const CSubNet& subnet, rpc_allow_subnets)
strAllowed += subnet.ToString() + " ";
LogPrint("http", "Allowing HTTP connections from: %s\n", strAllowed);
return true;
}
/** HTTP request method as string - use for logging only */
static std::string RequestMethodString(HTTPRequest::RequestMethod m)
{
switch (m) {
case HTTPRequest::GET:
return "GET";
break;
case HTTPRequest::POST:
return "POST";
break;
case HTTPRequest::HEAD:
return "HEAD";
break;
case HTTPRequest::PUT:
return "PUT";
break;
default:
return "unknown";
}
}
/** HTTP request callback */
static void http_request_cb(struct evhttp_request* req, void* arg)
{
std::auto_ptr<HTTPRequest> hreq(new HTTPRequest(req));
LogPrint("http", "Received a %s request for %s from %s\n",
RequestMethodString(hreq->GetRequestMethod()), hreq->GetURI(), hreq->GetPeer().ToString());
// Early address-based allow check
if (!ClientAllowed(hreq->GetPeer())) {
hreq->WriteReply(HTTP_FORBIDDEN);
return;
}
// Early reject unknown HTTP methods
if (hreq->GetRequestMethod() == HTTPRequest::UNKNOWN) {
hreq->WriteReply(HTTP_BADMETHOD);
return;
}
// Find registered handler for prefix
std::string strURI = hreq->GetURI();
std::string path;
std::vector<HTTPPathHandler>::const_iterator i = pathHandlers.begin();
std::vector<HTTPPathHandler>::const_iterator iend = pathHandlers.end();
for (; i != iend; ++i) {
bool match = false;
if (i->exactMatch)
match = (strURI == i->prefix);
else
match = (strURI.substr(0, i->prefix.size()) == i->prefix);
if (match) {
path = strURI.substr(i->prefix.size());
break;
}
}
// Dispatch to worker thread
if (i != iend) {
std::auto_ptr<HTTPWorkItem> item(new HTTPWorkItem(hreq.release(), path, i->handler));
assert(workQueue);
if (workQueue->Enqueue(item.get()))
item.release(); /* if true, queue took ownership */
else
item->req->WriteReply(HTTP_INTERNAL, "Work queue depth exceeded");
} else {
hreq->WriteReply(HTTP_NOTFOUND);
}
}
/** Event dispatcher thread */
static void ThreadHTTP(struct event_base* base, struct evhttp* http)
{
RenameThread("bitcoin-http");
LogPrint("http", "Entering http event loop\n");
event_base_dispatch(base);
// Event loop will be interrupted by InterruptHTTPServer()
LogPrint("http", "Exited http event loop\n");
}
/** Bind HTTP server to specified addresses */
static bool HTTPBindAddresses(struct evhttp* http)
{
int defaultPort = GetArg("-rpcport", BaseParams().RPCPort());
int nBound = 0;
std::vector<std::pair<std::string, uint16_t> > endpoints;
// Determine what addresses to bind to
if (!mapArgs.count("-rpcallowip")) { // Default to loopback if not allowing external IPs
endpoints.push_back(std::make_pair("::1", defaultPort));
endpoints.push_back(std::make_pair("127.0.0.1", defaultPort));
if (mapArgs.count("-rpcbind")) {
LogPrintf("WARNING: option -rpcbind was ignored because -rpcallowip was not specified, refusing to allow everyone to connect\n");
}
} else if (mapArgs.count("-rpcbind")) { // Specific bind address
const std::vector<std::string>& vbind = mapMultiArgs["-rpcbind"];
for (std::vector<std::string>::const_iterator i = vbind.begin(); i != vbind.end(); ++i) {
int port = defaultPort;
std::string host;
SplitHostPort(*i, port, host);
endpoints.push_back(std::make_pair(host, port));
}
} else { // No specific bind address specified, bind to any
endpoints.push_back(std::make_pair("::", defaultPort));
endpoints.push_back(std::make_pair("0.0.0.0", defaultPort));
}
// Bind addresses
for (std::vector<std::pair<std::string, uint16_t> >::iterator i = endpoints.begin(); i != endpoints.end(); ++i) {
LogPrint("http", "Binding RPC on address %s port %i\n", i->first, i->second);
if (evhttp_bind_socket(http, i->first.empty() ? NULL : i->first.c_str(), i->second) == 0) {
nBound += 1;
} else {
LogPrintf("Binding RPC on address %s port %i failed.\n", i->first, i->second);
}
}
return nBound > 0;
}
/** Simple wrapper to set thread name and run work queue */
static void HTTPWorkQueueRun(WorkQueue<HTTPClosure>* queue)
{
RenameThread("bitcoin-httpworker");
queue->Run();
}
bool StartHTTPServer(boost::thread_group& threadGroup)
{
struct evhttp* http = 0;
struct event_base* base = 0;
if (!InitHTTPAllowList())
return false;
if (GetBoolArg("-rpcssl", false)) {
uiInterface.ThreadSafeMessageBox(
"SSL mode for RPC (-rpcssl) is no longer supported.",
"", CClientUIInterface::MSG_ERROR);
return false;
}
#ifdef WIN32
evthread_use_windows_threads();
#else
evthread_use_pthreads();
#endif
base = event_base_new(); // XXX RAII
if (!base) {
LogPrintf("Couldn't create an event_base: exiting\n");
return false;
}
/* Create a new evhttp object to handle requests. */
http = evhttp_new(base); // XXX RAII
if (!http) {
LogPrintf("couldn't create evhttp. Exiting.\n");
event_base_free(base);
return false;
}
evhttp_set_timeout(http, GetArg("-rpctimeout", 30));
evhttp_set_max_body_size(http, MAX_SIZE);
evhttp_set_gencb(http, http_request_cb, NULL);
if (!HTTPBindAddresses(http)) {
LogPrintf("Unable to bind any endpoint for RPC server\n");
evhttp_free(http);
event_base_free(base);
return false;
}
LogPrint("http", "Starting HTTP server\n");
int workQueueDepth = std::max((long)GetArg("-rpcworkqueue", 16), 1L);
int rpcThreads = std::max((long)GetArg("-rpcthreads", 4), 1L);
LogPrintf("HTTP: creating work queue of depth %d and %d worker threads\n", workQueueDepth, rpcThreads);
workQueue = new WorkQueue<HTTPClosure>(workQueueDepth);
threadGroup.create_thread(boost::bind(&ThreadHTTP, base, http));
for (int i = 0; i < rpcThreads; i++)
threadGroup.create_thread(boost::bind(&HTTPWorkQueueRun, workQueue));
eventBase = base;
eventHTTP = http;
return true;
}
void InterruptHTTPServer()
{
LogPrint("http", "Interrupting HTTP server\n");
if (eventBase)
event_base_loopbreak(eventBase);
if (workQueue)
workQueue->Interrupt();
}
void StopHTTPServer()
{
LogPrint("http", "Stopping HTTP server\n");
delete workQueue;
if (eventHTTP) {
evhttp_free(eventHTTP);
eventHTTP = 0;
}
if (eventBase) {
event_base_free(eventBase);
eventBase = 0;
}
}
struct event_base* EventBase()
{
return eventBase;
}
static void httpevent_callback_fn(evutil_socket_t, short, void* data)
{
// Static handler simply passes through execution flow to _handle method
((HTTPEvent*)data)->_handle();
}
void HTTPEvent::_handle()
{
(*handler)();
if (deleteWhenTriggered)
delete this;
}
HTTPEvent::HTTPEvent(struct event_base* base, bool deleteWhenTriggered, HTTPClosure* handler) : deleteWhenTriggered(deleteWhenTriggered), handler(handler)
{
ev = event_new(base, -1, 0, httpevent_callback_fn, this);
assert(ev);
}
HTTPEvent::~HTTPEvent()
{
event_free(ev);
}
void HTTPEvent::trigger(struct timeval* tv)
{
if (tv == NULL)
event_active(ev, 0, 0); // immediately trigger event in main thread
else
evtimer_add(ev, tv); // trigger after timeval passed
}
HTTPRequest::HTTPRequest(struct evhttp_request* req) : req(req),
replySent(false)
{
}
HTTPRequest::~HTTPRequest()
{
if (!replySent) {
// Keep track of whether reply was sent to avoid request leaks
LogPrintf("%s: Unhandled request\n", __func__);
WriteReply(HTTP_INTERNAL, "Unhandled request");
}
// evhttpd cleans up the request, as long as a reply was sent.
}
std::pair<bool, std::string> HTTPRequest::GetHeader(const std::string& hdr)
{
const struct evkeyvalq* headers = evhttp_request_get_input_headers(req);
assert(headers);
const char* val = evhttp_find_header(headers, hdr.c_str());
if (val)
return std::make_pair(true, val);
else
return std::make_pair(false, "");
}
std::string HTTPRequest::ReadBody()
{
struct evbuffer* buf = evhttp_request_get_input_buffer(req);
if (!buf)
return "";
size_t size = evbuffer_get_length(buf);
/** Trivial implementation: if this is ever a performance bottleneck,
* internal copying can be avoided in multi-segment buffers by using
* evbuffer_peek and an awkward loop. Though in that case, it'd be even
* better to not copy into an intermediate string but use a stream
* abstraction to consume the evbuffer on the fly in the parsing algorithm.
*/
const char* data = (const char*)evbuffer_pullup(buf, size);
if (!data) // returns NULL in case of empty buffer
return "";
std::string rv(data, size);
evbuffer_drain(buf, size);
return rv;
}
void HTTPRequest::WriteHeader(const std::string& hdr, const std::string& value)
{
struct evkeyvalq* headers = evhttp_request_get_output_headers(req);
assert(headers);
evhttp_add_header(headers, hdr.c_str(), value.c_str());
}
/** Closure sent to main thread to request a reply to be sent to
* a HTTP request.
* Replies must be sent in the main loop in the main http thread,
* this cannot be done from worker threads.
*/
struct HTTPSendReplyHandler : HTTPClosure {
public:
HTTPSendReplyHandler(struct evhttp_request* req, int nStatus) : req(req), nStatus(nStatus)
{
}
void operator()()
{
evhttp_send_reply(req, nStatus, NULL, NULL);
}
private:
struct evhttp_request* req;
int nStatus;
};
void HTTPRequest::WriteReply(int nStatus, const std::string& strReply)
{
assert(!replySent && req);
// Send event to main http thread to send reply message
struct evbuffer* evb = evhttp_request_get_output_buffer(req);
assert(evb);
evbuffer_add(evb, strReply.data(), strReply.size());
HTTPEvent* ev = new HTTPEvent(eventBase, true,
new HTTPSendReplyHandler(req, nStatus));
ev->trigger(0);
replySent = true;
req = 0; // transferred back to main thread
}
CService HTTPRequest::GetPeer()
{
evhttp_connection* con = evhttp_request_get_connection(req);
CService peer;
if (con) {
// evhttp retains ownership over returned address string
const char* address = "";
uint16_t port = 0;
evhttp_connection_get_peer(con, (char**)&address, &port);
peer = CService(address, port);
}
return peer;
}
std::string HTTPRequest::GetURI()
{
return evhttp_request_get_uri(req);
}
HTTPRequest::RequestMethod HTTPRequest::GetRequestMethod()
{
switch (evhttp_request_get_command(req)) {
case EVHTTP_REQ_GET:
return GET;
break;
case EVHTTP_REQ_POST:
return POST;
break;
case EVHTTP_REQ_HEAD:
return HEAD;
break;
case EVHTTP_REQ_PUT:
return PUT;
break;
default:
return UNKNOWN;
break;
}
}
void RegisterHTTPHandler(const std::string &prefix, bool exactMatch, const HTTPRequestHandler &handler)
{
LogPrint("http", "Registering HTTP handler for %s (exactmath %d)\n", prefix, exactMatch);
pathHandlers.push_back(HTTPPathHandler(prefix, exactMatch, handler));
}
void UnregisterHTTPHandler(const std::string &prefix, bool exactMatch)
{
std::vector<HTTPPathHandler>::iterator i = pathHandlers.begin();
std::vector<HTTPPathHandler>::iterator iend = pathHandlers.end();
for (; i != iend; ++i)
if (i->prefix == prefix && i->exactMatch == exactMatch)
break;
if (i != iend)
{
LogPrint("http", "Unregistering HTTP handler for %s (exactmath %d)\n", prefix, exactMatch);
pathHandlers.erase(i);
}
}

138
src/httpserver.h

@ -0,0 +1,138 @@ @@ -0,0 +1,138 @@
// Copyright (c) 2015 The Bitcoin Core developers
// Distributed under the MIT software license, see the accompanying
// file COPYING or http://www.opensource.org/licenses/mit-license.php.
#ifndef BITCOIN_HTTPSERVER_H
#define BITCOIN_HTTPSERVER_H
#include <string>
#include <stdint.h>
#include <boost/thread.hpp>
#include <boost/scoped_ptr.hpp>
#include <boost/function.hpp>
struct evhttp_request;
struct event_base;
class CService;
class HTTPRequest;
/** Start HTTP server */
bool StartHTTPServer(boost::thread_group& threadGroup);
/** Interrupt HTTP server threads */
void InterruptHTTPServer();
/** Stop HTTP server */
void StopHTTPServer();
/** Handler for requests to a certain HTTP path */
typedef boost::function<void(HTTPRequest* req, const std::string &)> HTTPRequestHandler;
/** Register handler for prefix.
* If multiple handlers match a prefix, the first-registered one will
* be invoked.
*/
void RegisterHTTPHandler(const std::string &prefix, bool exactMatch, const HTTPRequestHandler &handler);
/** Unregister handler for prefix */
void UnregisterHTTPHandler(const std::string &prefix, bool exactMatch);
/** Return evhttp event base. This can be used by submodules to
* queue timers or custom events.
*/
struct event_base* EventBase();
/** In-flight HTTP request.
* Thin C++ wrapper around evhttp_request.
*/
class HTTPRequest
{
private:
struct evhttp_request* req;
bool replySent;
public:
HTTPRequest(struct evhttp_request* req);
~HTTPRequest();
enum RequestMethod {
UNKNOWN,
GET,
POST,
HEAD,
PUT
};
/** Get requested URI.
*/
std::string GetURI();
/** Get CService (address:ip) for the origin of the http request.
*/
CService GetPeer();
/** Get request method.
*/
RequestMethod GetRequestMethod();
/**
* Get the request header specified by hdr, or an empty string.
* Return an pair (isPresent,string).
*/
std::pair<bool, std::string> GetHeader(const std::string& hdr);
/**
* Read request body.
*
* @note As this consumes the underlying buffer, call this only once.
* Repeated calls will return an empty string.
*/
std::string ReadBody();
/**
* Write output header.
*
* @note call this before calling WriteErrorReply or Reply.
*/
void WriteHeader(const std::string& hdr, const std::string& value);
/**
* Write HTTP reply.
* nStatus is the HTTP status code to send.
* strReply is the body of the reply. Keep it empty to send a standard message.
*
* @note Can be called only once. As this will give the request back to the
* main thread, do not call any other HTTPRequest methods after calling this.
*/
void WriteReply(int nStatus, const std::string& strReply = "");
};
/** Event handler closure.
*/
class HTTPClosure
{
public:
virtual void operator()() = 0;
virtual ~HTTPClosure() {}
};
/** Event class. This can be used either as an cross-thread trigger or as a timer.
*/
class HTTPEvent
{
public:
/** Create a new event */
HTTPEvent(struct event_base* base, bool deleteWhenTriggered, HTTPClosure* handler);
~HTTPEvent();
/** Trigger the event. If tv is 0, trigger it immediately. Otherwise trigger it after
* the given time has elapsed.
*/
void trigger(struct timeval* tv);
/** Internal function for handling, do not call directly */
void _handle();
private:
bool deleteWhenTriggered;
struct event* ev;
boost::scoped_ptr<HTTPClosure> handler;
};
#endif // BITCOIN_HTTPSERVER_H

44
src/init.cpp

@ -16,6 +16,8 @@ @@ -16,6 +16,8 @@
#include "checkpoints.h"
#include "compat/sanity.h"
#include "consensus/validation.h"
#include "httpserver.h"
#include "httprpc.h"
#include "key.h"
#include "main.h"
#include "miner.h"
@ -144,6 +146,15 @@ public: @@ -144,6 +146,15 @@ public:
static CCoinsViewDB *pcoinsdbview = NULL;
static CCoinsViewErrorCatcher *pcoinscatcher = NULL;
void Interrupt(boost::thread_group& threadGroup)
{
InterruptHTTPServer();
InterruptHTTPRPC();
InterruptRPC();
InterruptREST();
threadGroup.interrupt_all();
}
void Shutdown()
{
LogPrintf("%s: In progress...\n", __func__);
@ -158,7 +169,11 @@ void Shutdown() @@ -158,7 +169,11 @@ void Shutdown()
/// module was initialized.
RenameThread("bitcoin-shutoff");
mempool.AddTransactionsUpdated(1);
StopRPCThreads();
StopHTTPRPC();
StopREST();
StopRPC();
StopHTTPServer();
#ifdef ENABLE_WALLET
if (pwalletMain)
pwalletMain->Flush(false);
@ -424,13 +439,6 @@ std::string HelpMessage(HelpMessageMode mode) @@ -424,13 +439,6 @@ std::string HelpMessage(HelpMessageMode mode)
strUsage += HelpMessageOpt("-rpcport=<port>", strprintf(_("Listen for JSON-RPC connections on <port> (default: %u or testnet: %u)"), 8332, 18332));
strUsage += HelpMessageOpt("-rpcallowip=<ip>", _("Allow JSON-RPC connections from specified source. Valid for <ip> are a single IP (e.g. 1.2.3.4), a network/netmask (e.g. 1.2.3.4/255.255.255.0) or a network/CIDR (e.g. 1.2.3.4/24). This option can be specified multiple times"));
strUsage += HelpMessageOpt("-rpcthreads=<n>", strprintf(_("Set the number of threads to service RPC calls (default: %d)"), 4));
strUsage += HelpMessageOpt("-rpckeepalive", strprintf(_("RPC support for HTTP persistent connections (default: %d)"), 1));
strUsage += HelpMessageGroup(_("RPC SSL options: (see the Bitcoin Wiki for SSL setup instructions)"));
strUsage += HelpMessageOpt("-rpcssl", _("Use OpenSSL (https) for JSON-RPC connections"));
strUsage += HelpMessageOpt("-rpcsslcertificatechainfile=<file.cert>", strprintf(_("Server certificate file (default: %s)"), "server.cert"));
strUsage += HelpMessageOpt("-rpcsslprivatekeyfile=<file.pem>", strprintf(_("Server private key (default: %s)"), "server.pem"));
strUsage += HelpMessageOpt("-rpcsslciphers=<ciphers>", strprintf(_("Acceptable ciphers (default: %s)"), "TLSv1.2+HIGH:TLSv1+HIGH:!SSLv2:!aNULL:!eNULL:!3DES:@STRENGTH"));
if (mode == HMM_BITCOIN_QT)
{
@ -602,6 +610,21 @@ bool InitSanityCheck(void) @@ -602,6 +610,21 @@ bool InitSanityCheck(void)
return true;
}
bool AppInitServers(boost::thread_group& threadGroup)
{
RPCServer::OnStopped(&OnRPCStopped);
RPCServer::OnPreCommand(&OnRPCPreCommand);
if (!StartHTTPServer(threadGroup))
return false;
if (!StartRPC())
return false;
if (!StartHTTPRPC())
return false;
if (GetBoolArg("-rest", false) && !StartREST())
return false;
return true;
}
/** Initialize bitcoin.
* @pre Parameters should be parsed and config file should be read.
*/
@ -990,9 +1013,8 @@ bool AppInit2(boost::thread_group& threadGroup, CScheduler& scheduler) @@ -990,9 +1013,8 @@ bool AppInit2(boost::thread_group& threadGroup, CScheduler& scheduler)
if (fServer)
{
uiInterface.InitMessage.connect(SetRPCWarmupStatus);
RPCServer::OnStopped(&OnRPCStopped);
RPCServer::OnPreCommand(&OnRPCPreCommand);
StartRPCThreads();
if (!AppInitServers(threadGroup))
return InitError(_("Unable to start HTTP server. See debug log for details."));
}
int64_t nStart;

2
src/init.h

@ -20,6 +20,8 @@ extern CWallet* pwalletMain; @@ -20,6 +20,8 @@ extern CWallet* pwalletMain;
void StartShutdown();
bool ShutdownRequested();
/** Interrupt threads */
void Interrupt(boost::thread_group& threadGroup);
void Shutdown();
bool AppInit2(boost::thread_group& threadGroup, CScheduler& scheduler);

9
src/qt/bitcoin.cpp

@ -266,13 +266,6 @@ void BitcoinCore::initialize() @@ -266,13 +266,6 @@ void BitcoinCore::initialize()
{
qDebug() << __func__ << ": Running AppInit2 in thread";
int rv = AppInit2(threadGroup, scheduler);
if(rv)
{
/* Start a dummy RPC thread if no RPC thread is active yet
* to handle timeouts.
*/
StartDummyRPCThread();
}
Q_EMIT initializeResult(rv);
} catch (const std::exception& e) {
handleRunawayException(&e);
@ -286,7 +279,7 @@ void BitcoinCore::shutdown() @@ -286,7 +279,7 @@ void BitcoinCore::shutdown()
try
{
qDebug() << __func__ << ": Running Shutdown in thread";
threadGroup.interrupt_all();
Interrupt(threadGroup);
threadGroup.join_all();
Shutdown();
qDebug() << __func__ << ": Shutdown finished";

243
src/rest.cpp

@ -7,6 +7,7 @@ @@ -7,6 +7,7 @@
#include "primitives/block.h"
#include "primitives/transaction.h"
#include "main.h"
#include "httpserver.h"
#include "rpcserver.h"
#include "streams.h"
#include "sync.h"
@ -56,13 +57,6 @@ struct CCoin { @@ -56,13 +57,6 @@ struct CCoin {
}
};
class RestErr
{
public:
enum HTTPStatusCode status;
string message;
};
extern void TxToJSON(const CTransaction& tx, const uint256 hashBlock, UniValue& entry);
extern UniValue blockToJSON(const CBlock& block, const CBlockIndex* blockindex, bool txDetails = false);
extern UniValue mempoolInfoToJSON();
@ -70,15 +64,14 @@ extern UniValue mempoolToJSON(bool fVerbose = false); @@ -70,15 +64,14 @@ extern UniValue mempoolToJSON(bool fVerbose = false);
extern void ScriptPubKeyToJSON(const CScript& scriptPubKey, UniValue& out, bool fIncludeHex);
extern UniValue blockheaderToJSON(const CBlockIndex* blockindex);
static RestErr RESTERR(enum HTTPStatusCode status, string message)
static bool RESTERR(HTTPRequest* req, enum HTTPStatusCode status, string message)
{
RestErr re;
re.status = status;
re.message = message;
return re;
req->WriteHeader("Content-Type", "text/plain");
req->WriteReply(status, message + "\r\n");
return false;
}
static enum RetFormat ParseDataFormat(vector<string>& params, const string strReq)
static enum RetFormat ParseDataFormat(vector<string>& params, const string& strReq)
{
boost::split(params, strReq, boost::is_any_of("."));
if (params.size() > 1) {
@ -115,28 +108,35 @@ static bool ParseHashStr(const string& strReq, uint256& v) @@ -115,28 +108,35 @@ static bool ParseHashStr(const string& strReq, uint256& v)
return true;
}
static bool rest_headers(AcceptedConnection* conn,
const std::string& strURIPart,
const std::string& strRequest,
const std::map<std::string, std::string>& mapHeaders,
bool fRun)
static bool CheckWarmup(HTTPRequest* req)
{
std::string statusmessage;
if (RPCIsInWarmup(&statusmessage))
return RESTERR(req, HTTP_SERVICE_UNAVAILABLE, "Service temporarily unavailable: " + statusmessage);
return true;
}
static bool rest_headers(HTTPRequest* req,
const std::string& strURIPart)
{
if (!CheckWarmup(req))
return false;
vector<string> params;
const RetFormat rf = ParseDataFormat(params, strURIPart);
vector<string> path;
boost::split(path, params[0], boost::is_any_of("/"));
if (path.size() != 2)
throw RESTERR(HTTP_BAD_REQUEST, "No header count specified. Use /rest/headers/<count>/<hash>.<ext>.");
return RESTERR(req, HTTP_BAD_REQUEST, "No header count specified. Use /rest/headers/<count>/<hash>.<ext>.");
long count = strtol(path[0].c_str(), NULL, 10);
if (count < 1 || count > 2000)
throw RESTERR(HTTP_BAD_REQUEST, "Header count out of range: " + path[0]);
return RESTERR(req, HTTP_BAD_REQUEST, "Header count out of range: " + path[0]);
string hashStr = path[1];
uint256 hash;
if (!ParseHashStr(hashStr, hash))
throw RESTERR(HTTP_BAD_REQUEST, "Invalid hash: " + hashStr);
return RESTERR(req, HTTP_BAD_REQUEST, "Invalid hash: " + hashStr);
std::vector<const CBlockIndex *> headers;
headers.reserve(count);
@ -160,28 +160,29 @@ static bool rest_headers(AcceptedConnection* conn, @@ -160,28 +160,29 @@ static bool rest_headers(AcceptedConnection* conn,
switch (rf) {
case RF_BINARY: {
string binaryHeader = ssHeader.str();
conn->stream() << HTTPReplyHeader(HTTP_OK, fRun