You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

db.cpp 17KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587
  1. // Copyright (c) 2009-2010 Satoshi Nakamoto
  2. // Copyright (c) 2009-2012 The Bitcoin developers
  3. // Distributed under the MIT/X11 software license, see the accompanying
  4. // file COPYING or http://www.opensource.org/licenses/mit-license.php.
  5. #include "chainparams.h"
  6. #include "db.h"
  7. #include "util.h"
  8. #include "hash.h"
  9. #include "addrman.h"
  10. #include <boost/version.hpp>
  11. #include <boost/filesystem.hpp>
  12. #include <boost/filesystem/fstream.hpp>
  13. #include <openssl/rand.h>
  14. #ifndef WIN32
  15. #include "sys/stat.h"
  16. #endif
  17. using namespace std;
  18. using namespace boost;
  19. unsigned int nWalletDBUpdated;
  20. //
  21. // CDB
  22. //
  23. CDBEnv bitdb;
  24. void CDBEnv::EnvShutdown()
  25. {
  26. if (!fDbEnvInit)
  27. return;
  28. fDbEnvInit = false;
  29. int ret = dbenv.close(0);
  30. if (ret != 0)
  31. printf("EnvShutdown exception: %s (%d)\n", DbEnv::strerror(ret), ret);
  32. if (!fMockDb)
  33. DbEnv(0).remove(path.string().c_str(), 0);
  34. }
  35. CDBEnv::CDBEnv() : dbenv(DB_CXX_NO_EXCEPTIONS)
  36. {
  37. fDbEnvInit = false;
  38. fMockDb = false;
  39. }
  40. CDBEnv::~CDBEnv()
  41. {
  42. EnvShutdown();
  43. }
  44. void CDBEnv::Close()
  45. {
  46. EnvShutdown();
  47. }
  48. bool CDBEnv::Open(const boost::filesystem::path& pathIn)
  49. {
  50. if (fDbEnvInit)
  51. return true;
  52. boost::this_thread::interruption_point();
  53. path = pathIn;
  54. filesystem::path pathLogDir = path / "database";
  55. filesystem::create_directory(pathLogDir);
  56. filesystem::path pathErrorFile = path / "db.log";
  57. printf("dbenv.open LogDir=%s ErrorFile=%s\n", pathLogDir.string().c_str(), pathErrorFile.string().c_str());
  58. unsigned int nEnvFlags = 0;
  59. if (GetBoolArg("-privdb", true))
  60. nEnvFlags |= DB_PRIVATE;
  61. dbenv.set_lg_dir(pathLogDir.string().c_str());
  62. dbenv.set_cachesize(0, 0x100000, 1); // 1 MiB should be enough for just the wallet
  63. dbenv.set_lg_bsize(0x10000);
  64. dbenv.set_lg_max(1048576);
  65. dbenv.set_lk_max_locks(40000);
  66. dbenv.set_lk_max_objects(40000);
  67. dbenv.set_errfile(fopen(pathErrorFile.string().c_str(), "a")); /// debug
  68. dbenv.set_flags(DB_AUTO_COMMIT, 1);
  69. dbenv.set_flags(DB_TXN_WRITE_NOSYNC, 1);
  70. dbenv.log_set_config(DB_LOG_AUTO_REMOVE, 1);
  71. int ret = dbenv.open(path.string().c_str(),
  72. DB_CREATE |
  73. DB_INIT_LOCK |
  74. DB_INIT_LOG |
  75. DB_INIT_MPOOL |
  76. DB_INIT_TXN |
  77. DB_THREAD |
  78. DB_RECOVER |
  79. nEnvFlags,
  80. S_IRUSR | S_IWUSR);
  81. if (ret != 0)
  82. return error("CDB() : error %s (%d) opening database environment", DbEnv::strerror(ret), ret);
  83. fDbEnvInit = true;
  84. fMockDb = false;
  85. return true;
  86. }
  87. void CDBEnv::MakeMock()
  88. {
  89. if (fDbEnvInit)
  90. throw runtime_error("CDBEnv::MakeMock(): already initialized");
  91. boost::this_thread::interruption_point();
  92. printf("CDBEnv::MakeMock()\n");
  93. dbenv.set_cachesize(1, 0, 1);
  94. dbenv.set_lg_bsize(10485760*4);
  95. dbenv.set_lg_max(10485760);
  96. dbenv.set_lk_max_locks(10000);
  97. dbenv.set_lk_max_objects(10000);
  98. dbenv.set_flags(DB_AUTO_COMMIT, 1);
  99. dbenv.log_set_config(DB_LOG_IN_MEMORY, 1);
  100. int ret = dbenv.open(NULL,
  101. DB_CREATE |
  102. DB_INIT_LOCK |
  103. DB_INIT_LOG |
  104. DB_INIT_MPOOL |
  105. DB_INIT_TXN |
  106. DB_THREAD |
  107. DB_PRIVATE,
  108. S_IRUSR | S_IWUSR);
  109. if (ret > 0)
  110. throw runtime_error(strprintf("CDBEnv::MakeMock(): error %d opening database environment", ret));
  111. fDbEnvInit = true;
  112. fMockDb = true;
  113. }
  114. CDBEnv::VerifyResult CDBEnv::Verify(std::string strFile, bool (*recoverFunc)(CDBEnv& dbenv, std::string strFile))
  115. {
  116. LOCK(cs_db);
  117. assert(mapFileUseCount.count(strFile) == 0);
  118. Db db(&dbenv, 0);
  119. int result = db.verify(strFile.c_str(), NULL, NULL, 0);
  120. if (result == 0)
  121. return VERIFY_OK;
  122. else if (recoverFunc == NULL)
  123. return RECOVER_FAIL;
  124. // Try to recover:
  125. bool fRecovered = (*recoverFunc)(*this, strFile);
  126. return (fRecovered ? RECOVER_OK : RECOVER_FAIL);
  127. }
  128. bool CDBEnv::Salvage(std::string strFile, bool fAggressive,
  129. std::vector<CDBEnv::KeyValPair >& vResult)
  130. {
  131. LOCK(cs_db);
  132. assert(mapFileUseCount.count(strFile) == 0);
  133. u_int32_t flags = DB_SALVAGE;
  134. if (fAggressive) flags |= DB_AGGRESSIVE;
  135. stringstream strDump;
  136. Db db(&dbenv, 0);
  137. int result = db.verify(strFile.c_str(), NULL, &strDump, flags);
  138. if (result == DB_VERIFY_BAD)
  139. {
  140. printf("Error: Salvage found errors, all data may not be recoverable.\n");
  141. if (!fAggressive)
  142. {
  143. printf("Error: Rerun with aggressive mode to ignore errors and continue.\n");
  144. return false;
  145. }
  146. }
  147. if (result != 0 && result != DB_VERIFY_BAD)
  148. {
  149. printf("ERROR: db salvage failed: %d\n",result);
  150. return false;
  151. }
  152. // Format of bdb dump is ascii lines:
  153. // header lines...
  154. // HEADER=END
  155. // hexadecimal key
  156. // hexadecimal value
  157. // ... repeated
  158. // DATA=END
  159. string strLine;
  160. while (!strDump.eof() && strLine != "HEADER=END")
  161. getline(strDump, strLine); // Skip past header
  162. std::string keyHex, valueHex;
  163. while (!strDump.eof() && keyHex != "DATA=END")
  164. {
  165. getline(strDump, keyHex);
  166. if (keyHex != "DATA_END")
  167. {
  168. getline(strDump, valueHex);
  169. vResult.push_back(make_pair(ParseHex(keyHex),ParseHex(valueHex)));
  170. }
  171. }
  172. return (result == 0);
  173. }
  174. void CDBEnv::CheckpointLSN(std::string strFile)
  175. {
  176. dbenv.txn_checkpoint(0, 0, 0);
  177. if (fMockDb)
  178. return;
  179. dbenv.lsn_reset(strFile.c_str(), 0);
  180. }
  181. CDB::CDB(const char *pszFile, const char* pszMode) :
  182. pdb(NULL), activeTxn(NULL)
  183. {
  184. int ret;
  185. if (pszFile == NULL)
  186. return;
  187. fReadOnly = (!strchr(pszMode, '+') && !strchr(pszMode, 'w'));
  188. bool fCreate = strchr(pszMode, 'c');
  189. unsigned int nFlags = DB_THREAD;
  190. if (fCreate)
  191. nFlags |= DB_CREATE;
  192. {
  193. LOCK(bitdb.cs_db);
  194. if (!bitdb.Open(GetDataDir()))
  195. throw runtime_error("env open failed");
  196. strFile = pszFile;
  197. ++bitdb.mapFileUseCount[strFile];
  198. pdb = bitdb.mapDb[strFile];
  199. if (pdb == NULL)
  200. {
  201. pdb = new Db(&bitdb.dbenv, 0);
  202. bool fMockDb = bitdb.IsMock();
  203. if (fMockDb)
  204. {
  205. DbMpoolFile*mpf = pdb->get_mpf();
  206. ret = mpf->set_flags(DB_MPOOL_NOFILE, 1);
  207. if (ret != 0)
  208. throw runtime_error(strprintf("CDB() : failed to configure for no temp file backing for database %s", pszFile));
  209. }
  210. ret = pdb->open(NULL, // Txn pointer
  211. fMockDb ? NULL : pszFile, // Filename
  212. fMockDb ? pszFile : "main", // Logical db name
  213. DB_BTREE, // Database type
  214. nFlags, // Flags
  215. 0);
  216. if (ret != 0)
  217. {
  218. delete pdb;
  219. pdb = NULL;
  220. --bitdb.mapFileUseCount[strFile];
  221. strFile = "";
  222. throw runtime_error(strprintf("CDB() : can't open database file %s, error %d", pszFile, ret));
  223. }
  224. if (fCreate && !Exists(string("version")))
  225. {
  226. bool fTmp = fReadOnly;
  227. fReadOnly = false;
  228. WriteVersion(CLIENT_VERSION);
  229. fReadOnly = fTmp;
  230. }
  231. bitdb.mapDb[strFile] = pdb;
  232. }
  233. }
  234. }
  235. void CDB::Flush()
  236. {
  237. if (activeTxn)
  238. return;
  239. // Flush database activity from memory pool to disk log
  240. unsigned int nMinutes = 0;
  241. if (fReadOnly)
  242. nMinutes = 1;
  243. bitdb.dbenv.txn_checkpoint(nMinutes ? GetArg("-dblogsize", 100)*1024 : 0, nMinutes, 0);
  244. }
  245. void CDB::Close()
  246. {
  247. if (!pdb)
  248. return;
  249. if (activeTxn)
  250. activeTxn->abort();
  251. activeTxn = NULL;
  252. pdb = NULL;
  253. Flush();
  254. {
  255. LOCK(bitdb.cs_db);
  256. --bitdb.mapFileUseCount[strFile];
  257. }
  258. }
  259. void CDBEnv::CloseDb(const string& strFile)
  260. {
  261. {
  262. LOCK(cs_db);
  263. if (mapDb[strFile] != NULL)
  264. {
  265. // Close the database handle
  266. Db* pdb = mapDb[strFile];
  267. pdb->close(0);
  268. delete pdb;
  269. mapDb[strFile] = NULL;
  270. }
  271. }
  272. }
  273. bool CDBEnv::RemoveDb(const string& strFile)
  274. {
  275. this->CloseDb(strFile);
  276. LOCK(cs_db);
  277. int rc = dbenv.dbremove(NULL, strFile.c_str(), NULL, DB_AUTO_COMMIT);
  278. return (rc == 0);
  279. }
  280. bool CDB::Rewrite(const string& strFile, const char* pszSkip)
  281. {
  282. while (true)
  283. {
  284. {
  285. LOCK(bitdb.cs_db);
  286. if (!bitdb.mapFileUseCount.count(strFile) || bitdb.mapFileUseCount[strFile] == 0)
  287. {
  288. // Flush log data to the dat file
  289. bitdb.CloseDb(strFile);
  290. bitdb.CheckpointLSN(strFile);
  291. bitdb.mapFileUseCount.erase(strFile);
  292. bool fSuccess = true;
  293. printf("Rewriting %s...\n", strFile.c_str());
  294. string strFileRes = strFile + ".rewrite";
  295. { // surround usage of db with extra {}
  296. CDB db(strFile.c_str(), "r");
  297. Db* pdbCopy = new Db(&bitdb.dbenv, 0);
  298. int ret = pdbCopy->open(NULL, // Txn pointer
  299. strFileRes.c_str(), // Filename
  300. "main", // Logical db name
  301. DB_BTREE, // Database type
  302. DB_CREATE, // Flags
  303. 0);
  304. if (ret > 0)
  305. {
  306. printf("Cannot create database file %s\n", strFileRes.c_str());
  307. fSuccess = false;
  308. }
  309. Dbc* pcursor = db.GetCursor();
  310. if (pcursor)
  311. while (fSuccess)
  312. {
  313. CDataStream ssKey(SER_DISK, CLIENT_VERSION);
  314. CDataStream ssValue(SER_DISK, CLIENT_VERSION);
  315. int ret = db.ReadAtCursor(pcursor, ssKey, ssValue, DB_NEXT);
  316. if (ret == DB_NOTFOUND)
  317. {
  318. pcursor->close();
  319. break;
  320. }
  321. else if (ret != 0)
  322. {
  323. pcursor->close();
  324. fSuccess = false;
  325. break;
  326. }
  327. if (pszSkip &&
  328. strncmp(&ssKey[0], pszSkip, std::min(ssKey.size(), strlen(pszSkip))) == 0)
  329. continue;
  330. if (strncmp(&ssKey[0], "\x07version", 8) == 0)
  331. {
  332. // Update version:
  333. ssValue.clear();
  334. ssValue << CLIENT_VERSION;
  335. }
  336. Dbt datKey(&ssKey[0], ssKey.size());
  337. Dbt datValue(&ssValue[0], ssValue.size());
  338. int ret2 = pdbCopy->put(NULL, &datKey, &datValue, DB_NOOVERWRITE);
  339. if (ret2 > 0)
  340. fSuccess = false;
  341. }
  342. if (fSuccess)
  343. {
  344. db.Close();
  345. bitdb.CloseDb(strFile);
  346. if (pdbCopy->close(0))
  347. fSuccess = false;
  348. delete pdbCopy;
  349. }
  350. }
  351. if (fSuccess)
  352. {
  353. Db dbA(&bitdb.dbenv, 0);
  354. if (dbA.remove(strFile.c_str(), NULL, 0))
  355. fSuccess = false;
  356. Db dbB(&bitdb.dbenv, 0);
  357. if (dbB.rename(strFileRes.c_str(), NULL, strFile.c_str(), 0))
  358. fSuccess = false;
  359. }
  360. if (!fSuccess)
  361. printf("Rewriting of %s FAILED!\n", strFileRes.c_str());
  362. return fSuccess;
  363. }
  364. }
  365. MilliSleep(100);
  366. }
  367. return false;
  368. }
  369. void CDBEnv::Flush(bool fShutdown)
  370. {
  371. int64 nStart = GetTimeMillis();
  372. // Flush log data to the actual data file
  373. // on all files that are not in use
  374. printf("Flush(%s)%s\n", fShutdown ? "true" : "false", fDbEnvInit ? "" : " db not started");
  375. if (!fDbEnvInit)
  376. return;
  377. {
  378. LOCK(cs_db);
  379. map<string, int>::iterator mi = mapFileUseCount.begin();
  380. while (mi != mapFileUseCount.end())
  381. {
  382. string strFile = (*mi).first;
  383. int nRefCount = (*mi).second;
  384. printf("%s refcount=%d\n", strFile.c_str(), nRefCount);
  385. if (nRefCount == 0)
  386. {
  387. // Move log data to the dat file
  388. CloseDb(strFile);
  389. printf("%s checkpoint\n", strFile.c_str());
  390. dbenv.txn_checkpoint(0, 0, 0);
  391. printf("%s detach\n", strFile.c_str());
  392. if (!fMockDb)
  393. dbenv.lsn_reset(strFile.c_str(), 0);
  394. printf("%s closed\n", strFile.c_str());
  395. mapFileUseCount.erase(mi++);
  396. }
  397. else
  398. mi++;
  399. }
  400. printf("DBFlush(%s)%s ended %15"PRI64d"ms\n", fShutdown ? "true" : "false", fDbEnvInit ? "" : " db not started", GetTimeMillis() - nStart);
  401. if (fShutdown)
  402. {
  403. char** listp;
  404. if (mapFileUseCount.empty())
  405. {
  406. dbenv.log_archive(&listp, DB_ARCH_REMOVE);
  407. Close();
  408. if (!fMockDb)
  409. boost::filesystem::remove_all(path / "database");
  410. }
  411. }
  412. }
  413. }
  414. //
  415. // CAddrDB
  416. //
  417. CAddrDB::CAddrDB()
  418. {
  419. pathAddr = GetDataDir() / "peers.dat";
  420. }
  421. bool CAddrDB::Write(const CAddrMan& addr)
  422. {
  423. // Generate random temporary filename
  424. unsigned short randv = 0;
  425. RAND_bytes((unsigned char *)&randv, sizeof(randv));
  426. std::string tmpfn = strprintf("peers.dat.%04x", randv);
  427. // serialize addresses, checksum data up to that point, then append csum
  428. CDataStream ssPeers(SER_DISK, CLIENT_VERSION);
  429. ssPeers << FLATDATA(Params().MessageStart());
  430. ssPeers << addr;
  431. uint256 hash = Hash(ssPeers.begin(), ssPeers.end());
  432. ssPeers << hash;
  433. // open temp output file, and associate with CAutoFile
  434. boost::filesystem::path pathTmp = GetDataDir() / tmpfn;
  435. FILE *file = fopen(pathTmp.string().c_str(), "wb");
  436. CAutoFile fileout = CAutoFile(file, SER_DISK, CLIENT_VERSION);
  437. if (!fileout)
  438. return error("CAddrman::Write() : open failed");
  439. // Write and commit header, data
  440. try {
  441. fileout << ssPeers;
  442. }
  443. catch (std::exception &e) {
  444. return error("CAddrman::Write() : I/O error");
  445. }
  446. FileCommit(fileout);
  447. fileout.fclose();
  448. // replace existing peers.dat, if any, with new peers.dat.XXXX
  449. if (!RenameOver(pathTmp, pathAddr))
  450. return error("CAddrman::Write() : Rename-into-place failed");
  451. return true;
  452. }
  453. bool CAddrDB::Read(CAddrMan& addr)
  454. {
  455. // open input file, and associate with CAutoFile
  456. FILE *file = fopen(pathAddr.string().c_str(), "rb");
  457. CAutoFile filein = CAutoFile(file, SER_DISK, CLIENT_VERSION);
  458. if (!filein)
  459. return error("CAddrman::Read() : open failed");
  460. // use file size to size memory buffer
  461. int fileSize = GetFilesize(filein);
  462. int dataSize = fileSize - sizeof(uint256);
  463. //Don't try to resize to a negative number if file is small
  464. if ( dataSize < 0 ) dataSize = 0;
  465. vector<unsigned char> vchData;
  466. vchData.resize(dataSize);
  467. uint256 hashIn;
  468. // read data and checksum from file
  469. try {
  470. filein.read((char *)&vchData[0], dataSize);
  471. filein >> hashIn;
  472. }
  473. catch (std::exception &e) {
  474. return error("CAddrman::Read() 2 : I/O error or stream data corrupted");
  475. }
  476. filein.fclose();
  477. CDataStream ssPeers(vchData, SER_DISK, CLIENT_VERSION);
  478. // verify stored checksum matches input data
  479. uint256 hashTmp = Hash(ssPeers.begin(), ssPeers.end());
  480. if (hashIn != hashTmp)
  481. return error("CAddrman::Read() : checksum mismatch; data corrupted");
  482. unsigned char pchMsgTmp[4];
  483. try {
  484. // de-serialize file header (network specific magic number) and ..
  485. ssPeers >> FLATDATA(pchMsgTmp);
  486. // ... verify the network matches ours
  487. if (memcmp(pchMsgTmp, Params().MessageStart(), sizeof(pchMsgTmp)))
  488. return error("CAddrman::Read() : invalid network magic number");
  489. // de-serialize address data into one CAddrMan object
  490. ssPeers >> addr;
  491. }
  492. catch (std::exception &e) {
  493. return error("CAddrman::Read() : I/O error or stream data corrupted");
  494. }
  495. return true;
  496. }