You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

dbwrapper.h 10KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343
  1. // Copyright (c) 2012-2016 The Starwels developers
  2. // Distributed under the MIT software license, see the accompanying
  3. // file COPYING or http://www.opensource.org/licenses/mit-license.php.
  4. #ifndef STARWELS_DBWRAPPER_H
  5. #define STARWELS_DBWRAPPER_H
  6. #include "clientversion.h"
  7. #include "fs.h"
  8. #include "serialize.h"
  9. #include "streams.h"
  10. #include "util.h"
  11. #include "utilstrencodings.h"
  12. #include "version.h"
  13. #include <leveldb/db.h>
  14. #include <leveldb/write_batch.h>
  15. static const size_t DBWRAPPER_PREALLOC_KEY_SIZE = 64;
  16. static const size_t DBWRAPPER_PREALLOC_VALUE_SIZE = 1024;
  17. class dbwrapper_error : public std::runtime_error
  18. {
  19. public:
  20. dbwrapper_error(const std::string& msg) : std::runtime_error(msg) {}
  21. };
  22. class CDBWrapper;
  23. /** These should be considered an implementation detail of the specific database.
  24. */
  25. namespace dbwrapper_private {
  26. /** Handle database error by throwing dbwrapper_error exception.
  27. */
  28. void HandleError(const leveldb::Status& status);
  29. /** Work around circular dependency, as well as for testing in dbwrapper_tests.
  30. * Database obfuscation should be considered an implementation detail of the
  31. * specific database.
  32. */
  33. const std::vector<unsigned char>& GetObfuscateKey(const CDBWrapper &w);
  34. };
  35. /** Batch of changes queued to be written to a CDBWrapper */
  36. class CDBBatch
  37. {
  38. friend class CDBWrapper;
  39. private:
  40. const CDBWrapper &parent;
  41. leveldb::WriteBatch batch;
  42. CDataStream ssKey;
  43. CDataStream ssValue;
  44. size_t size_estimate;
  45. public:
  46. /**
  47. * @param[in] _parent CDBWrapper that this batch is to be submitted to
  48. */
  49. CDBBatch(const CDBWrapper &_parent) : parent(_parent), ssKey(SER_DISK, CLIENT_VERSION), ssValue(SER_DISK, CLIENT_VERSION), size_estimate(0) { };
  50. void Clear()
  51. {
  52. batch.Clear();
  53. size_estimate = 0;
  54. }
  55. template <typename K, typename V>
  56. void Write(const K& key, const V& value)
  57. {
  58. ssKey.reserve(DBWRAPPER_PREALLOC_KEY_SIZE);
  59. ssKey << key;
  60. leveldb::Slice slKey(ssKey.data(), ssKey.size());
  61. ssValue.reserve(DBWRAPPER_PREALLOC_VALUE_SIZE);
  62. ssValue << value;
  63. ssValue.Xor(dbwrapper_private::GetObfuscateKey(parent));
  64. leveldb::Slice slValue(ssValue.data(), ssValue.size());
  65. batch.Put(slKey, slValue);
  66. // LevelDB serializes writes as:
  67. // - byte: header
  68. // - varint: key length (1 byte up to 127B, 2 bytes up to 16383B, ...)
  69. // - byte[]: key
  70. // - varint: value length
  71. // - byte[]: value
  72. // The formula below assumes the key and value are both less than 16k.
  73. size_estimate += 3 + (slKey.size() > 127) + slKey.size() + (slValue.size() > 127) + slValue.size();
  74. ssKey.clear();
  75. ssValue.clear();
  76. }
  77. template <typename K>
  78. void Erase(const K& key)
  79. {
  80. ssKey.reserve(DBWRAPPER_PREALLOC_KEY_SIZE);
  81. ssKey << key;
  82. leveldb::Slice slKey(ssKey.data(), ssKey.size());
  83. batch.Delete(slKey);
  84. // LevelDB serializes erases as:
  85. // - byte: header
  86. // - varint: key length
  87. // - byte[]: key
  88. // The formula below assumes the key is less than 16kB.
  89. size_estimate += 2 + (slKey.size() > 127) + slKey.size();
  90. ssKey.clear();
  91. }
  92. size_t SizeEstimate() const { return size_estimate; }
  93. };
  94. class CDBIterator
  95. {
  96. private:
  97. const CDBWrapper &parent;
  98. leveldb::Iterator *piter;
  99. public:
  100. /**
  101. * @param[in] _parent Parent CDBWrapper instance.
  102. * @param[in] _piter The original leveldb iterator.
  103. */
  104. CDBIterator(const CDBWrapper &_parent, leveldb::Iterator *_piter) :
  105. parent(_parent), piter(_piter) { };
  106. ~CDBIterator();
  107. bool Valid();
  108. void SeekToFirst();
  109. template<typename K> void Seek(const K& key) {
  110. CDataStream ssKey(SER_DISK, CLIENT_VERSION);
  111. ssKey.reserve(DBWRAPPER_PREALLOC_KEY_SIZE);
  112. ssKey << key;
  113. leveldb::Slice slKey(ssKey.data(), ssKey.size());
  114. piter->Seek(slKey);
  115. }
  116. void Next();
  117. template<typename K> bool GetKey(K& key) {
  118. leveldb::Slice slKey = piter->key();
  119. try {
  120. CDataStream ssKey(slKey.data(), slKey.data() + slKey.size(), SER_DISK, CLIENT_VERSION);
  121. ssKey >> key;
  122. } catch (const std::exception&) {
  123. return false;
  124. }
  125. return true;
  126. }
  127. template<typename V> bool GetValue(V& value) {
  128. leveldb::Slice slValue = piter->value();
  129. try {
  130. CDataStream ssValue(slValue.data(), slValue.data() + slValue.size(), SER_DISK, CLIENT_VERSION);
  131. ssValue.Xor(dbwrapper_private::GetObfuscateKey(parent));
  132. ssValue >> value;
  133. } catch (const std::exception&) {
  134. return false;
  135. }
  136. return true;
  137. }
  138. unsigned int GetValueSize() {
  139. return piter->value().size();
  140. }
  141. };
  142. class CDBWrapper
  143. {
  144. friend const std::vector<unsigned char>& dbwrapper_private::GetObfuscateKey(const CDBWrapper &w);
  145. private:
  146. //! custom environment this database is using (may be nullptr in case of default environment)
  147. leveldb::Env* penv;
  148. //! database options used
  149. leveldb::Options options;
  150. //! options used when reading from the database
  151. leveldb::ReadOptions readoptions;
  152. //! options used when iterating over values of the database
  153. leveldb::ReadOptions iteroptions;
  154. //! options used when writing to the database
  155. leveldb::WriteOptions writeoptions;
  156. //! options used when sync writing to the database
  157. leveldb::WriteOptions syncoptions;
  158. //! the database itself
  159. leveldb::DB* pdb;
  160. //! a key used for optional XOR-obfuscation of the database
  161. std::vector<unsigned char> obfuscate_key;
  162. //! the key under which the obfuscation key is stored
  163. static const std::string OBFUSCATE_KEY_KEY;
  164. //! the length of the obfuscate key in number of bytes
  165. static const unsigned int OBFUSCATE_KEY_NUM_BYTES;
  166. std::vector<unsigned char> CreateObfuscateKey() const;
  167. public:
  168. /**
  169. * @param[in] path Location in the filesystem where leveldb data will be stored.
  170. * @param[in] nCacheSize Configures various leveldb cache settings.
  171. * @param[in] fMemory If true, use leveldb's memory environment.
  172. * @param[in] fWipe If true, remove all existing data.
  173. * @param[in] obfuscate If true, store data obfuscated via simple XOR. If false, XOR
  174. * with a zero'd byte array.
  175. */
  176. CDBWrapper(const fs::path& path, size_t nCacheSize, bool fMemory = false, bool fWipe = false, bool obfuscate = false);
  177. ~CDBWrapper();
  178. template <typename K, typename V>
  179. bool Read(const K& key, V& value) const
  180. {
  181. CDataStream ssKey(SER_DISK, CLIENT_VERSION);
  182. ssKey.reserve(DBWRAPPER_PREALLOC_KEY_SIZE);
  183. ssKey << key;
  184. leveldb::Slice slKey(ssKey.data(), ssKey.size());
  185. std::string strValue;
  186. leveldb::Status status = pdb->Get(readoptions, slKey, &strValue);
  187. if (!status.ok()) {
  188. if (status.IsNotFound())
  189. return false;
  190. LogPrintf("LevelDB read failure: %s\n", status.ToString());
  191. dbwrapper_private::HandleError(status);
  192. }
  193. try {
  194. CDataStream ssValue(strValue.data(), strValue.data() + strValue.size(), SER_DISK, CLIENT_VERSION);
  195. ssValue.Xor(obfuscate_key);
  196. ssValue >> value;
  197. } catch (const std::exception&) {
  198. return false;
  199. }
  200. return true;
  201. }
  202. template <typename K, typename V>
  203. bool Write(const K& key, const V& value, bool fSync = false)
  204. {
  205. CDBBatch batch(*this);
  206. batch.Write(key, value);
  207. return WriteBatch(batch, fSync);
  208. }
  209. template <typename K>
  210. bool Exists(const K& key) const
  211. {
  212. CDataStream ssKey(SER_DISK, CLIENT_VERSION);
  213. ssKey.reserve(DBWRAPPER_PREALLOC_KEY_SIZE);
  214. ssKey << key;
  215. leveldb::Slice slKey(ssKey.data(), ssKey.size());
  216. std::string strValue;
  217. leveldb::Status status = pdb->Get(readoptions, slKey, &strValue);
  218. if (!status.ok()) {
  219. if (status.IsNotFound())
  220. return false;
  221. LogPrintf("LevelDB read failure: %s\n", status.ToString());
  222. dbwrapper_private::HandleError(status);
  223. }
  224. return true;
  225. }
  226. template <typename K>
  227. bool Erase(const K& key, bool fSync = false)
  228. {
  229. CDBBatch batch(*this);
  230. batch.Erase(key);
  231. return WriteBatch(batch, fSync);
  232. }
  233. bool WriteBatch(CDBBatch& batch, bool fSync = false);
  234. // not available for LevelDB; provide for compatibility with BDB
  235. bool Flush()
  236. {
  237. return true;
  238. }
  239. bool Sync()
  240. {
  241. CDBBatch batch(*this);
  242. return WriteBatch(batch, true);
  243. }
  244. CDBIterator *NewIterator()
  245. {
  246. return new CDBIterator(*this, pdb->NewIterator(iteroptions));
  247. }
  248. /**
  249. * Return true if the database managed by this class contains no entries.
  250. */
  251. bool IsEmpty();
  252. template<typename K>
  253. size_t EstimateSize(const K& key_begin, const K& key_end) const
  254. {
  255. CDataStream ssKey1(SER_DISK, CLIENT_VERSION), ssKey2(SER_DISK, CLIENT_VERSION);
  256. ssKey1.reserve(DBWRAPPER_PREALLOC_KEY_SIZE);
  257. ssKey2.reserve(DBWRAPPER_PREALLOC_KEY_SIZE);
  258. ssKey1 << key_begin;
  259. ssKey2 << key_end;
  260. leveldb::Slice slKey1(ssKey1.data(), ssKey1.size());
  261. leveldb::Slice slKey2(ssKey2.data(), ssKey2.size());
  262. uint64_t size = 0;
  263. leveldb::Range range(slKey1, slKey2);
  264. pdb->GetApproximateSizes(&range, 1, &size);
  265. return size;
  266. }
  267. /**
  268. * Compact a certain range of keys in the database.
  269. */
  270. template<typename K>
  271. void CompactRange(const K& key_begin, const K& key_end) const
  272. {
  273. CDataStream ssKey1(SER_DISK, CLIENT_VERSION), ssKey2(SER_DISK, CLIENT_VERSION);
  274. ssKey1.reserve(DBWRAPPER_PREALLOC_KEY_SIZE);
  275. ssKey2.reserve(DBWRAPPER_PREALLOC_KEY_SIZE);
  276. ssKey1 << key_begin;
  277. ssKey2 << key_end;
  278. leveldb::Slice slKey1(ssKey1.data(), ssKey1.size());
  279. leveldb::Slice slKey2(ssKey2.data(), ssKey2.size());
  280. pdb->CompactRange(&slKey1, &slKey2);
  281. }
  282. };
  283. #endif // STARWELS_DBWRAPPER_H