Opera 12.15 Source Code
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

BlockStorage.h 20KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586
  1. /* -*- Mode: c++; tab-width: 4; indent-tabs-mode: t; c-basic-offset: 4 -*-
  2. **
  3. ** Copyright (C) 1995-2011 Opera Software ASA. All rights reserved.
  4. **
  5. ** This file is part of the Opera web browser.
  6. ** It may not be distributed under any circumstances.
  7. */
  8. #ifndef BLOCKSTORAGE_H
  9. #define BLOCKSTORAGE_H
  10. #include "modules/pi/system/OpLowLevelFile.h"
  11. #include "modules/search_engine/BinCompressor.h"
  12. class BlockStorage;
  13. /**
  14. * Parent class for BlockStorage based search indexes that can be grouped
  15. * together with others for coordinated Commit.
  16. *
  17. * <p>Note! When several search indexes are grouped, it is the responsibility
  18. * of the code performing the grouping to make sure that Commit is called
  19. * on all search indexes in the group in one operation. Committing only some
  20. * of the indexes will not result in a full commit. Only a successful result
  21. * from all commits including the last ensures that all indexes are committed.
  22. * This can be checked with IsFullyCommitted().
  23. */
  24. class SearchGroupable
  25. {
  26. public:
  27. /**
  28. * Merge this group with (the group of) another SearchGroupable. Grouping
  29. * should be done before any of the members of the group are opened or used
  30. */
  31. void GroupWith(SearchGroupable &group_member);
  32. /**
  33. * Get the BlockStorage (of a member of the group) that this
  34. * SearchGroupable represents. If this SearchGroupable represents several
  35. * BlockStorages or SearchGroupables, they must themselves already be
  36. * grouped.
  37. */
  38. virtual BlockStorage &GetGroupMember() = 0;
  39. /**
  40. * Check if all members of a group have been committed. Note that this
  41. * function can only be used for verifying a full commit at the highest
  42. * level of grouping. It is not necessary to check the other members.
  43. * @return TRUE if all members of the group have been fully committed
  44. */
  45. virtual BOOL IsFullyCommitted();
  46. };
  47. /**
  48. * @brief File storage designed to avoid safe file replace.
  49. * @author Pavel Studeny <pavels@opera.com>
  50. *
  51. * When you save your data into the file, you receive a file position (can be viewed as ID)
  52. * for further manipulation with the data.
  53. *
  54. * File structure:
  55. * @li 8 B position of the first free block / 0 if no free block is available
  56. * @li 4 B header
  57. * @li 4 B block size
  58. *
  59. * n * m_blocksize linked blocks
  60. *
  61. * Block sturcture:
  62. * @li 8 B position of the next block (msb == 0 for start blocks, 1 for consecutive blocks)
  63. * @li (first block in a chain only) 4 B size of the data
  64. *
  65. *
  66. *
  67. * @par BlockStorage offers 4 levels of safety for write operations:
  68. *
  69. * @par 1. Plain write, no transactions
  70. * Fast and reasonably safe for applications not requring consistency among several data records.
  71. * @par
  72. * Methods to use:
  73. * @li Write/Update
  74. *
  75. * @par 2. Write in a transaction
  76. * Safe except loss of electric power or operating system crash, allows commit or rollback from any point of the transaction.
  77. * @par
  78. * Methods to use:
  79. * @li BeginTransaction
  80. * @li Write/Update
  81. * @li Commit/Rollback
  82. *
  83. * @par 3. Transaction with prejournalling
  84. * All the free blocks and blocks going to be modified are journallled before the modification, journal is SafeClosed
  85. * before any modification of the file. Safety for loss of electric power or operating system crash depends on the
  86. * filesystem used.
  87. * @par
  88. * Methods to use:
  89. * @li BeginTransaction
  90. * @li PreJournal
  91. * @li FlushJournal
  92. * @li Write/Update
  93. * @li Commit/Rollback
  94. *
  95. * @par 4. Prejournalling to invalid journal
  96. * Journal is marked as invalid until FlushJournal is called. Safe on any filesystem.
  97. * @par
  98. * Methods to use:
  99. * @li BeginTransaction(TRUE)
  100. * @li PreJournal
  101. * @li FlushJournal
  102. * @li Write/Update
  103. * @li Commit/Rollback
  104. *
  105. * @par Preformance tweaks
  106. * Safety level 2 calls SafeClose on Commit, safety level 3 and 4 call SafeClose on Commit and FlushJournal.
  107. * SafeClose makes sure that all the data written to the file are physically on the disk, but it's extremely
  108. * slow. However, if there was a delay between writing the data and SafeClose, there is a high probability,
  109. * that the data were already physically written to the disk and SafeClose is then much faster. To avoid
  110. * freezing in SafeClose in a single threaded program, try something like this (example for safety level 4):
  111. * @code
  112. * BeginTransaction(TRUE);
  113. * PreJournal(blockN); PreJournal(blockM); PreJournal(blockU);
  114. * // do something else
  115. * FlushJournal();
  116. * Write(blockN); Write(blockM); Update(blockU);
  117. * // do something else
  118. * Commit();
  119. * @endcode
  120. *
  121. */
  122. class BlockStorage : public NonCopyable, public SearchGroupable
  123. {
  124. public:
  125. enum OpenMode
  126. {
  127. OpenRead = 1,
  128. OpenReadWrite = 2
  129. };
  130. /**
  131. * @param data pointer to the data to switch
  132. * @param size total size of the current data block
  133. * @param user_arg value supplied by user on setup
  134. * @return length of processed data; if less than size, the callback will be called again with the remaining data; abort operation if <= 0
  135. */
  136. typedef int (* SwitchEndianCallback)(void *data, int size, void *user_arg);
  137. BlockStorage(void)
  138. : ReadInt32(NULL)
  139. , WriteInt32(NULL)
  140. , ReadInt64(NULL)
  141. , WriteInt64(NULL)
  142. , m_file(NULL)
  143. , m_blocksize(0)
  144. , m_buffered_blocks(0)
  145. , m_EndianCallback(NULL)
  146. , m_callback_arg(NULL)
  147. , m_start_blocks_supported(FALSE)
  148. , m_freeblock(INVALID_FILE_LENGTH)
  149. , m_transaction(NULL)
  150. , m_transaction_blocks(0)
  151. , m_reserved_area(0)
  152. , m_reserved_size(0)
  153. , m_reserved_deleted(INVALID_FILE_LENGTH)
  154. , m_next_in_group(this)
  155. , m_first_in_group(FALSE)
  156. , m_group_commit_incomplete(FALSE)
  157. {}
  158. ~BlockStorage(void);
  159. /**
  160. * open the file for read or read/write access
  161. * @param path file location
  162. * @param mode open mode, OPFILE_XXX constants
  163. * @param blocksize size of one data block including block header
  164. * @param buffered_blocks buffer multiple blocks when reading
  165. * @param folder might be one of predefind folders
  166. */
  167. CHECK_RESULT(OP_STATUS Open(const uni_char* path, OpenMode mode, int blocksize,
  168. int buffered_blocks = 0, OpFileFolder folder = OPFILE_ABSOLUTE_FOLDER));
  169. /**
  170. * close the file and rollback any pending transaction
  171. */
  172. void Close(void);
  173. #ifdef SELFTEST
  174. /**
  175. * crash simulation for selftests
  176. */
  177. void Crash(void);
  178. #endif
  179. /**
  180. * @return TRUE if the file is opened
  181. */
  182. BOOL IsOpen(void) const {return m_file != NULL;}
  183. /**
  184. * erase all data
  185. */
  186. CHECK_RESULT(OP_STATUS Clear(int blocksize = 0));
  187. /**
  188. * @return size of one data block
  189. */
  190. int GetBlockSize(void) const
  191. {
  192. return m_blocksize;
  193. }
  194. OpFileLength GetFileSize(void)
  195. {
  196. OpFileLength size;
  197. if (m_file == NULL)
  198. return 0;
  199. RETURN_VALUE_IF_ERROR(m_file->GetFileLength(&size), 0);
  200. return size;
  201. }
  202. /**
  203. * @return TRUE if endians on disk are the same like in memory
  204. */
  205. BOOL IsNativeEndian(void) const;
  206. /**
  207. * set conversion callback called on every read/write/update
  208. * @param cb callback converting either the full data block or, in case of arrays, it's items one by one; can be NULL
  209. * @param user_arg argument to be passed to callback
  210. */
  211. void SetOnTheFlyCnvFunc(SwitchEndianCallback cb, void *user_arg = NULL);
  212. /**
  213. * just a helper function to change the endain of an integer
  214. */
  215. static void SwitchEndian(void *buf, int size);
  216. /**
  217. * Write, Update and Delete called within a transaction are crash-safe
  218. * @param invalidate_journal set journal as invalid until FlushJournal is called,
  219. * no changes are allowed until FlushJournal is called, use just PreJournal and Reserve
  220. * @return OpStatus::ERR if there is already another transaction running
  221. */
  222. CHECK_RESULT(OP_STATUS BeginTransaction(BOOL invalidate_journal = FALSE));
  223. /**
  224. * confirm all changes since BeginTransaction
  225. */
  226. CHECK_RESULT(OP_STATUS Commit(void));
  227. /**
  228. * reject all changes since BeginTransaction
  229. */
  230. CHECK_RESULT(OP_STATUS Rollback(void));
  231. /**
  232. * @return TRUE on ongoing transaction
  233. */
  234. BOOL InTransaction(void) const {return m_transaction && m_transaction->IsOpen();}
  235. /**
  236. * journal blocks in advance, see FlushJournal
  237. */
  238. CHECK_RESULT(OP_STATUS PreJournal(OpFileLength pos));
  239. /**
  240. * make sure that the prejournaled blocks are on the metal - slow, but 100% safe
  241. */
  242. CHECK_RESULT(OP_STATUS FlushJournal(void));
  243. /**
  244. * reserve and journal a block for a future write without modifying the block itself
  245. * @return position of the block to be passed to Write or 0 on failure
  246. */
  247. CHECK_RESULT(OpFileLength Reserve());
  248. /**
  249. * write data to previously reserved block, supposed to be called after FlushJournal
  250. * @param data block of data to write
  251. * @param len should fit into one block, otherwise a new block will be allocated and journalled
  252. * @param reserved_pos value returned by Reserve()
  253. * @return reserved_pos or 0 on failure, failure is only possible on an allocation of a new block
  254. */
  255. CHECK_RESULT(OpFileLength Write(const void *data, int len, OpFileLength reserved_pos));
  256. /**
  257. * add new data
  258. * @return position of the first block of the data in the file / 0 on failure
  259. */
  260. CHECK_RESULT(OpFileLength Write(const void *data, int len));
  261. /**
  262. * @param pos value returned from Write
  263. * @return size of the stored data
  264. */
  265. int DataLength(OpFileLength pos);
  266. /**
  267. * read previously written data into memory
  268. * @param data buffer to to the data into
  269. * @param len size of the buffer
  270. * @param pos value returned from Write
  271. * @return TRUE on success
  272. */
  273. CHECK_RESULT(BOOL Read(void *data, int len, OpFileLength pos));
  274. /**
  275. * replace existing data
  276. * @param data new data
  277. * @param len new data size
  278. * @param pos value returned from Write
  279. * @return TRUE on success
  280. */
  281. CHECK_RESULT(BOOL Update(const void *data, int len, OpFileLength pos));
  282. /**
  283. * replace a piece of existing data, doesn't decrease the size of the data, but it can increase it if offset+size > data length
  284. * @param data new data
  285. * @param offset position of the modified data in the data block, shouldn't be more than a size of the current data
  286. * @param len size of the piece of modified data
  287. * @param pos value returned from Write
  288. * @return TRUE on success
  289. */
  290. CHECK_RESULT(BOOL Update(const void *data, int offset, int len, OpFileLength pos));
  291. /**
  292. * strip existing data
  293. * @param len new data size, the block remains valid even if len was 0
  294. * @param pos value returned from Write
  295. * @return TRUE on success
  296. */
  297. CHECK_RESULT(BOOL Update(int len, OpFileLength pos));
  298. /**
  299. * remove data from the file
  300. * @param pos value returned from Write
  301. * @return TRUE on success
  302. */
  303. CHECK_RESULT(BOOL Delete(OpFileLength pos));
  304. /**
  305. * write an additional header data
  306. * @param offset offset in bytes from the first user value
  307. * @param data user data
  308. * @param len size of the data, must fit in the header, the space left is blocksize - 16
  309. * @param disk_len if not 0, endian will be adjusted and saved on disk_len bytes
  310. * @param count if > 1, data is an array of len * count bytes
  311. * @return TRUE on success
  312. */
  313. CHECK_RESULT(BOOL WriteUserHeader(unsigned offset, const void *data, int len, int disk_len = 0, int count = 1));
  314. CHECK_RESULT(BOOL WriteUserHeader(const void *data, int len, int disk_len = 0, int count = 1))
  315. {
  316. return WriteUserHeader(0, data, len, disk_len, count);
  317. }
  318. /**
  319. * read an additional header data
  320. * @param offset offset in bytes from the first user value
  321. * @param data user data, data are filled by 0 if WriteUserHeader hasn't been called yet
  322. * @param len size of the data
  323. * @param disk_len if not 0, endian will be adjusted and read from disk_len bytes
  324. * @param count if > 1, data is an array of len * count bytes
  325. * @return TRUE on success
  326. */
  327. CHECK_RESULT(BOOL ReadUserHeader(unsigned offset, void *data, int len, int disk_len = 0, int count = 1));
  328. CHECK_RESULT(BOOL ReadUserHeader(void *data, int len, int disk_len = 0, int count = 1))
  329. {
  330. return ReadUserHeader(0, data, len, disk_len, count);
  331. }
  332. /**
  333. * Special mode optimized for fast appending of data, cannot be mixed with Write/Update/Read.
  334. * When using this method IsStartBlock(OpFileLength) will not work correctly.
  335. * @param data new or additional data to append
  336. * @param len length of data in bytes
  337. * @param pos last value returned from Append or 0 if no data had been written yet
  338. * @return position of the first block, note that this position can change, 0 on error
  339. */
  340. CHECK_RESULT(OpFileLength Append(const void *data, int len, OpFileLength pos));
  341. /**
  342. * read data written by Append
  343. * @param data buffer to to the data into
  344. * @param len size of the buffer
  345. * @param pos value returned from Write
  346. * @return TRUE on success
  347. */
  348. CHECK_RESULT(BOOL ReadApnd(void *data, int len, OpFileLength pos));
  349. const uni_char *GetFullName(void) const {return m_file->GetFullPath();}
  350. /** normal read that returns error if data could not be fully read.
  351. * The running thread is blocked until all data is read.
  352. */
  353. CHECK_RESULT(static OP_STATUS ReadFully(OpLowLevelFile *f, void* data, OpFileLength len));
  354. /** checks whether a file or directory exists */
  355. CHECK_RESULT(static OP_BOOLEAN FileExists(const uni_char *name, OpFileFolder folder = OPFILE_ABSOLUTE_FOLDER));
  356. /** deletes a file or directory, return IS_TRUE if file existed; may delete not-empty directories, depending on the actual OpFile implementation */
  357. CHECK_RESULT(static OP_BOOLEAN DeleteFile(const uni_char *name, OpFileFolder folder = OPFILE_ABSOLUTE_FOLDER));
  358. /** checks whether a directory is empty */
  359. CHECK_RESULT(static OP_BOOLEAN DirectoryEmpty(const uni_char *name, OpFileFolder folder = OPFILE_ABSOLUTE_FOLDER));
  360. /** renames a file */
  361. CHECK_RESULT(static OP_STATUS RenameFile(const uni_char *old_path, const uni_char *new_path, OpFileFolder folder = OPFILE_ABSOLUTE_FOLDER));
  362. /** renames storage and journal files */
  363. CHECK_RESULT(static OP_STATUS RenameStorage(const uni_char *old_path, const uni_char *new_path, OpFileFolder folder = OPFILE_ABSOLUTE_FOLDER));
  364. /**
  365. * Check if this block storage file supports querying of start blocks.
  366. *
  367. * @return TRUE if start blocks are supported
  368. */
  369. BOOL IsStartBlocksSupported() const { return m_start_blocks_supported; }
  370. /**
  371. * Check if a block is a valid (non-deleted) start block.
  372. * Start blocks must be supported for this method to work, see IsStartBlocksSupported().
  373. * An optimized way to read all data in a file more or less sequentially, is something
  374. * like this:
  375. *
  376. * <pre>
  377. * int block_size = bs->GetBlockSize();
  378. * OpFileLength file_size = bs->GetFileSize();
  379. * for (OpFileLength pos=0; pos < file_size; pos += block_size)
  380. * {
  381. * if (bs->IsStartBlock(pos))
  382. * {
  383. * int length = bs->DataLength(pos);
  384. * data = new ...;
  385. * bs->Read(data, length, pos);
  386. * }
  387. * }
  388. * </pre>
  389. *
  390. * @param pos a block file position
  391. * @return TRUE if the given block is a valid start block
  392. */
  393. BOOL IsStartBlock(OpFileLength pos);
  394. /**
  395. * Mark a specific block as a start block. This is only allowed for files
  396. * that do not yet support start blocks, and is intended for upgrading such a file
  397. * to support start blocks, see IsStartBlocksSupported(). When all start blocks
  398. * have the start block flag set, the file must be upgraded for supporting start blocks
  399. * using StartBlocksUpdated().
  400. *
  401. * @param pos a block file position
  402. * @return TRUE if the given block is a valid, non-deleted start block
  403. */
  404. CHECK_RESULT(OP_STATUS SetStartBlock(OpFileLength pos));
  405. /**
  406. * When all start blocks have been marked using SetStartBlock(OpFileLength),
  407. * use this method to upgrade the file. This is only allowed for files
  408. * that do not yet support start blocks, see IsStartBlocksSupported(). After
  409. * calling this method, IsStartBlocksSupported() will return TRUE.
  410. *
  411. * <p>SetStartBlock(OpFileLength) and StartBlocksUpdated() can take the place
  412. * of Write/Update operations in transaction/journalling operations. When
  413. * prejournaling, all blocks <i>except</i> start blocks should be prejournaled.
  414. *
  415. * @return TRUE if the given block is a valid, non-deleted start block
  416. */
  417. CHECK_RESULT(OP_STATUS StartBlocksUpdated());
  418. /**
  419. * @return an estimate of the memory used (in RAM) by this data structure
  420. */
  421. #ifdef ESTIMATE_MEMORY_USED_AVAILABLE
  422. size_t EstimateMemoryUsed() const;
  423. #endif
  424. /**
  425. * Group this BlockStorage with another
  426. */
  427. void GroupWith(BlockStorage &group_member);
  428. /**
  429. * Remove this BlockStorage from the group
  430. */
  431. void UnGroup();
  432. /**
  433. * Get SearchGroupable group member
  434. */
  435. virtual BlockStorage &GetGroupMember() { return *this; }
  436. /**
  437. * @return TRUE if all BlockStorages in the group has been committed
  438. */
  439. virtual BOOL IsFullyCommitted();
  440. #ifndef SELFTEST
  441. protected:
  442. #endif
  443. /**
  444. * Get the first member of the group
  445. */
  446. BlockStorage *GetFirstInGroup();
  447. /** Create (an empty, closed) file with a name starting with the same name as this BlockStorage */
  448. CHECK_RESULT(OP_STATUS CreateAssociatedFile(const uni_char *suffix));
  449. /** Check whether an associated file exists */
  450. CHECK_RESULT(OP_BOOLEAN AssociatedFileExists(const uni_char *suffix));
  451. /** Delete an associated file, return IS_TRUE if the file existed */
  452. CHECK_RESULT(OP_BOOLEAN DeleteAssociatedFile(const uni_char *suffix));
  453. /** Open an associated file */
  454. CHECK_RESULT(OP_STATUS OpenAssociatedFile(OpLowLevelFile **f, OpenMode mode, const uni_char *suffix));
  455. /** Build name of associated file */
  456. CHECK_RESULT(OP_STATUS BuildAssociatedFileName(OpString &name, const uni_char *suffix));
  457. protected:
  458. // endian-insensitive integers
  459. CHECK_RESULT(OP_STATUS (* ReadInt32)(OpLowLevelFile *, INT32 *));
  460. CHECK_RESULT(OP_STATUS (* WriteInt32)(OpLowLevelFile *, INT32));
  461. // It's safe to change OpFileLength to 64 bits
  462. CHECK_RESULT(OP_STATUS (* ReadInt64)(OpLowLevelFile *, OpFileLength *, UINT32 *msb32));
  463. CHECK_RESULT(OP_STATUS (* WriteInt64)(OpLowLevelFile *, OpFileLength, UINT32 msb32));
  464. CHECK_RESULT(OP_STATUS ReadOFL(OpLowLevelFile *f, OpFileLength *ofl));
  465. CHECK_RESULT(OP_STATUS WriteOFL(OpLowLevelFile *f, OpFileLength ofl));
  466. CHECK_RESULT(OP_STATUS ReadBlockHeader(OpLowLevelFile *f, OpFileLength *next_pos, BOOL *start_block = NULL));
  467. CHECK_RESULT(OP_STATUS WriteBlockHeader(OpLowLevelFile *f, OpFileLength next_pos, BOOL start_block));
  468. /**
  469. * Sanity check for file positions read from file (in case of corrupted files).
  470. * @param pos The file position to be checked
  471. * @return ERR if pos is negative or above the sanity check limit, otherwise OK
  472. */
  473. CHECK_RESULT(OP_STATUS CheckFilePosition(OpFileLength pos));
  474. // current_block == 0 -> new chain
  475. CHECK_RESULT(OpFileLength CreateBlock(OpFileLength current_block, BOOL current_start_block, BOOL append_mode = FALSE));
  476. // to be called if m_freeblock > 0
  477. CHECK_RESULT(OpFileLength GetFreeBlock(BOOL update_file));
  478. CHECK_RESULT(OP_STATUS Truncate(void));
  479. CHECK_RESULT(OP_STATUS JournalBlock(OpFileLength pos));
  480. void SwitchEndianInData(void *data, int size);
  481. BOOL WaitingForGroupCommit() const { return m_transaction && !m_transaction->IsOpen(); }
  482. OpLowLevelFile *m_file;
  483. OpString m_associated_file_name_buf;
  484. int m_blocksize;
  485. int m_buffered_blocks;
  486. SwitchEndianCallback m_EndianCallback;
  487. void *m_callback_arg;
  488. BOOL m_start_blocks_supported;
  489. private:
  490. // m_freeblock == 0 -> no free block available
  491. // m_freeblock == -1 -> free block status is unknown and should be read from the file
  492. OpFileLength m_freeblock;
  493. OpLowLevelFile *m_transaction;
  494. BinCompressor m_journal_compressor;
  495. // blocks present in the file before a transaction
  496. int m_transaction_blocks;
  497. OpFileLength m_reserved_area;
  498. OpFileLength m_reserved_size;
  499. OpFileLength m_reserved_deleted; // previously deleted block reserved for writing
  500. BlockStorage *m_next_in_group; // BlockStorage groups are linked in a loop
  501. BOOL m_first_in_group; // TRUE if this BlockStorage was opened first
  502. BOOL m_group_commit_incomplete; // TRUE if this BlockStorage detected an incomplete group commit
  503. #ifdef _DEBUG
  504. BOOL m_journal_invalid;
  505. #endif
  506. };
  507. #endif // BLOCKSTORAGE_H