Opera 12.15 Source Code
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

StringTable.h 13KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388
  1. /* -*- Mode: c++; tab-width: 4; indent-tabs-mode: t; c-basic-offset: 4 -*-
  2. **
  3. ** Copyright (C) 1995-2011 Opera Software ASA. All rights reserved.
  4. **
  5. ** This file is part of the Opera web browser.
  6. ** It may not be distributed under any circumstances.
  7. */
  8. #ifndef STRINGTABLE_H
  9. #define STRINGTABLE_H
  10. #include "modules/util/adt/opvector.h"
  11. #include "modules/search_engine/ACT.h"
  12. #include "modules/search_engine/BTree.h"
  13. #include "modules/search_engine/Vector.h"
  14. #include "modules/search_engine/PhraseSearch.h"
  15. #include "modules/probetools/probepoints.h"
  16. #if defined SEARCH_ENGINE_LOG && (SEARCH_ENGINE_LOG & SEARCH_ENGINE_LOG_STRINGTABLE)
  17. #include "modules/search_engine/log/Log.h"
  18. #endif
  19. /**
  20. * @brief Holds relationship between words and file IDs.
  21. * @author Pavel Studeny <pavels@opera.com>
  22. *
  23. * There are two files on disk: \<table name\>.act holds words and pointers to the second file,
  24. * \<table name\>.lex (the "wordbag") holds file IDs for appropriate words. There aren't duplicate file IDs for one word.
  25. */
  26. class StringTable : public SearchGroupable
  27. {
  28. public:
  29. /** flags passed to Open */
  30. enum OpenFlags
  31. {
  32. OverwriteCorrupted = 1, /**< Clean the files if they contain wrong data */
  33. CaseSensitive = 2, /**< Case sensitive words */
  34. ReadOnly = 4, /**< The files can be opened from multiple threads for a read-only access */
  35. OpenFlagMask = 4095, /**< Used internally, mask for the non-internal flags */
  36. PreFlushed = 4096, /**< Used internally, Flush had been called before Commit */
  37. CachesSorted = 8192, /**< Used internally, SortCaches finished successfully */
  38. CachesMerged = 16384, /**< Used internally, MergeCaches finished successfully */
  39. PreFlushing = 32768, /**< Used internally, time limit occured during PreFlush */
  40. UseNUR = 65536 /**< Used internally, PreFlush hadn't been called before Flush */
  41. };
  42. StringTable(void)
  43. {
  44. #ifdef SEARCH_ENGINE_PHRASESEARCH
  45. m_document_source = NULL;
  46. m_phrase_search_cutoff = 0;
  47. #endif
  48. m_act.GroupWith(m_wordbag);
  49. #if defined SEARCH_ENGINE_LOG && (SEARCH_ENGINE_LOG & SEARCH_ENGINE_LOG_STRINGTABLE)
  50. m_log = NULL;
  51. #endif
  52. }
  53. /**
  54. * open the data files
  55. * @param path file directory
  56. * @param table_name table name to cunstruct the file names
  57. * @param flags optional flags, see OpenFlags
  58. * @return error code / IS_TRUE if the tables existed / IS_FALSE if the table was newly created
  59. */
  60. CHECK_RESULT(OP_BOOLEAN Open(const uni_char *path, const uni_char *table_name, int flags = OverwriteCorrupted));
  61. /**
  62. * Flush all cached data and closes all resources
  63. * @param force_close close all resources even if all operations cannot be completed e.g. because out of disk space
  64. * @return if force_close was set, returns error anyway, but the resources are released
  65. */
  66. CHECK_RESULT(OP_STATUS Close(BOOL force_close = TRUE));
  67. /**
  68. * erase all data
  69. */
  70. CHECK_RESULT(OP_STATUS Clear());
  71. /**
  72. * begin transaction and prepare data from the cache to be written to disk;
  73. * all Inserts or Deletes until Commit will not be included in this transaction;
  74. * an error cancels the whole transaction and the data are returned back to cache
  75. *
  76. * @param max_ms maximum time to spend by PreFlush in miliseconds, 0 means unlimited
  77. * @return OpBoolean::IS_TRUE if finished successfully, OpBoolean::IS_FALSE if time limit was reached (call PreFlush again)
  78. * @see BlockStorage for more information about the transaction modes
  79. */
  80. CHECK_RESULT(OP_BOOLEAN PreFlush(int max_ms = 0));
  81. /**
  82. * write the data prepared by PreFlush;
  83. * calling PreFlush before Flush is optional, but a delay (roughly 30s) between PreFlush and Flush
  84. * reduces the time spent in operating system calls;
  85. * an error cancels the whole transaction and the data are returned back to cache
  86. *
  87. * @param max_ms maximum time to spend by PreFlush in miliseconds, 0 means unlimited
  88. * @return OpBoolean::IS_TRUE if finished successfully, OpBoolean::IS_FALSE if time limit was reached (call Flush again)
  89. */
  90. CHECK_RESULT(OP_BOOLEAN Flush(int max_ms = 0));
  91. /**
  92. * finish the transaction begun by PreFlush;
  93. * calling Flush before Commit is optional, but a delay (roughly 30s) between Flush and Commit
  94. * reduces the time spent in operating system calls
  95. * @return not supposed to fail under normal circumstances if Flush was called and finished successfully
  96. */
  97. CHECK_RESULT(OP_STATUS Commit(void));
  98. /**
  99. * if TRUE, there is no data to be flushed; this doesn't include any data being written by current transaction (PreFlush .. Commit) at the moment
  100. */
  101. BOOL CacheEmpty(void);
  102. /**
  103. * insert a word with its file ID
  104. */
  105. CHECK_RESULT(OP_STATUS Insert(const uni_char *word, INT32 file_id))
  106. {
  107. OP_PROBE4(OP_PROBE_SEARCH_ENGINE_STRINGTABLE_INSERT);
  108. // highest 3 bits are flags
  109. if ((file_id & 0xE0000000) != 0)
  110. RETURN_IF_ERROR(Insert(m_word_cache, word, file_id & 0x1FFFFFFF));
  111. return Insert(m_word_cache, word, file_id);
  112. }
  113. /**
  114. * parse and insert a block of plaintext words
  115. */
  116. CHECK_RESULT(OP_STATUS InsertBlock(const uni_char *words, INT32 file_id))
  117. {
  118. return InsertBlock(m_word_cache, words, file_id);
  119. }
  120. /**
  121. * deletes a file ID from the list of file IDs of the word
  122. */
  123. CHECK_RESULT(OP_STATUS Delete(const uni_char *word, INT32 file_id))
  124. {
  125. OP_PROBE4(OP_PROBE_SEARCH_ENGINE_STRINGTABLE_DELETE);
  126. // highest 3 bits are flags
  127. if ((file_id & 0xE0000000) != 0)
  128. RETURN_IF_ERROR(Insert(m_deleted_cache, word, file_id & 0x1FFFFFFF));
  129. return Insert(m_deleted_cache, word, file_id);
  130. }
  131. /**
  132. * parse and delete a block of plaintext words
  133. */
  134. CHECK_RESULT(OP_STATUS DeleteBlock(const uni_char *words, INT32 file_id))
  135. {
  136. return InsertBlock(m_deleted_cache, words, file_id);
  137. }
  138. /**
  139. * deletes a number of file IDs from a specific word
  140. * @param word
  141. * @param file_ids must be sorted
  142. */
  143. CHECK_RESULT(OP_STATUS Delete(const uni_char *word, const OpINT32Vector &file_ids));
  144. /**
  145. * deletes a number of file IDs from all words associated with these IDs
  146. * @param file_ids must be sorted
  147. */
  148. CHECK_RESULT(OP_STATUS Delete(const OpINT32Vector &file_ids));
  149. /**
  150. * deletes a word and all file IDs associated with it
  151. * @param word
  152. */
  153. CHECK_RESULT(OP_STATUS Delete(const uni_char *word));
  154. /**
  155. * find all the file IDs belonging to the word, Flushes all cached data before the search
  156. * @param word word (or a word prefix) to search
  157. * @param result resulting (sorted) file IDs, mustn't be NULL, is cleared before the search
  158. * @param prefix_search if not 0, search all the words with given prefix up to the given number
  159. */
  160. CHECK_RESULT(OP_STATUS Search(const uni_char *word, TVector<INT32> *result, int prefix_search = 0));
  161. CHECK_RESULT(OP_STATUS Search(const uni_char *word, OpINT32Vector *result, int prefix_search = 0))
  162. {
  163. TVector<INT32> tvres;
  164. unsigned i;
  165. result->Clear();
  166. RETURN_IF_ERROR(Search(word, &tvres, prefix_search));
  167. for (i = 0; i < tvres.GetCount(); ++i)
  168. RETURN_IF_ERROR(result->Add(tvres[i]));
  169. return OpStatus::OK;
  170. }
  171. /**
  172. * find the indexed words, Flushes all cached data before the search
  173. * @param word word (or a word prefix) to search
  174. * @param result resulting words, mustn't be NULL, the fields must be freed by caller
  175. * @param result_size maximum number of results on input, number of results on output
  176. */
  177. CHECK_RESULT(OP_STATUS WordSearch(const uni_char *word, uni_char **result, int *result_size));
  178. /**
  179. * find all the file IDs belonging to at least one of the words, Flushes all cached data before the search
  180. * @param words phrase to search for
  181. * @param result resulting (sorted) file IDs, mustn't be NULL
  182. * @param match_any if FALSE, document must contain all the words, if TRUE, document must contain at least one word
  183. * @param prefix_search if not 0, apply a prefix search for the last word and search for max. prefix_search prefixes
  184. * @param phrase_flags flags built up from PhraseMatcher::PhraseFlags, controlling what kind of phrase search is performed
  185. */
  186. CHECK_RESULT(OP_STATUS MultiSearch(const uni_char *words, TVector<INT32> *result, BOOL match_any, int prefix_search = 0, int phrase_flags = 0));
  187. CHECK_RESULT(OP_STATUS MultiSearch(const uni_char *words, OpINT32Vector *result, BOOL match_any, int prefix_search = 0, int phrase_flags = 0))
  188. {
  189. TVector<INT32> tvres;
  190. unsigned i;
  191. result->Clear();
  192. RETURN_IF_ERROR(MultiSearch(words, &tvres, match_any, prefix_search, phrase_flags));
  193. for (i = 0; i < tvres.GetCount(); ++i)
  194. RETURN_IF_ERROR(result->Add(tvres[i]));
  195. return OpStatus::OK;
  196. }
  197. #ifdef SEARCH_ENGINE_PHRASESEARCH
  198. /**
  199. * Configure phrase search (when using nonzero phrase_flags in MultiSearch).
  200. * @param document_source A document source for post-processing to filter out results that don't contain the phrase.
  201. * @param phrase_search_cutoff Cutoff number to limit the number of preliminary results that are used in
  202. * phrase search. If there are too many results for the words alone, post-processing might take extremely long.
  203. * Setting 0 disables the cutoff limit.
  204. */
  205. void ConfigurePhraseSearch(DocumentSource<INT32>* document_source, UINT32 phrase_search_cutoff)
  206. { m_document_source = document_source; m_phrase_search_cutoff = phrase_search_cutoff; }
  207. #endif
  208. /**
  209. * search for words separated by non-word characters
  210. * @param words phrase to search for
  211. * @param prefix_search if not 0, apply a prefix search for the last word and search for max. prefix_search prefixes
  212. * @param phrase_flags flags built up from PhraseMatcher::PhraseFlags, controlling what kind of phrase search is performed
  213. * @return iterator which must be deleted by a caller or NULL on error
  214. */
  215. SearchIterator<INT32> *PhraseSearch(const uni_char *words, int prefix_search = 1024, int phrase_flags = 0);
  216. /**
  217. * @return TRUE if endians on disk are the same like in memory
  218. */
  219. BOOL IsNativeEndian(void)
  220. {
  221. return m_wordbag.GetStorage()->IsNativeEndian();
  222. }
  223. static BOOL CompareID(const void *left, const void *right) {return (*(INT32 *)left & 0x1FFFFFFF) < (*(INT32 *)right & 0x1FFFFFFF);}
  224. /**
  225. * check if all the data in files are correct
  226. * @param thorough If TRUE, the BTree will be checked in depth for recursive sorting errors
  227. * @return IS_TRUE if data are OK, IS_FALSE if data are corrupted, other value on error during the check
  228. */
  229. CHECK_RESULT(OP_BOOLEAN CheckConsistency(BOOL thorough = TRUE));
  230. /**
  231. * recovers what it can from a corrupted StringTable. the stringtable has to be closed when calling this function.
  232. * @param path file directory
  233. * @param table_name table name to cunstruct the file names
  234. * @return OpStatus::OK if everything went fine
  235. */
  236. CHECK_RESULT(OP_STATUS Recover(const uni_char* path, const uni_char* tablename));
  237. /**
  238. * @return an estimate of the memory used by this data structure
  239. */
  240. #ifdef ESTIMATE_MEMORY_USED_AVAILABLE
  241. size_t EstimateMemoryUsed() const;
  242. #endif
  243. /**
  244. * Get SearchGroupable group member
  245. */
  246. virtual BlockStorage &GetGroupMember() { return m_act.GetGroupMember(); }
  247. friend class STPrefixIterator;
  248. #ifdef SELFTEST
  249. ACT *GetACT() {return &m_act;}
  250. TPool<INT32> *GetBT() {return &m_wordbag;}
  251. #endif
  252. struct FileWord : public NonCopyable
  253. {
  254. uni_char *word;
  255. TBTree<INT32> *btree;
  256. TVector<INT32> *file_ids;
  257. BOOL is_new_word;
  258. FileWord(void)
  259. {
  260. word = NULL;
  261. btree = NULL;
  262. file_ids = NULL;
  263. is_new_word = FALSE;
  264. }
  265. ~FileWord(void)
  266. {
  267. if (btree != NULL)
  268. OP_DELETE(btree);
  269. if (word != NULL)
  270. op_free(word);
  271. if (file_ids != NULL)
  272. OP_DELETE(file_ids);
  273. }
  274. INT32 GetLastID(void)
  275. {
  276. return file_ids->Get(file_ids->GetCount() - 1);
  277. }
  278. CHECK_RESULT(OP_STATUS Add(INT32 file_id))
  279. {
  280. if (GetLastID() == file_id)
  281. return OpStatus::OK;
  282. return file_ids->Add(file_id);
  283. }
  284. #ifdef ESTIMATE_MEMORY_USED_AVAILABLE
  285. size_t EstimateMemoryUsed() const;
  286. #endif
  287. static FileWord *Create(const uni_char *word, INT32 file_id);
  288. };
  289. protected:
  290. class WordCache : private OpGenericVector
  291. {
  292. public:
  293. int BinarySearch(const uni_char *key);
  294. FileWord *Get(int index) {return (FileWord *)OpGenericVector::Get(index);}
  295. FileWord *operator[](int index) {return (FileWord *)OpGenericVector::Get(index);}
  296. CHECK_RESULT(OP_STATUS Insert(int index, FileWord *value)) {return OpGenericVector::Insert(index, value);}
  297. int GetCount(void) {return OpGenericVector::GetCount();}
  298. FileWord *Remove(int index) {return (FileWord *)OpGenericVector::Remove(index);}
  299. #ifdef ESTIMATE_MEMORY_USED_AVAILABLE
  300. size_t EstimateMemoryUsed() const;
  301. #endif
  302. void Clear();
  303. CHECK_RESULT(OP_STATUS DuplicateOf(const WordCache& vec)) { return OpGenericVector::DuplicateOf(vec); }
  304. void MoveFrom(WordCache &src);
  305. CHECK_RESULT(OP_STATUS CopyFrom(WordCache &src));
  306. };
  307. CHECK_RESULT(OP_STATUS Insert(WordCache &cache, const uni_char *word, INT32 file_id));
  308. CHECK_RESULT(OP_STATUS InsertBlock(WordCache &cache, const uni_char *words, INT32 file_id));
  309. CHECK_RESULT(OP_STATUS SearchForBTree(FileWord *fw, BOOL must_exist = FALSE));
  310. CHECK_RESULT(OP_STATUS SortCaches(void));
  311. void MergeCaches(void);
  312. ACT m_act;
  313. TPool<INT32> m_wordbag;
  314. WordCache m_word_cache;
  315. WordCache m_deleted_cache;
  316. WordCache m_word_preflush;
  317. WordCache m_deleted_preflush;
  318. WordCache m_word_backup;
  319. WordCache m_deleted_backup;
  320. int m_word_pos;
  321. int m_deleted_pos;
  322. int m_flags;
  323. #ifdef SEARCH_ENGINE_PHRASESEARCH
  324. DocumentSource<INT32>* m_document_source;
  325. UINT32 m_phrase_search_cutoff;
  326. #endif
  327. #if defined SEARCH_ENGINE_LOG && (SEARCH_ENGINE_LOG & SEARCH_ENGINE_LOG_STRINGTABLE)
  328. OutputLogDevice *m_log;
  329. #endif
  330. private:
  331. CHECK_RESULT(OP_STATUS AbortPreFlush(OP_STATUS err));
  332. CHECK_RESULT(OP_STATUS AbortFlush(OP_STATUS err));
  333. };
  334. #endif // STRINGTABLE_H