Opera 12.15 Source Code
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

PhraseSearch.cpp 13KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490
  1. /* -*- Mode: c++; tab-width: 4; indent-tabs-mode: t; c-basic-offset: 4 -*-
  2. **
  3. ** Copyright (C) 1995-2011 Opera Software ASA. All rights reserved.
  4. **
  5. ** This file is part of the Opera web browser.
  6. ** It may not be distributed under any circumstances.
  7. */
  8. #include "core/pch.h"
  9. #if defined(SEARCH_ENGINE_PHRASESEARCH) || defined(USE_SEARCH_ENGINE_WORDHIGHLIGHTER)
  10. #include "modules/search_engine/PhraseSearch.h"
  11. #include "modules/unicode/unicode.h"
  12. PhraseMatcher::PhraseMatcher()
  13. : m_query(NULL)
  14. , m_phrases(PtrDescriptor<TVector<Word> >())
  15. , m_phrase_flags(0)
  16. {
  17. }
  18. PhraseMatcher::~PhraseMatcher()
  19. {
  20. if ((m_phrase_flags & DontCopyInputString) == 0)
  21. OP_DELETEA(const_cast<uni_char*>(m_query));
  22. }
  23. OP_STATUS PhraseMatcher::Init(const uni_char *query, int phrase_flags)
  24. {
  25. BOOL in_quote = FALSE, quote_found;
  26. const uni_char *q, *q2, *end;
  27. WordSegmenter ws(WordSegmenter::DontCopyInputString);
  28. OpAutoPtr<TVector<WordSegmenter::Word> > plain_words;
  29. OpAutoPtr<TVector<Word> > phrase;
  30. Word word;
  31. UnicodePoint c;
  32. int inc;
  33. if ((m_phrase_flags & DontCopyInputString) == 0) // Using the old flags!
  34. OP_DELETEA(const_cast<uni_char*>(m_query));
  35. m_query = NULL;
  36. m_phrases.Clear();
  37. m_phrase_flags = phrase_flags;
  38. if (query == NULL || *query == 0 || (phrase_flags & (AllPhrases|FullSearch)) == 0)
  39. return OpStatus::OK;
  40. if ((m_phrase_flags & DontCopyInputString) == 0)
  41. RETURN_OOM_IF_NULL(m_query = WordSegmenter::PreprocessDup(query));
  42. else
  43. m_query = query;
  44. q = m_query;
  45. end = m_query + uni_strlen(m_query);
  46. RETURN_IF_ERROR(ws.Set(m_query));
  47. plain_words.reset(ws.GetTokens());
  48. phrase.reset(OP_NEW(TVector<Word>, ()));
  49. RETURN_OOM_IF_NULL(phrase.get());
  50. while (*q)
  51. {
  52. c = Unicode::GetUnicodePoint(q, (int)(end+1-q), inc);
  53. while (TreatAsSpace(c))
  54. {
  55. q += inc;
  56. c = Unicode::GetUnicodePoint(q, (int)(end+1-q), inc);
  57. }
  58. q2 = q;
  59. while (c && !IsDoubleQuote(c) && !TreatAsSpace(c))
  60. {
  61. q2 += inc;
  62. c = Unicode::GetUnicodePoint(q2, (int)(end+1-q2), inc);
  63. }
  64. word.Set(q, (int)(q2-q));
  65. if (c)
  66. q2 += inc;
  67. quote_found = IsDoubleQuote(c) && (phrase_flags & QuotedPhrases) != 0;
  68. if (word.len > 0 && (in_quote || plain_words->Find(word) == -1))
  69. RETURN_IF_ERROR(phrase->Add(word));
  70. if (phrase->GetCount() != 0 && (!in_quote || quote_found || !*q2))
  71. {
  72. if (phrase->GetCount() == 1 &&
  73. ((phrase_flags & (CJKPhrases|PunctuationPhrases)) == 0 ||
  74. plain_words->Find(phrase->Get(0)) >= 0))
  75. {
  76. phrase->Clear();
  77. }
  78. else
  79. {
  80. RETURN_IF_ERROR(m_phrases.Add(phrase.release()));
  81. phrase.reset(OP_NEW(TVector<Word>, ()));
  82. RETURN_OOM_IF_NULL(phrase.get());
  83. }
  84. }
  85. if (quote_found)
  86. in_quote = !in_quote;
  87. q = q2;
  88. }
  89. // In case of FullSearch, add all plain words that are not part of a phrase
  90. if ((phrase_flags & FullSearch) != 0)
  91. {
  92. UINT32 p=0;
  93. for (UINT32 w=0; w < plain_words->GetCount(); w++)
  94. {
  95. const WordSegmenter::Word plain_word = plain_words->Get(w);
  96. const uni_char *word_start = plain_word.ptr;
  97. const uni_char *word_end = plain_word.ptr + plain_word.len;
  98. const uni_char *phrase_start = NULL, *phrase_end = NULL;
  99. while (p < m_phrases.GetCount())
  100. {
  101. TVector<Word> *phr = m_phrases[p];
  102. phrase_start = (*phr)[0].ptr;
  103. phrase_end = (*phr)[phr->GetCount()-1].ptr + (*phr)[phr->GetCount()-1].len;
  104. if (word_end <= phrase_end)
  105. break;
  106. p++;
  107. }
  108. if (p >= m_phrases.GetCount() || word_start < phrase_start || word_end > phrase_end)
  109. {
  110. phrase.reset(OP_NEW(TVector<Word>, ()));
  111. RETURN_OOM_IF_NULL(phrase.get());
  112. word.Set(plain_word.ptr, plain_word.len);
  113. RETURN_IF_ERROR(phrase->Add(word));
  114. RETURN_IF_ERROR(m_phrases.Insert(p, phrase.release()));
  115. p++;
  116. }
  117. }
  118. }
  119. if ((phrase_flags & PrefixSearch) != 0 && m_phrases.GetCount() != 0)
  120. {
  121. TVector<Word> *last_phrase = m_phrases[m_phrases.GetCount()-1];
  122. Word &last_word = (*last_phrase)[last_phrase->GetCount()-1];
  123. if (last_word.ptr + last_word.len == end)
  124. last_word.is_last = TRUE;
  125. }
  126. return OpStatus::OK;
  127. }
  128. BOOL PhraseMatcher::Matches(const uni_char *haystack) const
  129. {
  130. if (Empty())
  131. return TRUE;
  132. if (haystack == NULL || *haystack == 0)
  133. return FALSE;
  134. for (UINT32 i=0; i < m_phrases.GetCount(); i++)
  135. if (!FindPhrase(m_phrases[i], haystack))
  136. return FALSE;
  137. return TRUE;
  138. }
  139. BOOL PhraseMatcher::IsDoubleQuote(UnicodePoint c)
  140. {
  141. // Unicode doesn't have a specific "class" for double quotes, so make it hardcoded according to the current spec
  142. return c == '"' || (c >= 0x201C && c <= 0x201F) || (c >= 0x301D && c <= 0x301F) || c == 0xFF02;
  143. }
  144. BOOL PhraseMatcher::TreatAsSpace(UnicodePoint c) const
  145. {
  146. return c && (Unicode::IsSpace(c) || Unicode::IsCntrl(c) || c == '+' ||
  147. ((m_phrase_flags & PunctuationPhrases) == 0 && !IsDoubleQuote(c) && !Unicode::IsAlphaOrDigit(c)));
  148. }
  149. BOOL PhraseMatcher::FindPhrase(TVector<Word> *phrase, const uni_char *haystack)
  150. {
  151. const uni_char *pos0 = haystack;
  152. const uni_char *pos;
  153. Word &word_0 = (*phrase)[0];
  154. UINT32 i;
  155. while (FindWord(word_0, pos0, haystack, TRUE))
  156. {
  157. pos0 = word_0.found_pos;
  158. pos = word_0.found_pos + word_0.found_len;
  159. for (i=1; i < phrase->GetCount(); i++)
  160. {
  161. Word &word_i = (*phrase)[i];
  162. if (!FindWord(word_i, pos, haystack, FALSE))
  163. break;
  164. pos = word_i.found_pos + word_i.found_len;
  165. }
  166. if (i == phrase->GetCount())
  167. return TRUE;
  168. pos0 ++;
  169. }
  170. return FALSE;
  171. }
  172. BOOL PhraseMatcher::FindWord(Word &word, const uni_char *s, const uni_char *haystack, BOOL first_word)
  173. {
  174. // Does not support case insensitive search for surrogate pairs, title-case characters, or Turkish i's
  175. // Cache lower-case and upper-case version of the first character in word
  176. UnicodePoint c = word.ptr[0];
  177. UnicodePoint w0_lo = Unicode::ToLower(c);
  178. UnicodePoint w0_up = Unicode::ToUpper(c);
  179. do
  180. {
  181. // Scan until we find the first letter of the word
  182. if (!first_word)
  183. {
  184. // A word following the first must be found with no intervening word characters
  185. while ((c = *s) != 0 && c != w0_lo && c != w0_up && !Unicode::IsAlphaOrDigit(c) && c != '\f')
  186. ++s;
  187. if (c == 0 || (c != w0_lo && c != w0_up))
  188. return FALSE;
  189. }
  190. else
  191. {
  192. while ((c = *s) != 0 && c != w0_lo && c != w0_up)
  193. s++;
  194. if (c == 0)
  195. return FALSE;
  196. }
  197. // After the first letter, match haystack and needle letter for letter
  198. int i,j;
  199. for (i=j=1; i < word.len; i++,j++)
  200. {
  201. while (WordSegmenter::IsInvisibleWordCharacter(s[j]))
  202. j++;
  203. if (Unicode::ToLower(s[j]) != Unicode::ToLower(word.ptr[i]))
  204. break;
  205. }
  206. if (i == word.len)
  207. {
  208. // Also match the remainder invisible characters to be sure we get to a word boundary
  209. while (WordSegmenter::IsInvisibleWordCharacter(s[j]))
  210. j++;
  211. if (WordSegmenter::WordBreak(haystack, s, TRUE) &&
  212. (word.is_last || WordSegmenter::WordBreak(haystack, s+j, TRUE)))
  213. {
  214. word.found_pos = s;
  215. word.found_len = j;
  216. return TRUE;
  217. }
  218. }
  219. s++;
  220. }
  221. while (TRUE);
  222. }
  223. OP_STATUS PhraseMatcher::AppendHighlight(OpString &dst,
  224. const uni_char *src,
  225. int max_chars,
  226. const OpStringC &start_tag,
  227. const OpStringC &end_tag,
  228. int prefix_ratio)
  229. {
  230. if (!src || *src == 0)
  231. return OpStatus::OK;
  232. // Max chars has not been set, the whole text should be used
  233. // Note that this is costly - users are encouraged to set max_chars
  234. if (max_chars <= 0)
  235. {
  236. max_chars = (int)uni_strlen(src);
  237. prefix_ratio = 0; // Start from beginning of text
  238. }
  239. // Make sure the buffer has all the size we'll need
  240. OpString buffer;
  241. int buffer_size = (max_chars * (start_tag.Length() + end_tag.Length())) + max_chars;
  242. uni_char *dst_ptr = buffer.Reserve(buffer_size);
  243. RETURN_OOM_IF_NULL(dst_ptr);
  244. // Reserve room for all the found phrases
  245. TVector<Word> results;
  246. Word result;
  247. RETURN_IF_ERROR(results.Reserve(m_phrases.GetCount()));
  248. const uni_char *first_phrase_start = NULL;
  249. // Find the occurrences of each phrase
  250. for (UINT32 p = 0; p < m_phrases.GetCount(); p++)
  251. {
  252. TVector<Word> *phrase = m_phrases[p];
  253. const uni_char *current_pos = src;
  254. while (FindPhrase(phrase, current_pos))
  255. {
  256. const uni_char *phrase_start = (*phrase)[0].found_pos;
  257. const uni_char *phrase_end = (*phrase)[phrase->GetCount()-1].found_pos + (*phrase)[phrase->GetCount()-1].found_len;
  258. result.Set(phrase_start, (int)(phrase_end-phrase_start));
  259. RETURN_IF_ERROR(results.Insert(result));
  260. if (first_phrase_start == NULL || phrase_start < first_phrase_start)
  261. first_phrase_start = phrase_start;
  262. current_pos = phrase_start+1;
  263. }
  264. }
  265. const uni_char *start = FindStart(src, first_phrase_start, max_chars, prefix_ratio);
  266. const uni_char *end = FindEnd(start, max_chars);
  267. // Merge overlapping results, clamp results at <end>
  268. UINT32 r = 0;
  269. while (r < results.GetCount())
  270. {
  271. while (r+1 < results.GetCount() && results[r+1].ptr <= results[r].ptr+results[r].len)
  272. {
  273. if (results[r+1].ptr + results[r+1].len > results[r].ptr + results[r].len)
  274. results[r].len = (int)(results[r+1].ptr + results[r+1].len - results[r].ptr); // Merge
  275. results.Remove(r+1);
  276. }
  277. if (results[r].ptr >= end)
  278. results.Remove(r);
  279. else
  280. {
  281. if (results[r].ptr + results[r].len > end)
  282. results[r].len = (int)(end - results[r].ptr); // Clamp
  283. r++;
  284. }
  285. }
  286. // Build the formatted quote :
  287. int appended_chars = 0;
  288. Word last;
  289. last.ptr = start;
  290. while (results.GetCount() > 0)
  291. {
  292. if (appended_chars + (results[0].ptr-(last.ptr+last.len)) > max_chars)
  293. break;
  294. appended_chars += CopyText(dst_ptr, last.ptr + last.len, results[0].ptr);
  295. if (appended_chars + results[0].len > max_chars)
  296. break;
  297. appended_chars += CopyResult(dst_ptr, results[0], start_tag, end_tag);
  298. last = results.Remove(0);
  299. }
  300. // Copy the rest of the string until end
  301. CopyText(dst_ptr, (last.ptr + last.len), end);
  302. return dst.Set(buffer);
  303. }
  304. /**
  305. * Copies text compressing whitespaces and returning the number
  306. * of characters copied.
  307. */
  308. int PhraseMatcher::CopyText(uni_char *&dst_ptr, const uni_char *from, const uni_char *end)
  309. {
  310. int appended_chars = 0;
  311. while (from < end)
  312. {
  313. if (uni_isspace(*from))
  314. {
  315. *dst_ptr++ = ' ';
  316. from++;
  317. // Skip the additional spaces
  318. while (uni_isspace(*from))
  319. from++;
  320. }
  321. else
  322. {
  323. *dst_ptr++ = *from++;
  324. }
  325. appended_chars++;
  326. }
  327. *dst_ptr = 0;
  328. return appended_chars;
  329. }
  330. /**
  331. * Copies a search result with formatting into the destination text and
  332. * returning number of characters copied - not including formatting
  333. * tags
  334. */
  335. int PhraseMatcher::CopyResult(uni_char *&dst_ptr, Word &result, const OpStringC &start_tag, const OpStringC &end_tag)
  336. {
  337. int appended_chars = 0;
  338. // Add on the prefix tag
  339. uni_strcpy(dst_ptr, start_tag.CStr());
  340. dst_ptr += start_tag.Length();
  341. // Add on the result itself
  342. appended_chars += CopyText(dst_ptr, result.ptr, result.ptr+result.len);
  343. // Add on the suffix tag
  344. uni_strcpy(dst_ptr, end_tag.CStr());
  345. dst_ptr += end_tag.Length();
  346. return appended_chars;
  347. }
  348. /**
  349. * Finds a uni_char pointer to the end of the text where compressing
  350. * of whitespaces is assumed.
  351. */
  352. const uni_char *PhraseMatcher::FindEnd(const uni_char *start, int length)
  353. {
  354. const uni_char *end = start;
  355. while (length > 0 && *end)
  356. {
  357. end++;
  358. if (uni_isspace(*(end-1)))
  359. {
  360. while (uni_isspace(*end))
  361. end++;
  362. }
  363. length--;
  364. }
  365. return end;
  366. }
  367. /**
  368. * Finds an appropriate start for a quote, attempting to get some context
  369. * for the first matching word which it assumes is the word pointed to by
  370. * first_start
  371. */
  372. const uni_char *PhraseMatcher::FindStart(const uni_char *text, const uni_char *first_start, int length, int prefix_ratio)
  373. {
  374. const uni_char *start = first_start;
  375. if (prefix_ratio <= 0 || first_start == NULL)
  376. return text;
  377. // Start some place before the first match, but don't cross "excerpt boundary" ('\r')
  378. while (start > text && start > first_start-length/prefix_ratio && *(start-1) != '\r')
  379. start --;
  380. while (start < first_start && !WordSegmenter::WordBreak(text, start))
  381. start++;
  382. while(uni_isspace(*start))
  383. start++;
  384. return start;
  385. }
  386. #ifdef SELFTEST
  387. uni_char *PhraseMatcher::GetPhrases() const
  388. {
  389. OpString s;
  390. for (UINT32 i=0; i < m_phrases.GetCount(); i++)
  391. {
  392. TVector<Word> *phrase = m_phrases[i];
  393. if (phrase->GetCount() > 1)
  394. s.Append("\"");
  395. for (UINT32 j=0; j < phrase->GetCount(); j++)
  396. {
  397. s.Append((*phrase)[j].ptr, (*phrase)[j].len);
  398. if (j < phrase->GetCount()-1)
  399. s.Append(" ");
  400. }
  401. if (phrase->GetCount() > 1)
  402. s.Append("\"");
  403. if (i < m_phrases.GetCount()-1)
  404. s.Append(" ");
  405. }
  406. if (m_phrases.GetCount() != 0)
  407. {
  408. TVector<Word> *last_phrase = m_phrases[m_phrases.GetCount()-1];
  409. Word &last_word = (*last_phrase)[last_phrase->GetCount()-1];
  410. if (last_word.is_last)
  411. s.Append("-");
  412. }
  413. return UniSetNewStr(s.IsEmpty() ? UNI_L("") : s.CStr());
  414. }
  415. #endif // SELFTEST
  416. #endif // defined(SEARCH_ENGINE_PHRASESEARCH) || defined(USE_SEARCH_ENGINE_WORDHIGHLIGHTER)