You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

cuckoocache.h 18KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481
  1. // Copyright (c) 2016 Jeremy Rubin
  2. // Distributed under the MIT software license, see the accompanying
  3. // file COPYING or http://www.opensource.org/licenses/mit-license.php.
  4. #ifndef _STARWELS_CUCKOOCACHE_H_
  5. #define _STARWELS_CUCKOOCACHE_H_
  6. #include <array>
  7. #include <algorithm>
  8. #include <atomic>
  9. #include <cstring>
  10. #include <cmath>
  11. #include <memory>
  12. #include <vector>
  13. /** namespace CuckooCache provides high performance cache primitives
  14. *
  15. * Summary:
  16. *
  17. * 1) bit_packed_atomic_flags is bit-packed atomic flags for garbage collection
  18. *
  19. * 2) cache is a cache which is performant in memory usage and lookup speed. It
  20. * is lockfree for erase operations. Elements are lazily erased on the next
  21. * insert.
  22. */
  23. namespace CuckooCache
  24. {
  25. /** bit_packed_atomic_flags implements a container for garbage collection flags
  26. * that is only thread unsafe on calls to setup. This class bit-packs collection
  27. * flags for memory efficiency.
  28. *
  29. * All operations are std::memory_order_relaxed so external mechanisms must
  30. * ensure that writes and reads are properly synchronized.
  31. *
  32. * On setup(n), all bits up to n are marked as collected.
  33. *
  34. * Under the hood, because it is an 8-bit type, it makes sense to use a multiple
  35. * of 8 for setup, but it will be safe if that is not the case as well.
  36. *
  37. */
  38. class bit_packed_atomic_flags
  39. {
  40. std::unique_ptr<std::atomic<uint8_t>[]> mem;
  41. public:
  42. /** No default constructor as there must be some size */
  43. bit_packed_atomic_flags() = delete;
  44. /**
  45. * bit_packed_atomic_flags constructor creates memory to sufficiently
  46. * keep track of garbage collection information for size entries.
  47. *
  48. * @param size the number of elements to allocate space for
  49. *
  50. * @post bit_set, bit_unset, and bit_is_set function properly forall x. x <
  51. * size
  52. * @post All calls to bit_is_set (without subsequent bit_unset) will return
  53. * true.
  54. */
  55. bit_packed_atomic_flags(uint32_t size)
  56. {
  57. // pad out the size if needed
  58. size = (size + 7) / 8;
  59. mem.reset(new std::atomic<uint8_t>[size]);
  60. for (uint32_t i = 0; i < size; ++i)
  61. mem[i].store(0xFF);
  62. };
  63. /** setup marks all entries and ensures that bit_packed_atomic_flags can store
  64. * at least size entries
  65. *
  66. * @param b the number of elements to allocate space for
  67. * @post bit_set, bit_unset, and bit_is_set function properly forall x. x <
  68. * b
  69. * @post All calls to bit_is_set (without subsequent bit_unset) will return
  70. * true.
  71. */
  72. inline void setup(uint32_t b)
  73. {
  74. bit_packed_atomic_flags d(b);
  75. std::swap(mem, d.mem);
  76. }
  77. /** bit_set sets an entry as discardable.
  78. *
  79. * @param s the index of the entry to bit_set.
  80. * @post immediately subsequent call (assuming proper external memory
  81. * ordering) to bit_is_set(s) == true.
  82. *
  83. */
  84. inline void bit_set(uint32_t s)
  85. {
  86. mem[s >> 3].fetch_or(1 << (s & 7), std::memory_order_relaxed);
  87. }
  88. /** bit_unset marks an entry as something that should not be overwritten
  89. *
  90. * @param s the index of the entry to bit_unset.
  91. * @post immediately subsequent call (assuming proper external memory
  92. * ordering) to bit_is_set(s) == false.
  93. */
  94. inline void bit_unset(uint32_t s)
  95. {
  96. mem[s >> 3].fetch_and(~(1 << (s & 7)), std::memory_order_relaxed);
  97. }
  98. /** bit_is_set queries the table for discardability at s
  99. *
  100. * @param s the index of the entry to read.
  101. * @returns if the bit at index s was set.
  102. * */
  103. inline bool bit_is_set(uint32_t s) const
  104. {
  105. return (1 << (s & 7)) & mem[s >> 3].load(std::memory_order_relaxed);
  106. }
  107. };
  108. /** cache implements a cache with properties similar to a cuckoo-set
  109. *
  110. * The cache is able to hold up to (~(uint32_t)0) - 1 elements.
  111. *
  112. * Read Operations:
  113. * - contains(*, false)
  114. *
  115. * Read+Erase Operations:
  116. * - contains(*, true)
  117. *
  118. * Erase Operations:
  119. * - allow_erase()
  120. *
  121. * Write Operations:
  122. * - setup()
  123. * - setup_bytes()
  124. * - insert()
  125. * - please_keep()
  126. *
  127. * Synchronization Free Operations:
  128. * - invalid()
  129. * - compute_hashes()
  130. *
  131. * User Must Guarantee:
  132. *
  133. * 1) Write Requires synchronized access (e.g., a lock)
  134. * 2) Read Requires no concurrent Write, synchronized with the last insert.
  135. * 3) Erase requires no concurrent Write, synchronized with last insert.
  136. * 4) An Erase caller must release all memory before allowing a new Writer.
  137. *
  138. *
  139. * Note on function names:
  140. * - The name "allow_erase" is used because the real discard happens later.
  141. * - The name "please_keep" is used because elements may be erased anyways on insert.
  142. *
  143. * @tparam Element should be a movable and copyable type
  144. * @tparam Hash should be a function/callable which takes a template parameter
  145. * hash_select and an Element and extracts a hash from it. Should return
  146. * high-entropy uint32_t hashes for `Hash h; h<0>(e) ... h<7>(e)`.
  147. */
  148. template <typename Element, typename Hash>
  149. class cache
  150. {
  151. private:
  152. /** table stores all the elements */
  153. std::vector<Element> table;
  154. /** size stores the total available slots in the hash table */
  155. uint32_t size;
  156. /** The bit_packed_atomic_flags array is marked mutable because we want
  157. * garbage collection to be allowed to occur from const methods */
  158. mutable bit_packed_atomic_flags collection_flags;
  159. /** epoch_flags tracks how recently an element was inserted into
  160. * the cache. true denotes recent, false denotes not-recent. See insert()
  161. * method for full semantics.
  162. */
  163. mutable std::vector<bool> epoch_flags;
  164. /** epoch_heuristic_counter is used to determine when an epoch might be aged
  165. * & an expensive scan should be done. epoch_heuristic_counter is
  166. * decremented on insert and reset to the new number of inserts which would
  167. * cause the epoch to reach epoch_size when it reaches zero.
  168. */
  169. uint32_t epoch_heuristic_counter;
  170. /** epoch_size is set to be the number of elements supposed to be in a
  171. * epoch. When the number of non-erased elements in an epoch
  172. * exceeds epoch_size, a new epoch should be started and all
  173. * current entries demoted. epoch_size is set to be 45% of size because
  174. * we want to keep load around 90%, and we support 3 epochs at once --
  175. * one "dead" which has been erased, one "dying" which has been marked to be
  176. * erased next, and one "living" which new inserts add to.
  177. */
  178. uint32_t epoch_size;
  179. /** depth_limit determines how many elements insert should try to replace.
  180. * Should be set to log2(n)*/
  181. uint8_t depth_limit;
  182. /** hash_function is a const instance of the hash function. It cannot be
  183. * static or initialized at call time as it may have internal state (such as
  184. * a nonce).
  185. * */
  186. const Hash hash_function;
  187. /** compute_hashes is convenience for not having to write out this
  188. * expression everywhere we use the hash values of an Element.
  189. *
  190. * We need to map the 32-bit input hash onto a hash bucket in a range [0, size) in a
  191. * manner which preserves as much of the hash's uniformity as possible. Ideally
  192. * this would be done by bitmasking but the size is usually not a power of two.
  193. *
  194. * The naive approach would be to use a mod -- which isn't perfectly uniform but so
  195. * long as the hash is much larger than size it is not that bad. Unfortunately,
  196. * mod/division is fairly slow on ordinary microprocessors (e.g. 90-ish cycles on
  197. * haswell, ARM doesn't even have an instruction for it.); when the divisor is a
  198. * constant the compiler will do clever tricks to turn it into a multiply+add+shift,
  199. * but size is a run-time value so the compiler can't do that here.
  200. *
  201. * One option would be to implement the same trick the compiler uses and compute the
  202. * constants for exact division based on the size, as described in "{N}-bit Unsigned
  203. * Division via {N}-bit Multiply-Add" by Arch D. Robison in 2005. But that code is
  204. * somewhat complicated and the result is still slower than other options:
  205. *
  206. * Instead we treat the 32-bit random number as a Q32 fixed-point number in the range
  207. * [0,1) and simply multiply it by the size. Then we just shift the result down by
  208. * 32-bits to get our bucket number. The results has non-uniformity the same as a
  209. * mod, but it is much faster to compute. More about this technique can be found at
  210. * http://lemire.me/blog/2016/06/27/a-fast-alternative-to-the-modulo-reduction/
  211. *
  212. * The resulting non-uniformity is also more equally distributed which would be
  213. * advantageous for something like linear probing, though it shouldn't matter
  214. * one way or the other for a cuckoo table.
  215. *
  216. * The primary disadvantage of this approach is increased intermediate precision is
  217. * required but for a 32-bit random number we only need the high 32 bits of a
  218. * 32*32->64 multiply, which means the operation is reasonably fast even on a
  219. * typical 32-bit processor.
  220. *
  221. * @param e the element whose hashes will be returned
  222. * @returns std::array<uint32_t, 8> of deterministic hashes derived from e
  223. */
  224. inline std::array<uint32_t, 8> compute_hashes(const Element& e) const
  225. {
  226. return {{(uint32_t)((hash_function.template operator()<0>(e) * (uint64_t)size) >> 32),
  227. (uint32_t)((hash_function.template operator()<1>(e) * (uint64_t)size) >> 32),
  228. (uint32_t)((hash_function.template operator()<2>(e) * (uint64_t)size) >> 32),
  229. (uint32_t)((hash_function.template operator()<3>(e) * (uint64_t)size) >> 32),
  230. (uint32_t)((hash_function.template operator()<4>(e) * (uint64_t)size) >> 32),
  231. (uint32_t)((hash_function.template operator()<5>(e) * (uint64_t)size) >> 32),
  232. (uint32_t)((hash_function.template operator()<6>(e) * (uint64_t)size) >> 32),
  233. (uint32_t)((hash_function.template operator()<7>(e) * (uint64_t)size) >> 32)}};
  234. }
  235. /* end
  236. * @returns a constexpr index that can never be inserted to */
  237. constexpr uint32_t invalid() const
  238. {
  239. return ~(uint32_t)0;
  240. }
  241. /** allow_erase marks the element at index n as discardable. Threadsafe
  242. * without any concurrent insert.
  243. * @param n the index to allow erasure of
  244. */
  245. inline void allow_erase(uint32_t n) const
  246. {
  247. collection_flags.bit_set(n);
  248. }
  249. /** please_keep marks the element at index n as an entry that should be kept.
  250. * Threadsafe without any concurrent insert.
  251. * @param n the index to prioritize keeping
  252. */
  253. inline void please_keep(uint32_t n) const
  254. {
  255. collection_flags.bit_unset(n);
  256. }
  257. /** epoch_check handles the changing of epochs for elements stored in the
  258. * cache. epoch_check should be run before every insert.
  259. *
  260. * First, epoch_check decrements and checks the cheap heuristic, and then does
  261. * a more expensive scan if the cheap heuristic runs out. If the expensive
  262. * scan succeeds, the epochs are aged and old elements are allow_erased. The
  263. * cheap heuristic is reset to retrigger after the worst case growth of the
  264. * current epoch's elements would exceed the epoch_size.
  265. */
  266. void epoch_check()
  267. {
  268. if (epoch_heuristic_counter != 0) {
  269. --epoch_heuristic_counter;
  270. return;
  271. }
  272. // count the number of elements from the latest epoch which
  273. // have not been erased.
  274. uint32_t epoch_unused_count = 0;
  275. for (uint32_t i = 0; i < size; ++i)
  276. epoch_unused_count += epoch_flags[i] &&
  277. !collection_flags.bit_is_set(i);
  278. // If there are more non-deleted entries in the current epoch than the
  279. // epoch size, then allow_erase on all elements in the old epoch (marked
  280. // false) and move all elements in the current epoch to the old epoch
  281. // but do not call allow_erase on their indices.
  282. if (epoch_unused_count >= epoch_size) {
  283. for (uint32_t i = 0; i < size; ++i)
  284. if (epoch_flags[i])
  285. epoch_flags[i] = false;
  286. else
  287. allow_erase(i);
  288. epoch_heuristic_counter = epoch_size;
  289. } else
  290. // reset the epoch_heuristic_counter to next do a scan when worst
  291. // case behavior (no intermittent erases) would exceed epoch size,
  292. // with a reasonable minimum scan size.
  293. // Ordinarily, we would have to sanity check std::min(epoch_size,
  294. // epoch_unused_count), but we already know that `epoch_unused_count
  295. // < epoch_size` in this branch
  296. epoch_heuristic_counter = std::max(1u, std::max(epoch_size / 16,
  297. epoch_size - epoch_unused_count));
  298. }
  299. public:
  300. /** You must always construct a cache with some elements via a subsequent
  301. * call to setup or setup_bytes, otherwise operations may segfault.
  302. */
  303. cache() : table(), size(), collection_flags(0), epoch_flags(),
  304. epoch_heuristic_counter(), epoch_size(), depth_limit(0), hash_function()
  305. {
  306. }
  307. /** setup initializes the container to store no more than new_size
  308. * elements.
  309. *
  310. * setup should only be called once.
  311. *
  312. * @param new_size the desired number of elements to store
  313. * @returns the maximum number of elements storable
  314. **/
  315. uint32_t setup(uint32_t new_size)
  316. {
  317. // depth_limit must be at least one otherwise errors can occur.
  318. depth_limit = static_cast<uint8_t>(std::log2(static_cast<float>(std::max((uint32_t)2, new_size))));
  319. size = std::max<uint32_t>(2, new_size);
  320. table.resize(size);
  321. collection_flags.setup(size);
  322. epoch_flags.resize(size);
  323. // Set to 45% as described above
  324. epoch_size = std::max((uint32_t)1, (45 * size) / 100);
  325. // Initially set to wait for a whole epoch
  326. epoch_heuristic_counter = epoch_size;
  327. return size;
  328. }
  329. /** setup_bytes is a convenience function which accounts for internal memory
  330. * usage when deciding how many elements to store. It isn't perfect because
  331. * it doesn't account for any overhead (struct size, MallocUsage, collection
  332. * and epoch flags). This was done to simplify selecting a power of two
  333. * size. In the expected use case, an extra two bits per entry should be
  334. * negligible compared to the size of the elements.
  335. *
  336. * @param bytes the approximate number of bytes to use for this data
  337. * structure.
  338. * @returns the maximum number of elements storable (see setup()
  339. * documentation for more detail)
  340. */
  341. uint32_t setup_bytes(size_t bytes)
  342. {
  343. return setup(bytes/sizeof(Element));
  344. }
  345. /** insert loops at most depth_limit times trying to insert a hash
  346. * at various locations in the table via a variant of the Cuckoo Algorithm
  347. * with eight hash locations.
  348. *
  349. * It drops the last tried element if it runs out of depth before
  350. * encountering an open slot.
  351. *
  352. * Thus
  353. *
  354. * insert(x);
  355. * return contains(x, false);
  356. *
  357. * is not guaranteed to return true.
  358. *
  359. * @param e the element to insert
  360. * @post one of the following: All previously inserted elements and e are
  361. * now in the table, one previously inserted element is evicted from the
  362. * table, the entry attempted to be inserted is evicted.
  363. *
  364. */
  365. inline void insert(Element e)
  366. {
  367. epoch_check();
  368. uint32_t last_loc = invalid();
  369. bool last_epoch = true;
  370. std::array<uint32_t, 8> locs = compute_hashes(e);
  371. // Make sure we have not already inserted this element
  372. // If we have, make sure that it does not get deleted
  373. for (uint32_t loc : locs)
  374. if (table[loc] == e) {
  375. please_keep(loc);
  376. epoch_flags[loc] = last_epoch;
  377. return;
  378. }
  379. for (uint8_t depth = 0; depth < depth_limit; ++depth) {
  380. // First try to insert to an empty slot, if one exists
  381. for (uint32_t loc : locs) {
  382. if (!collection_flags.bit_is_set(loc))
  383. continue;
  384. table[loc] = std::move(e);
  385. please_keep(loc);
  386. epoch_flags[loc] = last_epoch;
  387. return;
  388. }
  389. /** Swap with the element at the location that was
  390. * not the last one looked at. Example:
  391. *
  392. * 1) On first iteration, last_loc == invalid(), find returns last, so
  393. * last_loc defaults to locs[0].
  394. * 2) On further iterations, where last_loc == locs[k], last_loc will
  395. * go to locs[k+1 % 8], i.e., next of the 8 indices wrapping around
  396. * to 0 if needed.
  397. *
  398. * This prevents moving the element we just put in.
  399. *
  400. * The swap is not a move -- we must switch onto the evicted element
  401. * for the next iteration.
  402. */
  403. last_loc = locs[(1 + (std::find(locs.begin(), locs.end(), last_loc) - locs.begin())) & 7];
  404. std::swap(table[last_loc], e);
  405. // Can't std::swap a std::vector<bool>::reference and a bool&.
  406. bool epoch = last_epoch;
  407. last_epoch = epoch_flags[last_loc];
  408. epoch_flags[last_loc] = epoch;
  409. // Recompute the locs -- unfortunately happens one too many times!
  410. locs = compute_hashes(e);
  411. }
  412. }
  413. /* contains iterates through the hash locations for a given element
  414. * and checks to see if it is present.
  415. *
  416. * contains does not check garbage collected state (in other words,
  417. * garbage is only collected when the space is needed), so:
  418. *
  419. * insert(x);
  420. * if (contains(x, true))
  421. * return contains(x, false);
  422. * else
  423. * return true;
  424. *
  425. * executed on a single thread will always return true!
  426. *
  427. * This is a great property for re-org performance for example.
  428. *
  429. * contains returns a bool set true if the element was found.
  430. *
  431. * @param e the element to check
  432. * @param erase
  433. *
  434. * @post if erase is true and the element is found, then the garbage collect
  435. * flag is set
  436. * @returns true if the element is found, false otherwise
  437. */
  438. inline bool contains(const Element& e, const bool erase) const
  439. {
  440. std::array<uint32_t, 8> locs = compute_hashes(e);
  441. for (uint32_t loc : locs)
  442. if (table[loc] == e) {
  443. if (erase)
  444. allow_erase(loc);
  445. return true;
  446. }
  447. return false;
  448. }
  449. };
  450. } // namespace CuckooCache
  451. #endif