You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

pruning.py 21KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443
  1. #!/usr/bin/env python3
  2. # Copyright (c) 2014-2016 The Bitcoin Core developers
  3. # Distributed under the MIT software license, see the accompanying
  4. # file COPYING or http://www.opensource.org/licenses/mit-license.php.
  5. """Test the pruning code.
  6. WARNING:
  7. This test uses 4GB of disk space.
  8. This test takes 30 mins or more (up to 2 hours)
  9. """
  10. from test_framework.test_framework import BitcoinTestFramework
  11. from test_framework.util import *
  12. import time
  13. import os
  14. MIN_BLOCKS_TO_KEEP = 288
  15. # Rescans start at the earliest block up to 2 hours before a key timestamp, so
  16. # the manual prune RPC avoids pruning blocks in the same window to be
  17. # compatible with pruning based on key creation time.
  18. TIMESTAMP_WINDOW = 2 * 60 * 60
  19. def calc_usage(blockdir):
  20. return sum(os.path.getsize(blockdir+f) for f in os.listdir(blockdir) if os.path.isfile(blockdir+f)) / (1024. * 1024.)
  21. class PruneTest(BitcoinTestFramework):
  22. def __init__(self):
  23. super().__init__()
  24. self.setup_clean_chain = True
  25. self.num_nodes = 6
  26. # Cache for utxos, as the listunspent may take a long time later in the test
  27. self.utxo_cache_0 = []
  28. self.utxo_cache_1 = []
  29. def setup_network(self):
  30. self.nodes = []
  31. self.is_network_split = False
  32. # Create nodes 0 and 1 to mine
  33. self.nodes.append(start_node(0, self.options.tmpdir, ["-maxreceivebuffer=20000","-blockmaxsize=999000", "-checkblocks=5"], timewait=900))
  34. self.nodes.append(start_node(1, self.options.tmpdir, ["-maxreceivebuffer=20000","-blockmaxsize=999000", "-checkblocks=5"], timewait=900))
  35. # Create node 2 to test pruning
  36. self.nodes.append(start_node(2, self.options.tmpdir, ["-maxreceivebuffer=20000","-prune=550"], timewait=900))
  37. self.prunedir = self.options.tmpdir+"/node2/regtest/blocks/"
  38. # Create nodes 3 and 4 to test manual pruning (they will be re-started with manual pruning later)
  39. self.nodes.append(start_node(3, self.options.tmpdir, ["-maxreceivebuffer=20000","-blockmaxsize=999000"], timewait=900))
  40. self.nodes.append(start_node(4, self.options.tmpdir, ["-maxreceivebuffer=20000","-blockmaxsize=999000"], timewait=900))
  41. # Create nodes 5 to test wallet in prune mode, but do not connect
  42. self.nodes.append(start_node(5, self.options.tmpdir, ["-prune=550"]))
  43. # Determine default relay fee
  44. self.relayfee = self.nodes[0].getnetworkinfo()["relayfee"]
  45. connect_nodes(self.nodes[0], 1)
  46. connect_nodes(self.nodes[1], 2)
  47. connect_nodes(self.nodes[2], 0)
  48. connect_nodes(self.nodes[0], 3)
  49. connect_nodes(self.nodes[0], 4)
  50. sync_blocks(self.nodes[0:5])
  51. def create_big_chain(self):
  52. # Start by creating some coinbases we can spend later
  53. self.nodes[1].generate(200)
  54. sync_blocks(self.nodes[0:2])
  55. self.nodes[0].generate(150)
  56. # Then mine enough full blocks to create more than 550MiB of data
  57. for i in range(645):
  58. mine_large_block(self.nodes[0], self.utxo_cache_0)
  59. sync_blocks(self.nodes[0:5])
  60. def test_height_min(self):
  61. if not os.path.isfile(self.prunedir+"blk00000.dat"):
  62. raise AssertionError("blk00000.dat is missing, pruning too early")
  63. self.log.info("Success")
  64. self.log.info("Though we're already using more than 550MiB, current usage: %d" % calc_usage(self.prunedir))
  65. self.log.info("Mining 25 more blocks should cause the first block file to be pruned")
  66. # Pruning doesn't run until we're allocating another chunk, 20 full blocks past the height cutoff will ensure this
  67. for i in range(25):
  68. mine_large_block(self.nodes[0], self.utxo_cache_0)
  69. waitstart = time.time()
  70. while os.path.isfile(self.prunedir+"blk00000.dat"):
  71. time.sleep(0.1)
  72. if time.time() - waitstart > 30:
  73. raise AssertionError("blk00000.dat not pruned when it should be")
  74. self.log.info("Success")
  75. usage = calc_usage(self.prunedir)
  76. self.log.info("Usage should be below target: %d" % usage)
  77. if (usage > 550):
  78. raise AssertionError("Pruning target not being met")
  79. def create_chain_with_staleblocks(self):
  80. # Create stale blocks in manageable sized chunks
  81. self.log.info("Mine 24 (stale) blocks on Node 1, followed by 25 (main chain) block reorg from Node 0, for 12 rounds")
  82. for j in range(12):
  83. # Disconnect node 0 so it can mine a longer reorg chain without knowing about node 1's soon-to-be-stale chain
  84. # Node 2 stays connected, so it hears about the stale blocks and then reorg's when node0 reconnects
  85. # Stopping node 0 also clears its mempool, so it doesn't have node1's transactions to accidentally mine
  86. self.stop_node(0)
  87. self.nodes[0]=start_node(0, self.options.tmpdir, ["-maxreceivebuffer=20000","-blockmaxsize=999000", "-checkblocks=5"], timewait=900)
  88. # Mine 24 blocks in node 1
  89. for i in range(24):
  90. if j == 0:
  91. mine_large_block(self.nodes[1], self.utxo_cache_1)
  92. else:
  93. self.nodes[1].generate(1) #tx's already in mempool from previous disconnects
  94. # Reorg back with 25 block chain from node 0
  95. for i in range(25):
  96. mine_large_block(self.nodes[0], self.utxo_cache_0)
  97. # Create connections in the order so both nodes can see the reorg at the same time
  98. connect_nodes(self.nodes[1], 0)
  99. connect_nodes(self.nodes[2], 0)
  100. sync_blocks(self.nodes[0:3])
  101. self.log.info("Usage can be over target because of high stale rate: %d" % calc_usage(self.prunedir))
  102. def reorg_test(self):
  103. # Node 1 will mine a 300 block chain starting 287 blocks back from Node 0 and Node 2's tip
  104. # This will cause Node 2 to do a reorg requiring 288 blocks of undo data to the reorg_test chain
  105. # Reboot node 1 to clear its mempool (hopefully make the invalidate faster)
  106. # Lower the block max size so we don't keep mining all our big mempool transactions (from disconnected blocks)
  107. self.stop_node(1)
  108. self.nodes[1]=start_node(1, self.options.tmpdir, ["-maxreceivebuffer=20000","-blockmaxsize=5000", "-checkblocks=5", "-disablesafemode"], timewait=900)
  109. height = self.nodes[1].getblockcount()
  110. self.log.info("Current block height: %d" % height)
  111. invalidheight = height-287
  112. badhash = self.nodes[1].getblockhash(invalidheight)
  113. self.log.info("Invalidating block %s at height %d" % (badhash,invalidheight))
  114. self.nodes[1].invalidateblock(badhash)
  115. # We've now switched to our previously mined-24 block fork on node 1, but thats not what we want
  116. # So invalidate that fork as well, until we're on the same chain as node 0/2 (but at an ancestor 288 blocks ago)
  117. mainchainhash = self.nodes[0].getblockhash(invalidheight - 1)
  118. curhash = self.nodes[1].getblockhash(invalidheight - 1)
  119. while curhash != mainchainhash:
  120. self.nodes[1].invalidateblock(curhash)
  121. curhash = self.nodes[1].getblockhash(invalidheight - 1)
  122. assert(self.nodes[1].getblockcount() == invalidheight - 1)
  123. self.log.info("New best height: %d" % self.nodes[1].getblockcount())
  124. # Reboot node1 to clear those giant tx's from mempool
  125. self.stop_node(1)
  126. self.nodes[1]=start_node(1, self.options.tmpdir, ["-maxreceivebuffer=20000","-blockmaxsize=5000", "-checkblocks=5", "-disablesafemode"], timewait=900)
  127. self.log.info("Generating new longer chain of 300 more blocks")
  128. self.nodes[1].generate(300)
  129. self.log.info("Reconnect nodes")
  130. connect_nodes(self.nodes[0], 1)
  131. connect_nodes(self.nodes[2], 1)
  132. sync_blocks(self.nodes[0:3], timeout=120)
  133. self.log.info("Verify height on node 2: %d" % self.nodes[2].getblockcount())
  134. self.log.info("Usage possibly still high bc of stale blocks in block files: %d" % calc_usage(self.prunedir))
  135. self.log.info("Mine 220 more blocks so we have requisite history (some blocks will be big and cause pruning of previous chain)")
  136. for i in range(22):
  137. # This can be slow, so do this in multiple RPC calls to avoid
  138. # RPC timeouts.
  139. self.nodes[0].generate(10) #node 0 has many large tx's in its mempool from the disconnects
  140. sync_blocks(self.nodes[0:3], timeout=300)
  141. usage = calc_usage(self.prunedir)
  142. self.log.info("Usage should be below target: %d" % usage)
  143. if (usage > 550):
  144. raise AssertionError("Pruning target not being met")
  145. return invalidheight,badhash
  146. def reorg_back(self):
  147. # Verify that a block on the old main chain fork has been pruned away
  148. assert_raises_jsonrpc(-1, "Block not available (pruned data)", self.nodes[2].getblock, self.forkhash)
  149. self.log.info("Will need to redownload block %d" % self.forkheight)
  150. # Verify that we have enough history to reorg back to the fork point
  151. # Although this is more than 288 blocks, because this chain was written more recently
  152. # and only its other 299 small and 220 large block are in the block files after it,
  153. # its expected to still be retained
  154. self.nodes[2].getblock(self.nodes[2].getblockhash(self.forkheight))
  155. first_reorg_height = self.nodes[2].getblockcount()
  156. curchainhash = self.nodes[2].getblockhash(self.mainchainheight)
  157. self.nodes[2].invalidateblock(curchainhash)
  158. goalbestheight = self.mainchainheight
  159. goalbesthash = self.mainchainhash2
  160. # As of 0.10 the current block download logic is not able to reorg to the original chain created in
  161. # create_chain_with_stale_blocks because it doesn't know of any peer thats on that chain from which to
  162. # redownload its missing blocks.
  163. # Invalidate the reorg_test chain in node 0 as well, it can successfully switch to the original chain
  164. # because it has all the block data.
  165. # However it must mine enough blocks to have a more work chain than the reorg_test chain in order
  166. # to trigger node 2's block download logic.
  167. # At this point node 2 is within 288 blocks of the fork point so it will preserve its ability to reorg
  168. if self.nodes[2].getblockcount() < self.mainchainheight:
  169. blocks_to_mine = first_reorg_height + 1 - self.mainchainheight
  170. self.log.info("Rewind node 0 to prev main chain to mine longer chain to trigger redownload. Blocks needed: %d" % blocks_to_mine)
  171. self.nodes[0].invalidateblock(curchainhash)
  172. assert(self.nodes[0].getblockcount() == self.mainchainheight)
  173. assert(self.nodes[0].getbestblockhash() == self.mainchainhash2)
  174. goalbesthash = self.nodes[0].generate(blocks_to_mine)[-1]
  175. goalbestheight = first_reorg_height + 1
  176. self.log.info("Verify node 2 reorged back to the main chain, some blocks of which it had to redownload")
  177. waitstart = time.time()
  178. while self.nodes[2].getblockcount() < goalbestheight:
  179. time.sleep(0.1)
  180. if time.time() - waitstart > 900:
  181. raise AssertionError("Node 2 didn't reorg to proper height")
  182. assert(self.nodes[2].getbestblockhash() == goalbesthash)
  183. # Verify we can now have the data for a block previously pruned
  184. assert(self.nodes[2].getblock(self.forkhash)["height"] == self.forkheight)
  185. def manual_test(self, node_number, use_timestamp):
  186. # at this point, node has 995 blocks and has not yet run in prune mode
  187. node = self.nodes[node_number] = start_node(node_number, self.options.tmpdir, timewait=900)
  188. assert_equal(node.getblockcount(), 995)
  189. assert_raises_jsonrpc(-1, "not in prune mode", node.pruneblockchain, 500)
  190. self.stop_node(node_number)
  191. # now re-start in manual pruning mode
  192. node = self.nodes[node_number] = start_node(node_number, self.options.tmpdir, ["-prune=1"], timewait=900)
  193. assert_equal(node.getblockcount(), 995)
  194. def height(index):
  195. if use_timestamp:
  196. return node.getblockheader(node.getblockhash(index))["time"] + TIMESTAMP_WINDOW
  197. else:
  198. return index
  199. def prune(index, expected_ret=None):
  200. ret = node.pruneblockchain(height(index))
  201. # Check the return value. When use_timestamp is True, just check
  202. # that the return value is less than or equal to the expected
  203. # value, because when more than one block is generated per second,
  204. # a timestamp will not be granular enough to uniquely identify an
  205. # individual block.
  206. if expected_ret is None:
  207. expected_ret = index
  208. if use_timestamp:
  209. assert_greater_than(ret, 0)
  210. assert_greater_than(expected_ret + 1, ret)
  211. else:
  212. assert_equal(ret, expected_ret)
  213. def has_block(index):
  214. return os.path.isfile(self.options.tmpdir + "/node{}/regtest/blocks/blk{:05}.dat".format(node_number, index))
  215. # should not prune because chain tip of node 3 (995) < PruneAfterHeight (1000)
  216. assert_raises_jsonrpc(-1, "Blockchain is too short for pruning", node.pruneblockchain, height(500))
  217. # mine 6 blocks so we are at height 1001 (i.e., above PruneAfterHeight)
  218. node.generate(6)
  219. assert_equal(node.getblockchaininfo()["blocks"], 1001)
  220. # negative heights should raise an exception
  221. assert_raises_jsonrpc(-8, "Negative", node.pruneblockchain, -10)
  222. # height=100 too low to prune first block file so this is a no-op
  223. prune(100)
  224. if not has_block(0):
  225. raise AssertionError("blk00000.dat is missing when should still be there")
  226. # Does nothing
  227. node.pruneblockchain(height(0))
  228. if not has_block(0):
  229. raise AssertionError("blk00000.dat is missing when should still be there")
  230. # height=500 should prune first file
  231. prune(500)
  232. if has_block(0):
  233. raise AssertionError("blk00000.dat is still there, should be pruned by now")
  234. if not has_block(1):
  235. raise AssertionError("blk00001.dat is missing when should still be there")
  236. # height=650 should prune second file
  237. prune(650)
  238. if has_block(1):
  239. raise AssertionError("blk00001.dat is still there, should be pruned by now")
  240. # height=1000 should not prune anything more, because tip-288 is in blk00002.dat.
  241. prune(1000, 1001 - MIN_BLOCKS_TO_KEEP)
  242. if not has_block(2):
  243. raise AssertionError("blk00002.dat is still there, should be pruned by now")
  244. # advance the tip so blk00002.dat and blk00003.dat can be pruned (the last 288 blocks should now be in blk00004.dat)
  245. node.generate(288)
  246. prune(1000)
  247. if has_block(2):
  248. raise AssertionError("blk00002.dat is still there, should be pruned by now")
  249. if has_block(3):
  250. raise AssertionError("blk00003.dat is still there, should be pruned by now")
  251. # stop node, start back up with auto-prune at 550MB, make sure still runs
  252. self.stop_node(node_number)
  253. self.nodes[node_number] = start_node(node_number, self.options.tmpdir, ["-prune=550"], timewait=900)
  254. self.log.info("Success")
  255. def wallet_test(self):
  256. # check that the pruning node's wallet is still in good shape
  257. self.log.info("Stop and start pruning node to trigger wallet rescan")
  258. self.stop_node(2)
  259. start_node(2, self.options.tmpdir, ["-prune=550"])
  260. self.log.info("Success")
  261. # check that wallet loads loads successfully when restarting a pruned node after IBD.
  262. # this was reported to fail in #7494.
  263. self.log.info("Syncing node 5 to test wallet")
  264. connect_nodes(self.nodes[0], 5)
  265. nds = [self.nodes[0], self.nodes[5]]
  266. sync_blocks(nds, wait=5, timeout=300)
  267. self.stop_node(5) #stop and start to trigger rescan
  268. start_node(5, self.options.tmpdir, ["-prune=550"])
  269. self.log.info("Success")
  270. def run_test(self):
  271. self.log.info("Warning! This test requires 4GB of disk space and takes over 30 mins (up to 2 hours)")
  272. self.log.info("Mining a big blockchain of 995 blocks")
  273. self.create_big_chain()
  274. # Chain diagram key:
  275. # * blocks on main chain
  276. # +,&,$,@ blocks on other forks
  277. # X invalidated block
  278. # N1 Node 1
  279. #
  280. # Start by mining a simple chain that all nodes have
  281. # N0=N1=N2 **...*(995)
  282. # stop manual-pruning node with 995 blocks
  283. self.stop_node(3)
  284. self.stop_node(4)
  285. self.log.info("Check that we haven't started pruning yet because we're below PruneAfterHeight")
  286. self.test_height_min()
  287. # Extend this chain past the PruneAfterHeight
  288. # N0=N1=N2 **...*(1020)
  289. self.log.info("Check that we'll exceed disk space target if we have a very high stale block rate")
  290. self.create_chain_with_staleblocks()
  291. # Disconnect N0
  292. # And mine a 24 block chain on N1 and a separate 25 block chain on N0
  293. # N1=N2 **...*+...+(1044)
  294. # N0 **...**...**(1045)
  295. #
  296. # reconnect nodes causing reorg on N1 and N2
  297. # N1=N2 **...*(1020) *...**(1045)
  298. # \
  299. # +...+(1044)
  300. #
  301. # repeat this process until you have 12 stale forks hanging off the
  302. # main chain on N1 and N2
  303. # N0 *************************...***************************(1320)
  304. #
  305. # N1=N2 **...*(1020) *...**(1045) *.. ..**(1295) *...**(1320)
  306. # \ \ \
  307. # +...+(1044) &.. $...$(1319)
  308. # Save some current chain state for later use
  309. self.mainchainheight = self.nodes[2].getblockcount() #1320
  310. self.mainchainhash2 = self.nodes[2].getblockhash(self.mainchainheight)
  311. self.log.info("Check that we can survive a 288 block reorg still")
  312. (self.forkheight,self.forkhash) = self.reorg_test() #(1033, )
  313. # Now create a 288 block reorg by mining a longer chain on N1
  314. # First disconnect N1
  315. # Then invalidate 1033 on main chain and 1032 on fork so height is 1032 on main chain
  316. # N1 **...*(1020) **...**(1032)X..
  317. # \
  318. # ++...+(1031)X..
  319. #
  320. # Now mine 300 more blocks on N1
  321. # N1 **...*(1020) **...**(1032) @@...@(1332)
  322. # \ \
  323. # \ X...
  324. # \ \
  325. # ++...+(1031)X.. ..
  326. #
  327. # Reconnect nodes and mine 220 more blocks on N1
  328. # N1 **...*(1020) **...**(1032) @@...@@@(1552)
  329. # \ \
  330. # \ X...
  331. # \ \
  332. # ++...+(1031)X.. ..
  333. #
  334. # N2 **...*(1020) **...**(1032) @@...@@@(1552)
  335. # \ \
  336. # \ *...**(1320)
  337. # \ \
  338. # ++...++(1044) ..
  339. #
  340. # N0 ********************(1032) @@...@@@(1552)
  341. # \
  342. # *...**(1320)
  343. self.log.info("Test that we can rerequest a block we previously pruned if needed for a reorg")
  344. self.reorg_back()
  345. # Verify that N2 still has block 1033 on current chain (@), but not on main chain (*)
  346. # Invalidate 1033 on current chain (@) on N2 and we should be able to reorg to
  347. # original main chain (*), but will require redownload of some blocks
  348. # In order to have a peer we think we can download from, must also perform this invalidation
  349. # on N0 and mine a new longest chain to trigger.
  350. # Final result:
  351. # N0 ********************(1032) **...****(1553)
  352. # \
  353. # X@...@@@(1552)
  354. #
  355. # N2 **...*(1020) **...**(1032) **...****(1553)
  356. # \ \
  357. # \ X@...@@@(1552)
  358. # \
  359. # +..
  360. #
  361. # N1 doesn't change because 1033 on main chain (*) is invalid
  362. self.log.info("Test manual pruning with block indices")
  363. self.manual_test(3, use_timestamp=False)
  364. self.log.info("Test manual pruning with timestamps")
  365. self.manual_test(4, use_timestamp=True)
  366. self.log.info("Test wallet re-scan")
  367. self.wallet_test()
  368. self.log.info("Done")
  369. if __name__ == '__main__':
  370. PruneTest().main()