You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

p2p-compactblocks.py 37KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843
  1. #!/usr/bin/env python3
  2. # Copyright (c) 2016 The Bitcoin Core developers
  3. # Distributed under the MIT software license, see the accompanying
  4. # file COPYING or http://www.opensource.org/licenses/mit-license.php.
  5. from test_framework.mininode import *
  6. from test_framework.test_framework import BitcoinTestFramework
  7. from test_framework.util import *
  8. from test_framework.blocktools import create_block, create_coinbase
  9. from test_framework.siphash import siphash256
  10. from test_framework.script import CScript, OP_TRUE
  11. '''
  12. CompactBlocksTest -- test compact blocks (BIP 152)
  13. Version 1 compact blocks are pre-segwit (txids)
  14. Version 2 compact blocks are post-segwit (wtxids)
  15. '''
  16. # TestNode: A peer we use to send messages to bitcoind, and store responses.
  17. class TestNode(SingleNodeConnCB):
  18. def __init__(self):
  19. SingleNodeConnCB.__init__(self)
  20. self.last_sendcmpct = []
  21. self.last_headers = None
  22. self.last_inv = None
  23. self.last_cmpctblock = None
  24. self.block_announced = False
  25. self.last_getdata = None
  26. self.last_getblocktxn = None
  27. self.last_block = None
  28. self.last_blocktxn = None
  29. # Store the hashes of blocks we've seen announced.
  30. # This is for synchronizing the p2p message traffic,
  31. # so we can eg wait until a particular block is announced.
  32. self.set_announced_blockhashes = set()
  33. def on_sendcmpct(self, conn, message):
  34. self.last_sendcmpct.append(message)
  35. def on_block(self, conn, message):
  36. self.last_block = message
  37. def on_cmpctblock(self, conn, message):
  38. self.last_cmpctblock = message
  39. self.block_announced = True
  40. self.last_cmpctblock.header_and_shortids.header.calc_sha256()
  41. self.set_announced_blockhashes.add(self.last_cmpctblock.header_and_shortids.header.sha256)
  42. def on_headers(self, conn, message):
  43. self.last_headers = message
  44. self.block_announced = True
  45. for x in self.last_headers.headers:
  46. x.calc_sha256()
  47. self.set_announced_blockhashes.add(x.sha256)
  48. def on_inv(self, conn, message):
  49. self.last_inv = message
  50. for x in self.last_inv.inv:
  51. if x.type == 2:
  52. self.block_announced = True
  53. self.set_announced_blockhashes.add(x.hash)
  54. def on_getdata(self, conn, message):
  55. self.last_getdata = message
  56. def on_getblocktxn(self, conn, message):
  57. self.last_getblocktxn = message
  58. def on_blocktxn(self, conn, message):
  59. self.last_blocktxn = message
  60. # Requires caller to hold mininode_lock
  61. def received_block_announcement(self):
  62. return self.block_announced
  63. def clear_block_announcement(self):
  64. with mininode_lock:
  65. self.block_announced = False
  66. self.last_inv = None
  67. self.last_headers = None
  68. self.last_cmpctblock = None
  69. def get_headers(self, locator, hashstop):
  70. msg = msg_getheaders()
  71. msg.locator.vHave = locator
  72. msg.hashstop = hashstop
  73. self.connection.send_message(msg)
  74. def send_header_for_blocks(self, new_blocks):
  75. headers_message = msg_headers()
  76. headers_message.headers = [CBlockHeader(b) for b in new_blocks]
  77. self.send_message(headers_message)
  78. def request_headers_and_sync(self, locator, hashstop=0):
  79. self.clear_block_announcement()
  80. self.get_headers(locator, hashstop)
  81. assert(wait_until(self.received_block_announcement, timeout=30))
  82. assert(self.received_block_announcement())
  83. self.clear_block_announcement()
  84. # Block until a block announcement for a particular block hash is
  85. # received.
  86. def wait_for_block_announcement(self, block_hash, timeout=30):
  87. def received_hash():
  88. return (block_hash in self.set_announced_blockhashes)
  89. return wait_until(received_hash, timeout=timeout)
  90. class CompactBlocksTest(BitcoinTestFramework):
  91. def __init__(self):
  92. super().__init__()
  93. self.setup_clean_chain = True
  94. # Node0 = pre-segwit, node1 = segwit-aware
  95. self.num_nodes = 2
  96. self.utxos = []
  97. def setup_network(self):
  98. self.nodes = []
  99. # Start up node0 to be a version 1, pre-segwit node.
  100. self.nodes = start_nodes(self.num_nodes, self.options.tmpdir,
  101. [["-debug", "-logtimemicros=1", "-bip9params=segwit:0:0"],
  102. ["-debug", "-logtimemicros", "-txindex"]])
  103. connect_nodes(self.nodes[0], 1)
  104. def build_block_on_tip(self, node):
  105. height = node.getblockcount()
  106. tip = node.getbestblockhash()
  107. mtp = node.getblockheader(tip)['mediantime']
  108. block = create_block(int(tip, 16), create_coinbase(height + 1), mtp + 1)
  109. block.solve()
  110. return block
  111. # Create 10 more anyone-can-spend utxo's for testing.
  112. def make_utxos(self):
  113. # Doesn't matter which node we use, just use node0.
  114. block = self.build_block_on_tip(self.nodes[0])
  115. self.test_node.send_and_ping(msg_block(block))
  116. assert(int(self.nodes[0].getbestblockhash(), 16) == block.sha256)
  117. self.nodes[0].generate(100)
  118. total_value = block.vtx[0].vout[0].nValue
  119. out_value = total_value // 10
  120. tx = CTransaction()
  121. tx.vin.append(CTxIn(COutPoint(block.vtx[0].sha256, 0), b''))
  122. for i in range(10):
  123. tx.vout.append(CTxOut(out_value, CScript([OP_TRUE])))
  124. tx.rehash()
  125. block2 = self.build_block_on_tip(self.nodes[0])
  126. block2.vtx.append(tx)
  127. block2.hashMerkleRoot = block2.calc_merkle_root()
  128. block2.solve()
  129. self.test_node.send_and_ping(msg_block(block2))
  130. assert_equal(int(self.nodes[0].getbestblockhash(), 16), block2.sha256)
  131. self.utxos.extend([[tx.sha256, i, out_value] for i in range(10)])
  132. return
  133. # Test "sendcmpct" (between peers preferring the same version):
  134. # - No compact block announcements unless sendcmpct is sent.
  135. # - If sendcmpct is sent with version > preferred_version, the message is ignored.
  136. # - If sendcmpct is sent with boolean 0, then block announcements are not
  137. # made with compact blocks.
  138. # - If sendcmpct is then sent with boolean 1, then new block announcements
  139. # are made with compact blocks.
  140. # If old_node is passed in, request compact blocks with version=preferred-1
  141. # and verify that it receives block announcements via compact block.
  142. def test_sendcmpct(self, node, test_node, preferred_version, old_node=None):
  143. # Make sure we get a SENDCMPCT message from our peer
  144. def received_sendcmpct():
  145. return (len(test_node.last_sendcmpct) > 0)
  146. got_message = wait_until(received_sendcmpct, timeout=30)
  147. assert(received_sendcmpct())
  148. assert(got_message)
  149. with mininode_lock:
  150. # Check that the first version received is the preferred one
  151. assert_equal(test_node.last_sendcmpct[0].version, preferred_version)
  152. # And that we receive versions down to 1.
  153. assert_equal(test_node.last_sendcmpct[-1].version, 1)
  154. test_node.last_sendcmpct = []
  155. tip = int(node.getbestblockhash(), 16)
  156. def check_announcement_of_new_block(node, peer, predicate):
  157. peer.clear_block_announcement()
  158. node.generate(1)
  159. got_message = wait_until(lambda: peer.block_announced, timeout=30)
  160. assert(peer.block_announced)
  161. assert(got_message)
  162. with mininode_lock:
  163. assert(predicate(peer))
  164. # We shouldn't get any block announcements via cmpctblock yet.
  165. check_announcement_of_new_block(node, test_node, lambda p: p.last_cmpctblock is None)
  166. # Try one more time, this time after requesting headers.
  167. test_node.request_headers_and_sync(locator=[tip])
  168. check_announcement_of_new_block(node, test_node, lambda p: p.last_cmpctblock is None and p.last_inv is not None)
  169. # Test a few ways of using sendcmpct that should NOT
  170. # result in compact block announcements.
  171. # Before each test, sync the headers chain.
  172. test_node.request_headers_and_sync(locator=[tip])
  173. # Now try a SENDCMPCT message with too-high version
  174. sendcmpct = msg_sendcmpct()
  175. sendcmpct.version = preferred_version+1
  176. sendcmpct.announce = True
  177. test_node.send_and_ping(sendcmpct)
  178. check_announcement_of_new_block(node, test_node, lambda p: p.last_cmpctblock is None)
  179. # Headers sync before next test.
  180. test_node.request_headers_and_sync(locator=[tip])
  181. # Now try a SENDCMPCT message with valid version, but announce=False
  182. sendcmpct.version = preferred_version
  183. sendcmpct.announce = False
  184. test_node.send_and_ping(sendcmpct)
  185. check_announcement_of_new_block(node, test_node, lambda p: p.last_cmpctblock is None)
  186. # Headers sync before next test.
  187. test_node.request_headers_and_sync(locator=[tip])
  188. # Finally, try a SENDCMPCT message with announce=True
  189. sendcmpct.version = preferred_version
  190. sendcmpct.announce = True
  191. test_node.send_and_ping(sendcmpct)
  192. check_announcement_of_new_block(node, test_node, lambda p: p.last_cmpctblock is not None)
  193. # Try one more time (no headers sync should be needed!)
  194. check_announcement_of_new_block(node, test_node, lambda p: p.last_cmpctblock is not None)
  195. # Try one more time, after turning on sendheaders
  196. test_node.send_and_ping(msg_sendheaders())
  197. check_announcement_of_new_block(node, test_node, lambda p: p.last_cmpctblock is not None)
  198. # Try one more time, after sending a version-1, announce=false message.
  199. sendcmpct.version = preferred_version-1
  200. sendcmpct.announce = False
  201. test_node.send_and_ping(sendcmpct)
  202. check_announcement_of_new_block(node, test_node, lambda p: p.last_cmpctblock is not None)
  203. # Now turn off announcements
  204. sendcmpct.version = preferred_version
  205. sendcmpct.announce = False
  206. test_node.send_and_ping(sendcmpct)
  207. check_announcement_of_new_block(node, test_node, lambda p: p.last_cmpctblock is None and p.last_headers is not None)
  208. if old_node is not None:
  209. # Verify that a peer using an older protocol version can receive
  210. # announcements from this node.
  211. sendcmpct.version = preferred_version-1
  212. sendcmpct.announce = True
  213. old_node.send_and_ping(sendcmpct)
  214. # Header sync
  215. old_node.request_headers_and_sync(locator=[tip])
  216. check_announcement_of_new_block(node, old_node, lambda p: p.last_cmpctblock is not None)
  217. # This test actually causes bitcoind to (reasonably!) disconnect us, so do this last.
  218. def test_invalid_cmpctblock_message(self):
  219. self.nodes[0].generate(101)
  220. block = self.build_block_on_tip(self.nodes[0])
  221. cmpct_block = P2PHeaderAndShortIDs()
  222. cmpct_block.header = CBlockHeader(block)
  223. cmpct_block.prefilled_txn_length = 1
  224. # This index will be too high
  225. prefilled_txn = PrefilledTransaction(1, block.vtx[0])
  226. cmpct_block.prefilled_txn = [prefilled_txn]
  227. self.test_node.send_and_ping(msg_cmpctblock(cmpct_block))
  228. assert(int(self.nodes[0].getbestblockhash(), 16) == block.hashPrevBlock)
  229. # Compare the generated shortids to what we expect based on BIP 152, given
  230. # bitcoind's choice of nonce.
  231. def test_compactblock_construction(self, node, test_node, version, use_witness_address):
  232. # Generate a bunch of transactions.
  233. node.generate(101)
  234. num_transactions = 25
  235. address = node.getnewaddress()
  236. if use_witness_address:
  237. # Want at least one segwit spend, so move all funds to
  238. # a witness address.
  239. address = node.addwitnessaddress(address)
  240. value_to_send = node.getbalance()
  241. node.sendtoaddress(address, satoshi_round(value_to_send-Decimal(0.1)))
  242. node.generate(1)
  243. segwit_tx_generated = False
  244. for i in range(num_transactions):
  245. txid = node.sendtoaddress(address, 0.1)
  246. hex_tx = node.gettransaction(txid)["hex"]
  247. tx = FromHex(CTransaction(), hex_tx)
  248. if not tx.wit.is_null():
  249. segwit_tx_generated = True
  250. if use_witness_address:
  251. assert(segwit_tx_generated) # check that our test is not broken
  252. # Wait until we've seen the block announcement for the resulting tip
  253. tip = int(self.nodes[0].getbestblockhash(), 16)
  254. assert(self.test_node.wait_for_block_announcement(tip))
  255. # Now mine a block, and look at the resulting compact block.
  256. test_node.clear_block_announcement()
  257. block_hash = int(node.generate(1)[0], 16)
  258. # Store the raw block in our internal format.
  259. block = FromHex(CBlock(), node.getblock("%02x" % block_hash, False))
  260. [tx.calc_sha256() for tx in block.vtx]
  261. block.rehash()
  262. # Don't care which type of announcement came back for this test; just
  263. # request the compact block if we didn't get one yet.
  264. wait_until(test_node.received_block_announcement, timeout=30)
  265. assert(test_node.received_block_announcement())
  266. with mininode_lock:
  267. if test_node.last_cmpctblock is None:
  268. test_node.clear_block_announcement()
  269. inv = CInv(4, block_hash) # 4 == "CompactBlock"
  270. test_node.send_message(msg_getdata([inv]))
  271. wait_until(test_node.received_block_announcement, timeout=30)
  272. assert(test_node.received_block_announcement())
  273. # Now we should have the compactblock
  274. header_and_shortids = None
  275. with mininode_lock:
  276. assert(test_node.last_cmpctblock is not None)
  277. # Convert the on-the-wire representation to absolute indexes
  278. header_and_shortids = HeaderAndShortIDs(test_node.last_cmpctblock.header_and_shortids)
  279. # Check that we got the right block!
  280. header_and_shortids.header.calc_sha256()
  281. assert_equal(header_and_shortids.header.sha256, block_hash)
  282. # Make sure the prefilled_txn appears to have included the coinbase
  283. assert(len(header_and_shortids.prefilled_txn) >= 1)
  284. assert_equal(header_and_shortids.prefilled_txn[0].index, 0)
  285. # Check that all prefilled_txn entries match what's in the block.
  286. for entry in header_and_shortids.prefilled_txn:
  287. entry.tx.calc_sha256()
  288. # This checks the non-witness parts of the tx agree
  289. assert_equal(entry.tx.sha256, block.vtx[entry.index].sha256)
  290. # And this checks the witness
  291. wtxid = entry.tx.calc_sha256(True)
  292. if version == 2:
  293. assert_equal(wtxid, block.vtx[entry.index].calc_sha256(True))
  294. else:
  295. # Shouldn't have received a witness
  296. assert(entry.tx.wit.is_null())
  297. # Check that the cmpctblock message announced all the transactions.
  298. assert_equal(len(header_and_shortids.prefilled_txn) + len(header_and_shortids.shortids), len(block.vtx))
  299. # And now check that all the shortids are as expected as well.
  300. # Determine the siphash keys to use.
  301. [k0, k1] = header_and_shortids.get_siphash_keys()
  302. index = 0
  303. while index < len(block.vtx):
  304. if (len(header_and_shortids.prefilled_txn) > 0 and
  305. header_and_shortids.prefilled_txn[0].index == index):
  306. # Already checked prefilled transactions above
  307. header_and_shortids.prefilled_txn.pop(0)
  308. else:
  309. tx_hash = block.vtx[index].sha256
  310. if version == 2:
  311. tx_hash = block.vtx[index].calc_sha256(True)
  312. shortid = calculate_shortid(k0, k1, tx_hash)
  313. assert_equal(shortid, header_and_shortids.shortids[0])
  314. header_and_shortids.shortids.pop(0)
  315. index += 1
  316. # Test that bitcoind requests compact blocks when we announce new blocks
  317. # via header or inv, and that responding to getblocktxn causes the block
  318. # to be successfully reconstructed.
  319. # Post-segwit: upgraded nodes would only make this request of cb-version-2,
  320. # NODE_WITNESS peers. Unupgraded nodes would still make this request of
  321. # any cb-version-1-supporting peer.
  322. def test_compactblock_requests(self, node, test_node):
  323. # Try announcing a block with an inv or header, expect a compactblock
  324. # request
  325. for announce in ["inv", "header"]:
  326. block = self.build_block_on_tip(node)
  327. with mininode_lock:
  328. test_node.last_getdata = None
  329. if announce == "inv":
  330. test_node.send_message(msg_inv([CInv(2, block.sha256)]))
  331. else:
  332. test_node.send_header_for_blocks([block])
  333. success = wait_until(lambda: test_node.last_getdata is not None, timeout=30)
  334. assert(success)
  335. assert_equal(len(test_node.last_getdata.inv), 1)
  336. assert_equal(test_node.last_getdata.inv[0].type, 4)
  337. assert_equal(test_node.last_getdata.inv[0].hash, block.sha256)
  338. # Send back a compactblock message that omits the coinbase
  339. comp_block = HeaderAndShortIDs()
  340. comp_block.header = CBlockHeader(block)
  341. comp_block.nonce = 0
  342. [k0, k1] = comp_block.get_siphash_keys()
  343. comp_block.shortids = [
  344. calculate_shortid(k0, k1, block.vtx[0].sha256) ]
  345. test_node.send_and_ping(msg_cmpctblock(comp_block.to_p2p()))
  346. assert_equal(int(node.getbestblockhash(), 16), block.hashPrevBlock)
  347. # Expect a getblocktxn message.
  348. with mininode_lock:
  349. assert(test_node.last_getblocktxn is not None)
  350. absolute_indexes = test_node.last_getblocktxn.block_txn_request.to_absolute()
  351. assert_equal(absolute_indexes, [0]) # should be a coinbase request
  352. # Send the coinbase, and verify that the tip advances.
  353. msg = msg_blocktxn()
  354. msg.block_transactions.blockhash = block.sha256
  355. msg.block_transactions.transactions = [block.vtx[0]]
  356. test_node.send_and_ping(msg)
  357. assert_equal(int(node.getbestblockhash(), 16), block.sha256)
  358. # Create a chain of transactions from given utxo, and add to a new block.
  359. def build_block_with_transactions(self, node, utxo, num_transactions):
  360. block = self.build_block_on_tip(node)
  361. for i in range(num_transactions):
  362. tx = CTransaction()
  363. tx.vin.append(CTxIn(COutPoint(utxo[0], utxo[1]), b''))
  364. tx.vout.append(CTxOut(utxo[2] - 1000, CScript([OP_TRUE])))
  365. tx.rehash()
  366. utxo = [tx.sha256, 0, tx.vout[0].nValue]
  367. block.vtx.append(tx)
  368. block.hashMerkleRoot = block.calc_merkle_root()
  369. block.solve()
  370. return block
  371. # Test that we only receive getblocktxn requests for transactions that the
  372. # node needs, and that responding to them causes the block to be
  373. # reconstructed.
  374. def test_getblocktxn_requests(self, node, test_node, version):
  375. with_witness = (version==2)
  376. def test_getblocktxn_response(compact_block, peer, expected_result):
  377. msg = msg_cmpctblock(compact_block.to_p2p())
  378. peer.send_and_ping(msg)
  379. with mininode_lock:
  380. assert(peer.last_getblocktxn is not None)
  381. absolute_indexes = peer.last_getblocktxn.block_txn_request.to_absolute()
  382. assert_equal(absolute_indexes, expected_result)
  383. def test_tip_after_message(node, peer, msg, tip):
  384. peer.send_and_ping(msg)
  385. assert_equal(int(node.getbestblockhash(), 16), tip)
  386. # First try announcing compactblocks that won't reconstruct, and verify
  387. # that we receive getblocktxn messages back.
  388. utxo = self.utxos.pop(0)
  389. block = self.build_block_with_transactions(node, utxo, 5)
  390. self.utxos.append([block.vtx[-1].sha256, 0, block.vtx[-1].vout[0].nValue])
  391. comp_block = HeaderAndShortIDs()
  392. comp_block.initialize_from_block(block, use_witness=with_witness)
  393. test_getblocktxn_response(comp_block, test_node, [1, 2, 3, 4, 5])
  394. msg_bt = msg_blocktxn()
  395. if with_witness:
  396. msg_bt = msg_witness_blocktxn() # serialize with witnesses
  397. msg_bt.block_transactions = BlockTransactions(block.sha256, block.vtx[1:])
  398. test_tip_after_message(node, test_node, msg_bt, block.sha256)
  399. utxo = self.utxos.pop(0)
  400. block = self.build_block_with_transactions(node, utxo, 5)
  401. self.utxos.append([block.vtx[-1].sha256, 0, block.vtx[-1].vout[0].nValue])
  402. # Now try interspersing the prefilled transactions
  403. comp_block.initialize_from_block(block, prefill_list=[0, 1, 5], use_witness=with_witness)
  404. test_getblocktxn_response(comp_block, test_node, [2, 3, 4])
  405. msg_bt.block_transactions = BlockTransactions(block.sha256, block.vtx[2:5])
  406. test_tip_after_message(node, test_node, msg_bt, block.sha256)
  407. # Now try giving one transaction ahead of time.
  408. utxo = self.utxos.pop(0)
  409. block = self.build_block_with_transactions(node, utxo, 5)
  410. self.utxos.append([block.vtx[-1].sha256, 0, block.vtx[-1].vout[0].nValue])
  411. test_node.send_and_ping(msg_tx(block.vtx[1]))
  412. assert(block.vtx[1].hash in node.getrawmempool())
  413. # Prefill 4 out of the 6 transactions, and verify that only the one
  414. # that was not in the mempool is requested.
  415. comp_block.initialize_from_block(block, prefill_list=[0, 2, 3, 4], use_witness=with_witness)
  416. test_getblocktxn_response(comp_block, test_node, [5])
  417. msg_bt.block_transactions = BlockTransactions(block.sha256, [block.vtx[5]])
  418. test_tip_after_message(node, test_node, msg_bt, block.sha256)
  419. # Now provide all transactions to the node before the block is
  420. # announced and verify reconstruction happens immediately.
  421. utxo = self.utxos.pop(0)
  422. block = self.build_block_with_transactions(node, utxo, 10)
  423. self.utxos.append([block.vtx[-1].sha256, 0, block.vtx[-1].vout[0].nValue])
  424. for tx in block.vtx[1:]:
  425. test_node.send_message(msg_tx(tx))
  426. test_node.sync_with_ping()
  427. # Make sure all transactions were accepted.
  428. mempool = node.getrawmempool()
  429. for tx in block.vtx[1:]:
  430. assert(tx.hash in mempool)
  431. # Clear out last request.
  432. with mininode_lock:
  433. test_node.last_getblocktxn = None
  434. # Send compact block
  435. comp_block.initialize_from_block(block, prefill_list=[0], use_witness=with_witness)
  436. test_tip_after_message(node, test_node, msg_cmpctblock(comp_block.to_p2p()), block.sha256)
  437. with mininode_lock:
  438. # Shouldn't have gotten a request for any transaction
  439. assert(test_node.last_getblocktxn is None)
  440. # Incorrectly responding to a getblocktxn shouldn't cause the block to be
  441. # permanently failed.
  442. def test_incorrect_blocktxn_response(self, node, test_node, version):
  443. if (len(self.utxos) == 0):
  444. self.make_utxos()
  445. utxo = self.utxos.pop(0)
  446. block = self.build_block_with_transactions(node, utxo, 10)
  447. self.utxos.append([block.vtx[-1].sha256, 0, block.vtx[-1].vout[0].nValue])
  448. # Relay the first 5 transactions from the block in advance
  449. for tx in block.vtx[1:6]:
  450. test_node.send_message(msg_tx(tx))
  451. test_node.sync_with_ping()
  452. # Make sure all transactions were accepted.
  453. mempool = node.getrawmempool()
  454. for tx in block.vtx[1:6]:
  455. assert(tx.hash in mempool)
  456. # Send compact block
  457. comp_block = HeaderAndShortIDs()
  458. comp_block.initialize_from_block(block, prefill_list=[0], use_witness=(version == 2))
  459. test_node.send_and_ping(msg_cmpctblock(comp_block.to_p2p()))
  460. absolute_indexes = []
  461. with mininode_lock:
  462. assert(test_node.last_getblocktxn is not None)
  463. absolute_indexes = test_node.last_getblocktxn.block_txn_request.to_absolute()
  464. assert_equal(absolute_indexes, [6, 7, 8, 9, 10])
  465. # Now give an incorrect response.
  466. # Note that it's possible for bitcoind to be smart enough to know we're
  467. # lying, since it could check to see if the shortid matches what we're
  468. # sending, and eg disconnect us for misbehavior. If that behavior
  469. # change were made, we could just modify this test by having a
  470. # different peer provide the block further down, so that we're still
  471. # verifying that the block isn't marked bad permanently. This is good
  472. # enough for now.
  473. msg = msg_blocktxn()
  474. if version==2:
  475. msg = msg_witness_blocktxn()
  476. msg.block_transactions = BlockTransactions(block.sha256, [block.vtx[5]] + block.vtx[7:])
  477. test_node.send_and_ping(msg)
  478. # Tip should not have updated
  479. assert_equal(int(node.getbestblockhash(), 16), block.hashPrevBlock)
  480. # We should receive a getdata request
  481. success = wait_until(lambda: test_node.last_getdata is not None, timeout=10)
  482. assert(success)
  483. assert_equal(len(test_node.last_getdata.inv), 1)
  484. assert(test_node.last_getdata.inv[0].type == 2 or test_node.last_getdata.inv[0].type == 2|MSG_WITNESS_FLAG)
  485. assert_equal(test_node.last_getdata.inv[0].hash, block.sha256)
  486. # Deliver the block
  487. if version==2:
  488. test_node.send_and_ping(msg_witness_block(block))
  489. else:
  490. test_node.send_and_ping(msg_block(block))
  491. assert_equal(int(node.getbestblockhash(), 16), block.sha256)
  492. def test_getblocktxn_handler(self, node, test_node, version):
  493. # bitcoind won't respond for blocks whose height is more than 15 blocks
  494. # deep.
  495. MAX_GETBLOCKTXN_DEPTH = 15
  496. chain_height = node.getblockcount()
  497. current_height = chain_height
  498. while (current_height >= chain_height - MAX_GETBLOCKTXN_DEPTH):
  499. block_hash = node.getblockhash(current_height)
  500. block = FromHex(CBlock(), node.getblock(block_hash, False))
  501. msg = msg_getblocktxn()
  502. msg.block_txn_request = BlockTransactionsRequest(int(block_hash, 16), [])
  503. num_to_request = random.randint(1, len(block.vtx))
  504. msg.block_txn_request.from_absolute(sorted(random.sample(range(len(block.vtx)), num_to_request)))
  505. test_node.send_message(msg)
  506. success = wait_until(lambda: test_node.last_blocktxn is not None, timeout=10)
  507. assert(success)
  508. [tx.calc_sha256() for tx in block.vtx]
  509. with mininode_lock:
  510. assert_equal(test_node.last_blocktxn.block_transactions.blockhash, int(block_hash, 16))
  511. all_indices = msg.block_txn_request.to_absolute()
  512. for index in all_indices:
  513. tx = test_node.last_blocktxn.block_transactions.transactions.pop(0)
  514. tx.calc_sha256()
  515. assert_equal(tx.sha256, block.vtx[index].sha256)
  516. if version == 1:
  517. # Witnesses should have been stripped
  518. assert(tx.wit.is_null())
  519. else:
  520. # Check that the witness matches
  521. assert_equal(tx.calc_sha256(True), block.vtx[index].calc_sha256(True))
  522. test_node.last_blocktxn = None
  523. current_height -= 1
  524. # Next request should be ignored, as we're past the allowed depth.
  525. block_hash = node.getblockhash(current_height)
  526. msg.block_txn_request = BlockTransactionsRequest(int(block_hash, 16), [0])
  527. test_node.send_and_ping(msg)
  528. with mininode_lock:
  529. assert_equal(test_node.last_blocktxn, None)
  530. def test_compactblocks_not_at_tip(self, node, test_node):
  531. # Test that requesting old compactblocks doesn't work.
  532. MAX_CMPCTBLOCK_DEPTH = 11
  533. new_blocks = []
  534. for i in range(MAX_CMPCTBLOCK_DEPTH):
  535. test_node.clear_block_announcement()
  536. new_blocks.append(node.generate(1)[0])
  537. wait_until(test_node.received_block_announcement, timeout=30)
  538. test_node.clear_block_announcement()
  539. test_node.send_message(msg_getdata([CInv(4, int(new_blocks[0], 16))]))
  540. success = wait_until(lambda: test_node.last_cmpctblock is not None, timeout=30)
  541. assert(success)
  542. test_node.clear_block_announcement()
  543. node.generate(1)
  544. wait_until(test_node.received_block_announcement, timeout=30)
  545. test_node.clear_block_announcement()
  546. test_node.send_message(msg_getdata([CInv(4, int(new_blocks[0], 16))]))
  547. success = wait_until(lambda: test_node.last_block is not None, timeout=30)
  548. assert(success)
  549. with mininode_lock:
  550. test_node.last_block.block.calc_sha256()
  551. assert_equal(test_node.last_block.block.sha256, int(new_blocks[0], 16))
  552. # Generate an old compactblock, and verify that it's not accepted.
  553. cur_height = node.getblockcount()
  554. hashPrevBlock = int(node.getblockhash(cur_height-5), 16)
  555. block = self.build_block_on_tip(node)
  556. block.hashPrevBlock = hashPrevBlock
  557. block.solve()
  558. comp_block = HeaderAndShortIDs()
  559. comp_block.initialize_from_block(block)
  560. test_node.send_and_ping(msg_cmpctblock(comp_block.to_p2p()))
  561. tips = node.getchaintips()
  562. found = False
  563. for x in tips:
  564. if x["hash"] == block.hash:
  565. assert_equal(x["status"], "headers-only")
  566. found = True
  567. break
  568. assert(found)
  569. # Requesting this block via getblocktxn should silently fail
  570. # (to avoid fingerprinting attacks).
  571. msg = msg_getblocktxn()
  572. msg.block_txn_request = BlockTransactionsRequest(block.sha256, [0])
  573. with mininode_lock:
  574. test_node.last_blocktxn = None
  575. test_node.send_and_ping(msg)
  576. with mininode_lock:
  577. assert(test_node.last_blocktxn is None)
  578. def activate_segwit(self, node):
  579. node.generate(144*3)
  580. assert_equal(get_bip9_status(node, "segwit")["status"], 'active')
  581. def test_end_to_end_block_relay(self, node, listeners):
  582. utxo = self.utxos.pop(0)
  583. block = self.build_block_with_transactions(node, utxo, 10)
  584. [l.clear_block_announcement() for l in listeners]
  585. # ToHex() won't serialize with witness, but this block has no witnesses
  586. # anyway. TODO: repeat this test with witness tx's to a segwit node.
  587. node.submitblock(ToHex(block))
  588. for l in listeners:
  589. wait_until(lambda: l.received_block_announcement(), timeout=30)
  590. with mininode_lock:
  591. for l in listeners:
  592. assert(l.last_cmpctblock is not None)
  593. l.last_cmpctblock.header_and_shortids.header.calc_sha256()
  594. assert_equal(l.last_cmpctblock.header_and_shortids.header.sha256, block.sha256)
  595. # Helper for enabling cb announcements
  596. # Send the sendcmpct request and sync headers
  597. def request_cb_announcements(self, peer, node, version):
  598. tip = node.getbestblockhash()
  599. peer.get_headers(locator=[int(tip, 16)], hashstop=0)
  600. msg = msg_sendcmpct()
  601. msg.version = version
  602. msg.announce = True
  603. peer.send_and_ping(msg)
  604. def run_test(self):
  605. # Setup the p2p connections and start up the network thread.
  606. self.test_node = TestNode()
  607. self.segwit_node = TestNode()
  608. self.old_node = TestNode() # version 1 peer <--> segwit node
  609. connections = []
  610. connections.append(NodeConn('127.0.0.1', p2p_port(0), self.nodes[0], self.test_node))
  611. connections.append(NodeConn('127.0.0.1', p2p_port(1), self.nodes[1],
  612. self.segwit_node, services=NODE_NETWORK|NODE_WITNESS))
  613. connections.append(NodeConn('127.0.0.1', p2p_port(1), self.nodes[1],
  614. self.old_node, services=NODE_NETWORK))
  615. self.test_node.add_connection(connections[0])
  616. self.segwit_node.add_connection(connections[1])
  617. self.old_node.add_connection(connections[2])
  618. NetworkThread().start() # Start up network handling in another thread
  619. # Test logic begins here
  620. self.test_node.wait_for_verack()
  621. # We will need UTXOs to construct transactions in later tests.
  622. self.make_utxos()
  623. print("Running tests, pre-segwit activation:")
  624. print("\tTesting SENDCMPCT p2p message... ")
  625. self.test_sendcmpct(self.nodes[0], self.test_node, 1)
  626. sync_blocks(self.nodes)
  627. self.test_sendcmpct(self.nodes[1], self.segwit_node, 2, old_node=self.old_node)
  628. sync_blocks(self.nodes)
  629. print("\tTesting compactblock construction...")
  630. self.test_compactblock_construction(self.nodes[0], self.test_node, 1, False)
  631. sync_blocks(self.nodes)
  632. self.test_compactblock_construction(self.nodes[1], self.segwit_node, 2, False)
  633. sync_blocks(self.nodes)
  634. print("\tTesting compactblock requests... ")
  635. self.test_compactblock_requests(self.nodes[0], self.test_node)
  636. sync_blocks(self.nodes)
  637. self.test_compactblock_requests(self.nodes[1], self.segwit_node)
  638. sync_blocks(self.nodes)
  639. print("\tTesting getblocktxn requests...")
  640. self.test_getblocktxn_requests(self.nodes[0], self.test_node, 1)
  641. sync_blocks(self.nodes)
  642. self.test_getblocktxn_requests(self.nodes[1], self.segwit_node, 2)
  643. sync_blocks(self.nodes)
  644. print("\tTesting getblocktxn handler...")
  645. self.test_getblocktxn_handler(self.nodes[0], self.test_node, 1)
  646. sync_blocks(self.nodes)
  647. self.test_getblocktxn_handler(self.nodes[1], self.segwit_node, 2)
  648. self.test_getblocktxn_handler(self.nodes[1], self.old_node, 1)
  649. sync_blocks(self.nodes)
  650. print("\tTesting compactblock requests/announcements not at chain tip...")
  651. self.test_compactblocks_not_at_tip(self.nodes[0], self.test_node)
  652. sync_blocks(self.nodes)
  653. self.test_compactblocks_not_at_tip(self.nodes[1], self.segwit_node)
  654. self.test_compactblocks_not_at_tip(self.nodes[1], self.old_node)
  655. sync_blocks(self.nodes)
  656. print("\tTesting handling of incorrect blocktxn responses...")
  657. self.test_incorrect_blocktxn_response(self.nodes[0], self.test_node, 1)
  658. sync_blocks(self.nodes)
  659. self.test_incorrect_blocktxn_response(self.nodes[1], self.segwit_node, 2)
  660. sync_blocks(self.nodes)
  661. # End-to-end block relay tests
  662. print("\tTesting end-to-end block relay...")
  663. self.request_cb_announcements(self.test_node, self.nodes[0], 1)
  664. self.request_cb_announcements(self.old_node, self.nodes[1], 1)
  665. self.request_cb_announcements(self.segwit_node, self.nodes[1], 2)
  666. self.test_end_to_end_block_relay(self.nodes[0], [self.segwit_node, self.test_node, self.old_node])
  667. self.test_end_to_end_block_relay(self.nodes[1], [self.segwit_node, self.test_node, self.old_node])
  668. # Advance to segwit activation
  669. print ("\nAdvancing to segwit activation\n")
  670. self.activate_segwit(self.nodes[1])
  671. print ("Running tests, post-segwit activation...")
  672. print("\tTesting compactblock construction...")
  673. self.test_compactblock_construction(self.nodes[1], self.old_node, 1, True)
  674. self.test_compactblock_construction(self.nodes[1], self.segwit_node, 2, True)
  675. sync_blocks(self.nodes)
  676. print("\tTesting compactblock requests (unupgraded node)... ")
  677. self.test_compactblock_requests(self.nodes[0], self.test_node)
  678. print("\tTesting getblocktxn requests (unupgraded node)...")
  679. self.test_getblocktxn_requests(self.nodes[0], self.test_node, 1)
  680. # Need to manually sync node0 and node1, because post-segwit activation,
  681. # node1 will not download blocks from node0.
  682. print("\tSyncing nodes...")
  683. assert(self.nodes[0].getbestblockhash() != self.nodes[1].getbestblockhash())
  684. while (self.nodes[0].getblockcount() > self.nodes[1].getblockcount()):
  685. block_hash = self.nodes[0].getblockhash(self.nodes[1].getblockcount()+1)
  686. self.nodes[1].submitblock(self.nodes[0].getblock(block_hash, False))
  687. assert_equal(self.nodes[0].getbestblockhash(), self.nodes[1].getbestblockhash())
  688. print("\tTesting compactblock requests (segwit node)... ")
  689. self.test_compactblock_requests(self.nodes[1], self.segwit_node)
  690. print("\tTesting getblocktxn requests (segwit node)...")
  691. self.test_getblocktxn_requests(self.nodes[1], self.segwit_node, 2)
  692. sync_blocks(self.nodes)
  693. print("\tTesting getblocktxn handler (segwit node should return witnesses)...")
  694. self.test_getblocktxn_handler(self.nodes[1], self.segwit_node, 2)
  695. self.test_getblocktxn_handler(self.nodes[1], self.old_node, 1)
  696. # Test that if we submitblock to node1, we'll get a compact block
  697. # announcement to all peers.
  698. # (Post-segwit activation, blocks won't propagate from node0 to node1
  699. # automatically, so don't bother testing a block announced to node0.)
  700. print("\tTesting end-to-end block relay...")
  701. self.request_cb_announcements(self.test_node, self.nodes[0], 1)
  702. self.request_cb_announcements(self.old_node, self.nodes[1], 1)
  703. self.request_cb_announcements(self.segwit_node, self.nodes[1], 2)
  704. self.test_end_to_end_block_relay(self.nodes[1], [self.segwit_node, self.test_node, self.old_node])
  705. print("\tTesting invalid index in cmpctblock message...")
  706. self.test_invalid_cmpctblock_message()
  707. if __name__ == '__main__':
  708. CompactBlocksTest().main()