You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

p2p-compactblocks.py 43KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968
  1. #!/usr/bin/env python3
  2. # Copyright (c) 2016 The Bitcoin Core developers
  3. # Distributed under the MIT software license, see the accompanying
  4. # file COPYING or http://www.opensource.org/licenses/mit-license.php.
  5. """Test compact blocks (BIP 152).
  6. Version 1 compact blocks are pre-segwit (txids)
  7. Version 2 compact blocks are post-segwit (wtxids)
  8. """
  9. from test_framework.mininode import *
  10. from test_framework.test_framework import BitcoinTestFramework
  11. from test_framework.util import *
  12. from test_framework.blocktools import create_block, create_coinbase, add_witness_commitment
  13. from test_framework.script import CScript, OP_TRUE
  14. # TestNode: A peer we use to send messages to bitcoind, and store responses.
  15. class TestNode(SingleNodeConnCB):
  16. def __init__(self):
  17. SingleNodeConnCB.__init__(self)
  18. self.last_sendcmpct = []
  19. self.last_headers = None
  20. self.last_inv = None
  21. self.last_cmpctblock = None
  22. self.block_announced = False
  23. self.last_getdata = None
  24. self.last_getheaders = None
  25. self.last_getblocktxn = None
  26. self.last_block = None
  27. self.last_blocktxn = None
  28. # Store the hashes of blocks we've seen announced.
  29. # This is for synchronizing the p2p message traffic,
  30. # so we can eg wait until a particular block is announced.
  31. self.set_announced_blockhashes = set()
  32. def on_sendcmpct(self, conn, message):
  33. self.last_sendcmpct.append(message)
  34. def on_block(self, conn, message):
  35. self.last_block = message
  36. def on_cmpctblock(self, conn, message):
  37. self.last_cmpctblock = message
  38. self.block_announced = True
  39. self.last_cmpctblock.header_and_shortids.header.calc_sha256()
  40. self.set_announced_blockhashes.add(self.last_cmpctblock.header_and_shortids.header.sha256)
  41. def on_headers(self, conn, message):
  42. self.last_headers = message
  43. self.block_announced = True
  44. for x in self.last_headers.headers:
  45. x.calc_sha256()
  46. self.set_announced_blockhashes.add(x.sha256)
  47. def on_inv(self, conn, message):
  48. self.last_inv = message
  49. for x in self.last_inv.inv:
  50. if x.type == 2:
  51. self.block_announced = True
  52. self.set_announced_blockhashes.add(x.hash)
  53. def on_getdata(self, conn, message):
  54. self.last_getdata = message
  55. def on_getheaders(self, conn, message):
  56. self.last_getheaders = message
  57. def on_getblocktxn(self, conn, message):
  58. self.last_getblocktxn = message
  59. def on_blocktxn(self, conn, message):
  60. self.last_blocktxn = message
  61. # Requires caller to hold mininode_lock
  62. def received_block_announcement(self):
  63. return self.block_announced
  64. def clear_block_announcement(self):
  65. with mininode_lock:
  66. self.block_announced = False
  67. self.last_inv = None
  68. self.last_headers = None
  69. self.last_cmpctblock = None
  70. def get_headers(self, locator, hashstop):
  71. msg = msg_getheaders()
  72. msg.locator.vHave = locator
  73. msg.hashstop = hashstop
  74. self.connection.send_message(msg)
  75. def send_header_for_blocks(self, new_blocks):
  76. headers_message = msg_headers()
  77. headers_message.headers = [CBlockHeader(b) for b in new_blocks]
  78. self.send_message(headers_message)
  79. def request_headers_and_sync(self, locator, hashstop=0):
  80. self.clear_block_announcement()
  81. self.get_headers(locator, hashstop)
  82. assert(wait_until(self.received_block_announcement, timeout=30))
  83. assert(self.received_block_announcement())
  84. self.clear_block_announcement()
  85. # Block until a block announcement for a particular block hash is
  86. # received.
  87. def wait_for_block_announcement(self, block_hash, timeout=30):
  88. def received_hash():
  89. return (block_hash in self.set_announced_blockhashes)
  90. return wait_until(received_hash, timeout=timeout)
  91. class CompactBlocksTest(BitcoinTestFramework):
  92. def __init__(self):
  93. super().__init__()
  94. self.setup_clean_chain = True
  95. # Node0 = pre-segwit, node1 = segwit-aware
  96. self.num_nodes = 2
  97. self.utxos = []
  98. def setup_network(self):
  99. self.nodes = []
  100. # Start up node0 to be a version 1, pre-segwit node.
  101. self.nodes = start_nodes(self.num_nodes, self.options.tmpdir,
  102. [["-bip9params=segwit:0:0"],
  103. ["-txindex"]])
  104. connect_nodes(self.nodes[0], 1)
  105. def build_block_on_tip(self, node, segwit=False):
  106. height = node.getblockcount()
  107. tip = node.getbestblockhash()
  108. mtp = node.getblockheader(tip)['mediantime']
  109. block = create_block(int(tip, 16), create_coinbase(height + 1), mtp + 1)
  110. block.nVersion = 4
  111. if segwit:
  112. add_witness_commitment(block)
  113. block.solve()
  114. return block
  115. # Create 10 more anyone-can-spend utxo's for testing.
  116. def make_utxos(self):
  117. # Doesn't matter which node we use, just use node0.
  118. block = self.build_block_on_tip(self.nodes[0])
  119. self.test_node.send_and_ping(msg_block(block))
  120. assert(int(self.nodes[0].getbestblockhash(), 16) == block.sha256)
  121. self.nodes[0].generate(100)
  122. total_value = block.vtx[0].vout[0].nValue
  123. out_value = total_value // 10
  124. tx = CTransaction()
  125. tx.vin.append(CTxIn(COutPoint(block.vtx[0].sha256, 0), b''))
  126. for i in range(10):
  127. tx.vout.append(CTxOut(out_value, CScript([OP_TRUE])))
  128. tx.rehash()
  129. block2 = self.build_block_on_tip(self.nodes[0])
  130. block2.vtx.append(tx)
  131. block2.hashMerkleRoot = block2.calc_merkle_root()
  132. block2.solve()
  133. self.test_node.send_and_ping(msg_block(block2))
  134. assert_equal(int(self.nodes[0].getbestblockhash(), 16), block2.sha256)
  135. self.utxos.extend([[tx.sha256, i, out_value] for i in range(10)])
  136. return
  137. # Test "sendcmpct" (between peers preferring the same version):
  138. # - No compact block announcements unless sendcmpct is sent.
  139. # - If sendcmpct is sent with version > preferred_version, the message is ignored.
  140. # - If sendcmpct is sent with boolean 0, then block announcements are not
  141. # made with compact blocks.
  142. # - If sendcmpct is then sent with boolean 1, then new block announcements
  143. # are made with compact blocks.
  144. # If old_node is passed in, request compact blocks with version=preferred-1
  145. # and verify that it receives block announcements via compact block.
  146. def test_sendcmpct(self, node, test_node, preferred_version, old_node=None):
  147. # Make sure we get a SENDCMPCT message from our peer
  148. def received_sendcmpct():
  149. return (len(test_node.last_sendcmpct) > 0)
  150. got_message = wait_until(received_sendcmpct, timeout=30)
  151. assert(received_sendcmpct())
  152. assert(got_message)
  153. with mininode_lock:
  154. # Check that the first version received is the preferred one
  155. assert_equal(test_node.last_sendcmpct[0].version, preferred_version)
  156. # And that we receive versions down to 1.
  157. assert_equal(test_node.last_sendcmpct[-1].version, 1)
  158. test_node.last_sendcmpct = []
  159. tip = int(node.getbestblockhash(), 16)
  160. def check_announcement_of_new_block(node, peer, predicate):
  161. peer.clear_block_announcement()
  162. block_hash = int(node.generate(1)[0], 16)
  163. peer.wait_for_block_announcement(block_hash, timeout=30)
  164. assert(peer.block_announced)
  165. assert(got_message)
  166. with mininode_lock:
  167. assert predicate(peer), (
  168. "block_hash={!r}, cmpctblock={!r}, inv={!r}".format(
  169. block_hash, peer.last_cmpctblock, peer.last_inv))
  170. # We shouldn't get any block announcements via cmpctblock yet.
  171. check_announcement_of_new_block(node, test_node, lambda p: p.last_cmpctblock is None)
  172. # Try one more time, this time after requesting headers.
  173. test_node.request_headers_and_sync(locator=[tip])
  174. check_announcement_of_new_block(node, test_node, lambda p: p.last_cmpctblock is None and p.last_inv is not None)
  175. # Test a few ways of using sendcmpct that should NOT
  176. # result in compact block announcements.
  177. # Before each test, sync the headers chain.
  178. test_node.request_headers_and_sync(locator=[tip])
  179. # Now try a SENDCMPCT message with too-high version
  180. sendcmpct = msg_sendcmpct()
  181. sendcmpct.version = preferred_version+1
  182. sendcmpct.announce = True
  183. test_node.send_and_ping(sendcmpct)
  184. check_announcement_of_new_block(node, test_node, lambda p: p.last_cmpctblock is None)
  185. # Headers sync before next test.
  186. test_node.request_headers_and_sync(locator=[tip])
  187. # Now try a SENDCMPCT message with valid version, but announce=False
  188. sendcmpct.version = preferred_version
  189. sendcmpct.announce = False
  190. test_node.send_and_ping(sendcmpct)
  191. check_announcement_of_new_block(node, test_node, lambda p: p.last_cmpctblock is None)
  192. # Headers sync before next test.
  193. test_node.request_headers_and_sync(locator=[tip])
  194. # Finally, try a SENDCMPCT message with announce=True
  195. sendcmpct.version = preferred_version
  196. sendcmpct.announce = True
  197. test_node.send_and_ping(sendcmpct)
  198. check_announcement_of_new_block(node, test_node, lambda p: p.last_cmpctblock is not None)
  199. # Try one more time (no headers sync should be needed!)
  200. check_announcement_of_new_block(node, test_node, lambda p: p.last_cmpctblock is not None)
  201. # Try one more time, after turning on sendheaders
  202. test_node.send_and_ping(msg_sendheaders())
  203. check_announcement_of_new_block(node, test_node, lambda p: p.last_cmpctblock is not None)
  204. # Try one more time, after sending a version-1, announce=false message.
  205. sendcmpct.version = preferred_version-1
  206. sendcmpct.announce = False
  207. test_node.send_and_ping(sendcmpct)
  208. check_announcement_of_new_block(node, test_node, lambda p: p.last_cmpctblock is not None)
  209. # Now turn off announcements
  210. sendcmpct.version = preferred_version
  211. sendcmpct.announce = False
  212. test_node.send_and_ping(sendcmpct)
  213. check_announcement_of_new_block(node, test_node, lambda p: p.last_cmpctblock is None and p.last_headers is not None)
  214. if old_node is not None:
  215. # Verify that a peer using an older protocol version can receive
  216. # announcements from this node.
  217. sendcmpct.version = preferred_version-1
  218. sendcmpct.announce = True
  219. old_node.send_and_ping(sendcmpct)
  220. # Header sync
  221. old_node.request_headers_and_sync(locator=[tip])
  222. check_announcement_of_new_block(node, old_node, lambda p: p.last_cmpctblock is not None)
  223. # This test actually causes bitcoind to (reasonably!) disconnect us, so do this last.
  224. def test_invalid_cmpctblock_message(self):
  225. self.nodes[0].generate(101)
  226. block = self.build_block_on_tip(self.nodes[0])
  227. cmpct_block = P2PHeaderAndShortIDs()
  228. cmpct_block.header = CBlockHeader(block)
  229. cmpct_block.prefilled_txn_length = 1
  230. # This index will be too high
  231. prefilled_txn = PrefilledTransaction(1, block.vtx[0])
  232. cmpct_block.prefilled_txn = [prefilled_txn]
  233. self.test_node.send_and_ping(msg_cmpctblock(cmpct_block))
  234. assert(int(self.nodes[0].getbestblockhash(), 16) == block.hashPrevBlock)
  235. # Compare the generated shortids to what we expect based on BIP 152, given
  236. # bitcoind's choice of nonce.
  237. def test_compactblock_construction(self, node, test_node, version, use_witness_address):
  238. # Generate a bunch of transactions.
  239. node.generate(101)
  240. num_transactions = 25
  241. address = node.getnewaddress()
  242. if use_witness_address:
  243. # Want at least one segwit spend, so move all funds to
  244. # a witness address.
  245. address = node.addwitnessaddress(address)
  246. value_to_send = node.getbalance()
  247. node.sendtoaddress(address, satoshi_round(value_to_send-Decimal(0.1)))
  248. node.generate(1)
  249. segwit_tx_generated = False
  250. for i in range(num_transactions):
  251. txid = node.sendtoaddress(address, 0.1)
  252. hex_tx = node.gettransaction(txid)["hex"]
  253. tx = FromHex(CTransaction(), hex_tx)
  254. if not tx.wit.is_null():
  255. segwit_tx_generated = True
  256. if use_witness_address:
  257. assert(segwit_tx_generated) # check that our test is not broken
  258. # Wait until we've seen the block announcement for the resulting tip
  259. tip = int(node.getbestblockhash(), 16)
  260. assert(test_node.wait_for_block_announcement(tip))
  261. # Make sure we will receive a fast-announce compact block
  262. self.request_cb_announcements(test_node, node, version)
  263. # Now mine a block, and look at the resulting compact block.
  264. test_node.clear_block_announcement()
  265. block_hash = int(node.generate(1)[0], 16)
  266. # Store the raw block in our internal format.
  267. block = FromHex(CBlock(), node.getblock("%02x" % block_hash, False))
  268. [tx.calc_sha256() for tx in block.vtx]
  269. block.rehash()
  270. # Wait until the block was announced (via compact blocks)
  271. wait_until(test_node.received_block_announcement, timeout=30)
  272. assert(test_node.received_block_announcement())
  273. # Now fetch and check the compact block
  274. header_and_shortids = None
  275. with mininode_lock:
  276. assert(test_node.last_cmpctblock is not None)
  277. # Convert the on-the-wire representation to absolute indexes
  278. header_and_shortids = HeaderAndShortIDs(test_node.last_cmpctblock.header_and_shortids)
  279. self.check_compactblock_construction_from_block(version, header_and_shortids, block_hash, block)
  280. # Now fetch the compact block using a normal non-announce getdata
  281. with mininode_lock:
  282. test_node.clear_block_announcement()
  283. inv = CInv(4, block_hash) # 4 == "CompactBlock"
  284. test_node.send_message(msg_getdata([inv]))
  285. wait_until(test_node.received_block_announcement, timeout=30)
  286. assert(test_node.received_block_announcement())
  287. # Now fetch and check the compact block
  288. header_and_shortids = None
  289. with mininode_lock:
  290. assert(test_node.last_cmpctblock is not None)
  291. # Convert the on-the-wire representation to absolute indexes
  292. header_and_shortids = HeaderAndShortIDs(test_node.last_cmpctblock.header_and_shortids)
  293. self.check_compactblock_construction_from_block(version, header_and_shortids, block_hash, block)
  294. def check_compactblock_construction_from_block(self, version, header_and_shortids, block_hash, block):
  295. # Check that we got the right block!
  296. header_and_shortids.header.calc_sha256()
  297. assert_equal(header_and_shortids.header.sha256, block_hash)
  298. # Make sure the prefilled_txn appears to have included the coinbase
  299. assert(len(header_and_shortids.prefilled_txn) >= 1)
  300. assert_equal(header_and_shortids.prefilled_txn[0].index, 0)
  301. # Check that all prefilled_txn entries match what's in the block.
  302. for entry in header_and_shortids.prefilled_txn:
  303. entry.tx.calc_sha256()
  304. # This checks the non-witness parts of the tx agree
  305. assert_equal(entry.tx.sha256, block.vtx[entry.index].sha256)
  306. # And this checks the witness
  307. wtxid = entry.tx.calc_sha256(True)
  308. if version == 2:
  309. assert_equal(wtxid, block.vtx[entry.index].calc_sha256(True))
  310. else:
  311. # Shouldn't have received a witness
  312. assert(entry.tx.wit.is_null())
  313. # Check that the cmpctblock message announced all the transactions.
  314. assert_equal(len(header_and_shortids.prefilled_txn) + len(header_and_shortids.shortids), len(block.vtx))
  315. # And now check that all the shortids are as expected as well.
  316. # Determine the siphash keys to use.
  317. [k0, k1] = header_and_shortids.get_siphash_keys()
  318. index = 0
  319. while index < len(block.vtx):
  320. if (len(header_and_shortids.prefilled_txn) > 0 and
  321. header_and_shortids.prefilled_txn[0].index == index):
  322. # Already checked prefilled transactions above
  323. header_and_shortids.prefilled_txn.pop(0)
  324. else:
  325. tx_hash = block.vtx[index].sha256
  326. if version == 2:
  327. tx_hash = block.vtx[index].calc_sha256(True)
  328. shortid = calculate_shortid(k0, k1, tx_hash)
  329. assert_equal(shortid, header_and_shortids.shortids[0])
  330. header_and_shortids.shortids.pop(0)
  331. index += 1
  332. # Test that bitcoind requests compact blocks when we announce new blocks
  333. # via header or inv, and that responding to getblocktxn causes the block
  334. # to be successfully reconstructed.
  335. # Post-segwit: upgraded nodes would only make this request of cb-version-2,
  336. # NODE_WITNESS peers. Unupgraded nodes would still make this request of
  337. # any cb-version-1-supporting peer.
  338. def test_compactblock_requests(self, node, test_node, version, segwit):
  339. # Try announcing a block with an inv or header, expect a compactblock
  340. # request
  341. for announce in ["inv", "header"]:
  342. block = self.build_block_on_tip(node, segwit=segwit)
  343. with mininode_lock:
  344. test_node.last_getdata = None
  345. if announce == "inv":
  346. test_node.send_message(msg_inv([CInv(2, block.sha256)]))
  347. success = wait_until(lambda: test_node.last_getheaders is not None, timeout=30)
  348. assert(success)
  349. test_node.send_header_for_blocks([block])
  350. else:
  351. test_node.send_header_for_blocks([block])
  352. success = wait_until(lambda: test_node.last_getdata is not None, timeout=30)
  353. assert(success)
  354. assert_equal(len(test_node.last_getdata.inv), 1)
  355. assert_equal(test_node.last_getdata.inv[0].type, 4)
  356. assert_equal(test_node.last_getdata.inv[0].hash, block.sha256)
  357. # Send back a compactblock message that omits the coinbase
  358. comp_block = HeaderAndShortIDs()
  359. comp_block.header = CBlockHeader(block)
  360. comp_block.nonce = 0
  361. [k0, k1] = comp_block.get_siphash_keys()
  362. coinbase_hash = block.vtx[0].sha256
  363. if version == 2:
  364. coinbase_hash = block.vtx[0].calc_sha256(True)
  365. comp_block.shortids = [
  366. calculate_shortid(k0, k1, coinbase_hash) ]
  367. test_node.send_and_ping(msg_cmpctblock(comp_block.to_p2p()))
  368. assert_equal(int(node.getbestblockhash(), 16), block.hashPrevBlock)
  369. # Expect a getblocktxn message.
  370. with mininode_lock:
  371. assert(test_node.last_getblocktxn is not None)
  372. absolute_indexes = test_node.last_getblocktxn.block_txn_request.to_absolute()
  373. assert_equal(absolute_indexes, [0]) # should be a coinbase request
  374. # Send the coinbase, and verify that the tip advances.
  375. if version == 2:
  376. msg = msg_witness_blocktxn()
  377. else:
  378. msg = msg_blocktxn()
  379. msg.block_transactions.blockhash = block.sha256
  380. msg.block_transactions.transactions = [block.vtx[0]]
  381. test_node.send_and_ping(msg)
  382. assert_equal(int(node.getbestblockhash(), 16), block.sha256)
  383. # Create a chain of transactions from given utxo, and add to a new block.
  384. def build_block_with_transactions(self, node, utxo, num_transactions):
  385. block = self.build_block_on_tip(node)
  386. for i in range(num_transactions):
  387. tx = CTransaction()
  388. tx.vin.append(CTxIn(COutPoint(utxo[0], utxo[1]), b''))
  389. tx.vout.append(CTxOut(utxo[2] - 1000, CScript([OP_TRUE])))
  390. tx.rehash()
  391. utxo = [tx.sha256, 0, tx.vout[0].nValue]
  392. block.vtx.append(tx)
  393. block.hashMerkleRoot = block.calc_merkle_root()
  394. block.solve()
  395. return block
  396. # Test that we only receive getblocktxn requests for transactions that the
  397. # node needs, and that responding to them causes the block to be
  398. # reconstructed.
  399. def test_getblocktxn_requests(self, node, test_node, version):
  400. with_witness = (version==2)
  401. def test_getblocktxn_response(compact_block, peer, expected_result):
  402. msg = msg_cmpctblock(compact_block.to_p2p())
  403. peer.send_and_ping(msg)
  404. with mininode_lock:
  405. assert(peer.last_getblocktxn is not None)
  406. absolute_indexes = peer.last_getblocktxn.block_txn_request.to_absolute()
  407. assert_equal(absolute_indexes, expected_result)
  408. def test_tip_after_message(node, peer, msg, tip):
  409. peer.send_and_ping(msg)
  410. assert_equal(int(node.getbestblockhash(), 16), tip)
  411. # First try announcing compactblocks that won't reconstruct, and verify
  412. # that we receive getblocktxn messages back.
  413. utxo = self.utxos.pop(0)
  414. block = self.build_block_with_transactions(node, utxo, 5)
  415. self.utxos.append([block.vtx[-1].sha256, 0, block.vtx[-1].vout[0].nValue])
  416. comp_block = HeaderAndShortIDs()
  417. comp_block.initialize_from_block(block, use_witness=with_witness)
  418. test_getblocktxn_response(comp_block, test_node, [1, 2, 3, 4, 5])
  419. msg_bt = msg_blocktxn()
  420. if with_witness:
  421. msg_bt = msg_witness_blocktxn() # serialize with witnesses
  422. msg_bt.block_transactions = BlockTransactions(block.sha256, block.vtx[1:])
  423. test_tip_after_message(node, test_node, msg_bt, block.sha256)
  424. utxo = self.utxos.pop(0)
  425. block = self.build_block_with_transactions(node, utxo, 5)
  426. self.utxos.append([block.vtx[-1].sha256, 0, block.vtx[-1].vout[0].nValue])
  427. # Now try interspersing the prefilled transactions
  428. comp_block.initialize_from_block(block, prefill_list=[0, 1, 5], use_witness=with_witness)
  429. test_getblocktxn_response(comp_block, test_node, [2, 3, 4])
  430. msg_bt.block_transactions = BlockTransactions(block.sha256, block.vtx[2:5])
  431. test_tip_after_message(node, test_node, msg_bt, block.sha256)
  432. # Now try giving one transaction ahead of time.
  433. utxo = self.utxos.pop(0)
  434. block = self.build_block_with_transactions(node, utxo, 5)
  435. self.utxos.append([block.vtx[-1].sha256, 0, block.vtx[-1].vout[0].nValue])
  436. test_node.send_and_ping(msg_tx(block.vtx[1]))
  437. assert(block.vtx[1].hash in node.getrawmempool())
  438. # Prefill 4 out of the 6 transactions, and verify that only the one
  439. # that was not in the mempool is requested.
  440. comp_block.initialize_from_block(block, prefill_list=[0, 2, 3, 4], use_witness=with_witness)
  441. test_getblocktxn_response(comp_block, test_node, [5])
  442. msg_bt.block_transactions = BlockTransactions(block.sha256, [block.vtx[5]])
  443. test_tip_after_message(node, test_node, msg_bt, block.sha256)
  444. # Now provide all transactions to the node before the block is
  445. # announced and verify reconstruction happens immediately.
  446. utxo = self.utxos.pop(0)
  447. block = self.build_block_with_transactions(node, utxo, 10)
  448. self.utxos.append([block.vtx[-1].sha256, 0, block.vtx[-1].vout[0].nValue])
  449. for tx in block.vtx[1:]:
  450. test_node.send_message(msg_tx(tx))
  451. test_node.sync_with_ping()
  452. # Make sure all transactions were accepted.
  453. mempool = node.getrawmempool()
  454. for tx in block.vtx[1:]:
  455. assert(tx.hash in mempool)
  456. # Clear out last request.
  457. with mininode_lock:
  458. test_node.last_getblocktxn = None
  459. # Send compact block
  460. comp_block.initialize_from_block(block, prefill_list=[0], use_witness=with_witness)
  461. test_tip_after_message(node, test_node, msg_cmpctblock(comp_block.to_p2p()), block.sha256)
  462. with mininode_lock:
  463. # Shouldn't have gotten a request for any transaction
  464. assert(test_node.last_getblocktxn is None)
  465. # Incorrectly responding to a getblocktxn shouldn't cause the block to be
  466. # permanently failed.
  467. def test_incorrect_blocktxn_response(self, node, test_node, version):
  468. if (len(self.utxos) == 0):
  469. self.make_utxos()
  470. utxo = self.utxos.pop(0)
  471. block = self.build_block_with_transactions(node, utxo, 10)
  472. self.utxos.append([block.vtx[-1].sha256, 0, block.vtx[-1].vout[0].nValue])
  473. # Relay the first 5 transactions from the block in advance
  474. for tx in block.vtx[1:6]:
  475. test_node.send_message(msg_tx(tx))
  476. test_node.sync_with_ping()
  477. # Make sure all transactions were accepted.
  478. mempool = node.getrawmempool()
  479. for tx in block.vtx[1:6]:
  480. assert(tx.hash in mempool)
  481. # Send compact block
  482. comp_block = HeaderAndShortIDs()
  483. comp_block.initialize_from_block(block, prefill_list=[0], use_witness=(version == 2))
  484. test_node.send_and_ping(msg_cmpctblock(comp_block.to_p2p()))
  485. absolute_indexes = []
  486. with mininode_lock:
  487. assert(test_node.last_getblocktxn is not None)
  488. absolute_indexes = test_node.last_getblocktxn.block_txn_request.to_absolute()
  489. assert_equal(absolute_indexes, [6, 7, 8, 9, 10])
  490. # Now give an incorrect response.
  491. # Note that it's possible for bitcoind to be smart enough to know we're
  492. # lying, since it could check to see if the shortid matches what we're
  493. # sending, and eg disconnect us for misbehavior. If that behavior
  494. # change were made, we could just modify this test by having a
  495. # different peer provide the block further down, so that we're still
  496. # verifying that the block isn't marked bad permanently. This is good
  497. # enough for now.
  498. msg = msg_blocktxn()
  499. if version==2:
  500. msg = msg_witness_blocktxn()
  501. msg.block_transactions = BlockTransactions(block.sha256, [block.vtx[5]] + block.vtx[7:])
  502. test_node.send_and_ping(msg)
  503. # Tip should not have updated
  504. assert_equal(int(node.getbestblockhash(), 16), block.hashPrevBlock)
  505. # We should receive a getdata request
  506. success = wait_until(lambda: test_node.last_getdata is not None, timeout=10)
  507. assert(success)
  508. assert_equal(len(test_node.last_getdata.inv), 1)
  509. assert(test_node.last_getdata.inv[0].type == 2 or test_node.last_getdata.inv[0].type == 2|MSG_WITNESS_FLAG)
  510. assert_equal(test_node.last_getdata.inv[0].hash, block.sha256)
  511. # Deliver the block
  512. if version==2:
  513. test_node.send_and_ping(msg_witness_block(block))
  514. else:
  515. test_node.send_and_ping(msg_block(block))
  516. assert_equal(int(node.getbestblockhash(), 16), block.sha256)
  517. def test_getblocktxn_handler(self, node, test_node, version):
  518. # bitcoind will not send blocktxn responses for blocks whose height is
  519. # more than 10 blocks deep.
  520. MAX_GETBLOCKTXN_DEPTH = 10
  521. chain_height = node.getblockcount()
  522. current_height = chain_height
  523. while (current_height >= chain_height - MAX_GETBLOCKTXN_DEPTH):
  524. block_hash = node.getblockhash(current_height)
  525. block = FromHex(CBlock(), node.getblock(block_hash, False))
  526. msg = msg_getblocktxn()
  527. msg.block_txn_request = BlockTransactionsRequest(int(block_hash, 16), [])
  528. num_to_request = random.randint(1, len(block.vtx))
  529. msg.block_txn_request.from_absolute(sorted(random.sample(range(len(block.vtx)), num_to_request)))
  530. test_node.send_message(msg)
  531. success = wait_until(lambda: test_node.last_blocktxn is not None, timeout=10)
  532. assert(success)
  533. [tx.calc_sha256() for tx in block.vtx]
  534. with mininode_lock:
  535. assert_equal(test_node.last_blocktxn.block_transactions.blockhash, int(block_hash, 16))
  536. all_indices = msg.block_txn_request.to_absolute()
  537. for index in all_indices:
  538. tx = test_node.last_blocktxn.block_transactions.transactions.pop(0)
  539. tx.calc_sha256()
  540. assert_equal(tx.sha256, block.vtx[index].sha256)
  541. if version == 1:
  542. # Witnesses should have been stripped
  543. assert(tx.wit.is_null())
  544. else:
  545. # Check that the witness matches
  546. assert_equal(tx.calc_sha256(True), block.vtx[index].calc_sha256(True))
  547. test_node.last_blocktxn = None
  548. current_height -= 1
  549. # Next request should send a full block response, as we're past the
  550. # allowed depth for a blocktxn response.
  551. block_hash = node.getblockhash(current_height)
  552. msg.block_txn_request = BlockTransactionsRequest(int(block_hash, 16), [0])
  553. with mininode_lock:
  554. test_node.last_block = None
  555. test_node.last_blocktxn = None
  556. test_node.send_and_ping(msg)
  557. with mininode_lock:
  558. test_node.last_block.block.calc_sha256()
  559. assert_equal(test_node.last_block.block.sha256, int(block_hash, 16))
  560. assert_equal(test_node.last_blocktxn, None)
  561. def test_compactblocks_not_at_tip(self, node, test_node):
  562. # Test that requesting old compactblocks doesn't work.
  563. MAX_CMPCTBLOCK_DEPTH = 5
  564. new_blocks = []
  565. for i in range(MAX_CMPCTBLOCK_DEPTH + 1):
  566. test_node.clear_block_announcement()
  567. new_blocks.append(node.generate(1)[0])
  568. wait_until(test_node.received_block_announcement, timeout=30)
  569. test_node.clear_block_announcement()
  570. test_node.send_message(msg_getdata([CInv(4, int(new_blocks[0], 16))]))
  571. success = wait_until(lambda: test_node.last_cmpctblock is not None, timeout=30)
  572. assert(success)
  573. test_node.clear_block_announcement()
  574. node.generate(1)
  575. wait_until(test_node.received_block_announcement, timeout=30)
  576. test_node.clear_block_announcement()
  577. with mininode_lock:
  578. test_node.last_block = None
  579. test_node.send_message(msg_getdata([CInv(4, int(new_blocks[0], 16))]))
  580. success = wait_until(lambda: test_node.last_block is not None, timeout=30)
  581. assert(success)
  582. with mininode_lock:
  583. test_node.last_block.block.calc_sha256()
  584. assert_equal(test_node.last_block.block.sha256, int(new_blocks[0], 16))
  585. # Generate an old compactblock, and verify that it's not accepted.
  586. cur_height = node.getblockcount()
  587. hashPrevBlock = int(node.getblockhash(cur_height-5), 16)
  588. block = self.build_block_on_tip(node)
  589. block.hashPrevBlock = hashPrevBlock
  590. block.solve()
  591. comp_block = HeaderAndShortIDs()
  592. comp_block.initialize_from_block(block)
  593. test_node.send_and_ping(msg_cmpctblock(comp_block.to_p2p()))
  594. tips = node.getchaintips()
  595. found = False
  596. for x in tips:
  597. if x["hash"] == block.hash:
  598. assert_equal(x["status"], "headers-only")
  599. found = True
  600. break
  601. assert(found)
  602. # Requesting this block via getblocktxn should silently fail
  603. # (to avoid fingerprinting attacks).
  604. msg = msg_getblocktxn()
  605. msg.block_txn_request = BlockTransactionsRequest(block.sha256, [0])
  606. with mininode_lock:
  607. test_node.last_blocktxn = None
  608. test_node.send_and_ping(msg)
  609. with mininode_lock:
  610. assert(test_node.last_blocktxn is None)
  611. def activate_segwit(self, node):
  612. node.generate(144*3)
  613. assert_equal(get_bip9_status(node, "segwit")["status"], 'active')
  614. def test_end_to_end_block_relay(self, node, listeners):
  615. utxo = self.utxos.pop(0)
  616. block = self.build_block_with_transactions(node, utxo, 10)
  617. [l.clear_block_announcement() for l in listeners]
  618. # ToHex() won't serialize with witness, but this block has no witnesses
  619. # anyway. TODO: repeat this test with witness tx's to a segwit node.
  620. node.submitblock(ToHex(block))
  621. for l in listeners:
  622. wait_until(lambda: l.received_block_announcement(), timeout=30)
  623. with mininode_lock:
  624. for l in listeners:
  625. assert(l.last_cmpctblock is not None)
  626. l.last_cmpctblock.header_and_shortids.header.calc_sha256()
  627. assert_equal(l.last_cmpctblock.header_and_shortids.header.sha256, block.sha256)
  628. # Test that we don't get disconnected if we relay a compact block with valid header,
  629. # but invalid transactions.
  630. def test_invalid_tx_in_compactblock(self, node, test_node, use_segwit):
  631. assert(len(self.utxos))
  632. utxo = self.utxos[0]
  633. block = self.build_block_with_transactions(node, utxo, 5)
  634. del block.vtx[3]
  635. block.hashMerkleRoot = block.calc_merkle_root()
  636. if use_segwit:
  637. # If we're testing with segwit, also drop the coinbase witness,
  638. # but include the witness commitment.
  639. add_witness_commitment(block)
  640. block.vtx[0].wit.vtxinwit = []
  641. block.solve()
  642. # Now send the compact block with all transactions prefilled, and
  643. # verify that we don't get disconnected.
  644. comp_block = HeaderAndShortIDs()
  645. comp_block.initialize_from_block(block, prefill_list=[0, 1, 2, 3, 4], use_witness=use_segwit)
  646. msg = msg_cmpctblock(comp_block.to_p2p())
  647. test_node.send_and_ping(msg)
  648. # Check that the tip didn't advance
  649. assert(int(node.getbestblockhash(), 16) is not block.sha256)
  650. test_node.sync_with_ping()
  651. # Helper for enabling cb announcements
  652. # Send the sendcmpct request and sync headers
  653. def request_cb_announcements(self, peer, node, version):
  654. tip = node.getbestblockhash()
  655. peer.get_headers(locator=[int(tip, 16)], hashstop=0)
  656. msg = msg_sendcmpct()
  657. msg.version = version
  658. msg.announce = True
  659. peer.send_and_ping(msg)
  660. def test_compactblock_reconstruction_multiple_peers(self, node, stalling_peer, delivery_peer):
  661. assert(len(self.utxos))
  662. def announce_cmpct_block(node, peer):
  663. utxo = self.utxos.pop(0)
  664. block = self.build_block_with_transactions(node, utxo, 5)
  665. cmpct_block = HeaderAndShortIDs()
  666. cmpct_block.initialize_from_block(block)
  667. msg = msg_cmpctblock(cmpct_block.to_p2p())
  668. peer.send_and_ping(msg)
  669. with mininode_lock:
  670. assert(peer.last_getblocktxn is not None)
  671. return block, cmpct_block
  672. block, cmpct_block = announce_cmpct_block(node, stalling_peer)
  673. for tx in block.vtx[1:]:
  674. delivery_peer.send_message(msg_tx(tx))
  675. delivery_peer.sync_with_ping()
  676. mempool = node.getrawmempool()
  677. for tx in block.vtx[1:]:
  678. assert(tx.hash in mempool)
  679. delivery_peer.send_and_ping(msg_cmpctblock(cmpct_block.to_p2p()))
  680. assert_equal(int(node.getbestblockhash(), 16), block.sha256)
  681. self.utxos.append([block.vtx[-1].sha256, 0, block.vtx[-1].vout[0].nValue])
  682. # Now test that delivering an invalid compact block won't break relay
  683. block, cmpct_block = announce_cmpct_block(node, stalling_peer)
  684. for tx in block.vtx[1:]:
  685. delivery_peer.send_message(msg_tx(tx))
  686. delivery_peer.sync_with_ping()
  687. cmpct_block.prefilled_txn[0].tx.wit.vtxinwit = [ CTxInWitness() ]
  688. cmpct_block.prefilled_txn[0].tx.wit.vtxinwit[0].scriptWitness.stack = [ser_uint256(0)]
  689. cmpct_block.use_witness = True
  690. delivery_peer.send_and_ping(msg_cmpctblock(cmpct_block.to_p2p()))
  691. assert(int(node.getbestblockhash(), 16) != block.sha256)
  692. msg = msg_blocktxn()
  693. msg.block_transactions.blockhash = block.sha256
  694. msg.block_transactions.transactions = block.vtx[1:]
  695. stalling_peer.send_and_ping(msg)
  696. assert_equal(int(node.getbestblockhash(), 16), block.sha256)
  697. def run_test(self):
  698. # Setup the p2p connections and start up the network thread.
  699. self.test_node = TestNode()
  700. self.segwit_node = TestNode()
  701. self.old_node = TestNode() # version 1 peer <--> segwit node
  702. connections = []
  703. connections.append(NodeConn('127.0.0.1', p2p_port(0), self.nodes[0], self.test_node))
  704. connections.append(NodeConn('127.0.0.1', p2p_port(1), self.nodes[1],
  705. self.segwit_node, services=NODE_NETWORK|NODE_WITNESS))
  706. connections.append(NodeConn('127.0.0.1', p2p_port(1), self.nodes[1],
  707. self.old_node, services=NODE_NETWORK))
  708. self.test_node.add_connection(connections[0])
  709. self.segwit_node.add_connection(connections[1])
  710. self.old_node.add_connection(connections[2])
  711. NetworkThread().start() # Start up network handling in another thread
  712. # Test logic begins here
  713. self.test_node.wait_for_verack()
  714. # We will need UTXOs to construct transactions in later tests.
  715. self.make_utxos()
  716. self.log.info("Running tests, pre-segwit activation:")
  717. self.log.info("Testing SENDCMPCT p2p message... ")
  718. self.test_sendcmpct(self.nodes[0], self.test_node, 1)
  719. sync_blocks(self.nodes)
  720. self.test_sendcmpct(self.nodes[1], self.segwit_node, 2, old_node=self.old_node)
  721. sync_blocks(self.nodes)
  722. self.log.info("Testing compactblock construction...")
  723. self.test_compactblock_construction(self.nodes[0], self.test_node, 1, False)
  724. sync_blocks(self.nodes)
  725. self.test_compactblock_construction(self.nodes[1], self.segwit_node, 2, False)
  726. sync_blocks(self.nodes)
  727. self.log.info("Testing compactblock requests... ")
  728. self.test_compactblock_requests(self.nodes[0], self.test_node, 1, False)
  729. sync_blocks(self.nodes)
  730. self.test_compactblock_requests(self.nodes[1], self.segwit_node, 2, False)
  731. sync_blocks(self.nodes)
  732. self.log.info("Testing getblocktxn requests...")
  733. self.test_getblocktxn_requests(self.nodes[0], self.test_node, 1)
  734. sync_blocks(self.nodes)
  735. self.test_getblocktxn_requests(self.nodes[1], self.segwit_node, 2)
  736. sync_blocks(self.nodes)
  737. self.log.info("Testing getblocktxn handler...")
  738. self.test_getblocktxn_handler(self.nodes[0], self.test_node, 1)
  739. sync_blocks(self.nodes)
  740. self.test_getblocktxn_handler(self.nodes[1], self.segwit_node, 2)
  741. self.test_getblocktxn_handler(self.nodes[1], self.old_node, 1)
  742. sync_blocks(self.nodes)
  743. self.log.info("Testing compactblock requests/announcements not at chain tip...")
  744. self.test_compactblocks_not_at_tip(self.nodes[0], self.test_node)
  745. sync_blocks(self.nodes)
  746. self.test_compactblocks_not_at_tip(self.nodes[1], self.segwit_node)
  747. self.test_compactblocks_not_at_tip(self.nodes[1], self.old_node)
  748. sync_blocks(self.nodes)
  749. self.log.info("Testing handling of incorrect blocktxn responses...")
  750. self.test_incorrect_blocktxn_response(self.nodes[0], self.test_node, 1)
  751. sync_blocks(self.nodes)
  752. self.test_incorrect_blocktxn_response(self.nodes[1], self.segwit_node, 2)
  753. sync_blocks(self.nodes)
  754. # End-to-end block relay tests
  755. self.log.info("Testing end-to-end block relay...")
  756. self.request_cb_announcements(self.test_node, self.nodes[0], 1)
  757. self.request_cb_announcements(self.old_node, self.nodes[1], 1)
  758. self.request_cb_announcements(self.segwit_node, self.nodes[1], 2)
  759. self.test_end_to_end_block_relay(self.nodes[0], [self.segwit_node, self.test_node, self.old_node])
  760. self.test_end_to_end_block_relay(self.nodes[1], [self.segwit_node, self.test_node, self.old_node])
  761. self.log.info("Testing handling of invalid compact blocks...")
  762. self.test_invalid_tx_in_compactblock(self.nodes[0], self.test_node, False)
  763. self.test_invalid_tx_in_compactblock(self.nodes[1], self.segwit_node, False)
  764. self.test_invalid_tx_in_compactblock(self.nodes[1], self.old_node, False)
  765. self.log.info("Testing reconstructing compact blocks from all peers...")
  766. self.test_compactblock_reconstruction_multiple_peers(self.nodes[1], self.segwit_node, self.old_node)
  767. sync_blocks(self.nodes)
  768. # Advance to segwit activation
  769. self.log.info("Advancing to segwit activation")
  770. self.activate_segwit(self.nodes[1])
  771. self.log.info("Running tests, post-segwit activation...")
  772. self.log.info("Testing compactblock construction...")
  773. self.test_compactblock_construction(self.nodes[1], self.old_node, 1, True)
  774. self.test_compactblock_construction(self.nodes[1], self.segwit_node, 2, True)
  775. sync_blocks(self.nodes)
  776. self.log.info("Testing compactblock requests (unupgraded node)... ")
  777. self.test_compactblock_requests(self.nodes[0], self.test_node, 1, True)
  778. self.log.info("Testing getblocktxn requests (unupgraded node)...")
  779. self.test_getblocktxn_requests(self.nodes[0], self.test_node, 1)
  780. # Need to manually sync node0 and node1, because post-segwit activation,
  781. # node1 will not download blocks from node0.
  782. self.log.info("Syncing nodes...")
  783. assert(self.nodes[0].getbestblockhash() != self.nodes[1].getbestblockhash())
  784. while (self.nodes[0].getblockcount() > self.nodes[1].getblockcount()):
  785. block_hash = self.nodes[0].getblockhash(self.nodes[1].getblockcount()+1)
  786. self.nodes[1].submitblock(self.nodes[0].getblock(block_hash, False))
  787. assert_equal(self.nodes[0].getbestblockhash(), self.nodes[1].getbestblockhash())
  788. self.log.info("Testing compactblock requests (segwit node)... ")
  789. self.test_compactblock_requests(self.nodes[1], self.segwit_node, 2, True)
  790. self.log.info("Testing getblocktxn requests (segwit node)...")
  791. self.test_getblocktxn_requests(self.nodes[1], self.segwit_node, 2)
  792. sync_blocks(self.nodes)
  793. self.log.info("Testing getblocktxn handler (segwit node should return witnesses)...")
  794. self.test_getblocktxn_handler(self.nodes[1], self.segwit_node, 2)
  795. self.test_getblocktxn_handler(self.nodes[1], self.old_node, 1)
  796. # Test that if we submitblock to node1, we'll get a compact block
  797. # announcement to all peers.
  798. # (Post-segwit activation, blocks won't propagate from node0 to node1
  799. # automatically, so don't bother testing a block announced to node0.)
  800. self.log.info("Testing end-to-end block relay...")
  801. self.request_cb_announcements(self.test_node, self.nodes[0], 1)
  802. self.request_cb_announcements(self.old_node, self.nodes[1], 1)
  803. self.request_cb_announcements(self.segwit_node, self.nodes[1], 2)
  804. self.test_end_to_end_block_relay(self.nodes[1], [self.segwit_node, self.test_node, self.old_node])
  805. self.log.info("Testing handling of invalid compact blocks...")
  806. self.test_invalid_tx_in_compactblock(self.nodes[0], self.test_node, False)
  807. self.test_invalid_tx_in_compactblock(self.nodes[1], self.segwit_node, True)
  808. self.test_invalid_tx_in_compactblock(self.nodes[1], self.old_node, True)
  809. self.log.info("Testing invalid index in cmpctblock message...")
  810. self.test_invalid_cmpctblock_message()
  811. if __name__ == '__main__':
  812. CompactBlocksTest().main()