You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

p2p-compactblocks.py 43KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929
  1. #!/usr/bin/env python3
  2. # Copyright (c) 2016 The Bitcoin Core developers
  3. # Distributed under the MIT software license, see the accompanying
  4. # file COPYING or http://www.opensource.org/licenses/mit-license.php.
  5. """Test compact blocks (BIP 152).
  6. Version 1 compact blocks are pre-segwit (txids)
  7. Version 2 compact blocks are post-segwit (wtxids)
  8. """
  9. from test_framework.mininode import *
  10. from test_framework.test_framework import BitcoinTestFramework
  11. from test_framework.util import *
  12. from test_framework.blocktools import create_block, create_coinbase, add_witness_commitment
  13. from test_framework.script import CScript, OP_TRUE
  14. # TestNode: A peer we use to send messages to bitcoind, and store responses.
  15. class TestNode(NodeConnCB):
  16. def __init__(self):
  17. super().__init__()
  18. self.last_sendcmpct = []
  19. self.block_announced = False
  20. # Store the hashes of blocks we've seen announced.
  21. # This is for synchronizing the p2p message traffic,
  22. # so we can eg wait until a particular block is announced.
  23. self.announced_blockhashes = set()
  24. def on_sendcmpct(self, conn, message):
  25. self.last_sendcmpct.append(message)
  26. def on_cmpctblock(self, conn, message):
  27. self.block_announced = True
  28. self.last_message["cmpctblock"].header_and_shortids.header.calc_sha256()
  29. self.announced_blockhashes.add(self.last_message["cmpctblock"].header_and_shortids.header.sha256)
  30. def on_headers(self, conn, message):
  31. self.block_announced = True
  32. for x in self.last_message["headers"].headers:
  33. x.calc_sha256()
  34. self.announced_blockhashes.add(x.sha256)
  35. def on_inv(self, conn, message):
  36. for x in self.last_message["inv"].inv:
  37. if x.type == 2:
  38. self.block_announced = True
  39. self.announced_blockhashes.add(x.hash)
  40. # Requires caller to hold mininode_lock
  41. def received_block_announcement(self):
  42. return self.block_announced
  43. def clear_block_announcement(self):
  44. with mininode_lock:
  45. self.block_announced = False
  46. self.last_message.pop("inv", None)
  47. self.last_message.pop("headers", None)
  48. self.last_message.pop("cmpctblock", None)
  49. def get_headers(self, locator, hashstop):
  50. msg = msg_getheaders()
  51. msg.locator.vHave = locator
  52. msg.hashstop = hashstop
  53. self.connection.send_message(msg)
  54. def send_header_for_blocks(self, new_blocks):
  55. headers_message = msg_headers()
  56. headers_message.headers = [CBlockHeader(b) for b in new_blocks]
  57. self.send_message(headers_message)
  58. def request_headers_and_sync(self, locator, hashstop=0):
  59. self.clear_block_announcement()
  60. self.get_headers(locator, hashstop)
  61. wait_until(self.received_block_announcement, timeout=30, lock=mininode_lock)
  62. self.clear_block_announcement()
  63. # Block until a block announcement for a particular block hash is
  64. # received.
  65. def wait_for_block_announcement(self, block_hash, timeout=30):
  66. def received_hash():
  67. return (block_hash in self.announced_blockhashes)
  68. wait_until(received_hash, timeout=timeout, lock=mininode_lock)
  69. def send_await_disconnect(self, message, timeout=30):
  70. """Sends a message to the node and wait for disconnect.
  71. This is used when we want to send a message into the node that we expect
  72. will get us disconnected, eg an invalid block."""
  73. self.send_message(message)
  74. wait_until(lambda: not self.connected, timeout=timeout, lock=mininode_lock)
  75. class CompactBlocksTest(BitcoinTestFramework):
  76. def set_test_params(self):
  77. self.setup_clean_chain = True
  78. # Node0 = pre-segwit, node1 = segwit-aware
  79. self.num_nodes = 2
  80. self.extra_args = [["-vbparams=segwit:0:0"], ["-txindex"]]
  81. self.utxos = []
  82. def build_block_on_tip(self, node, segwit=False):
  83. height = node.getblockcount()
  84. tip = node.getbestblockhash()
  85. mtp = node.getblockheader(tip)['mediantime']
  86. block = create_block(int(tip, 16), create_coinbase(height + 1), mtp + 1)
  87. block.nVersion = 4
  88. if segwit:
  89. add_witness_commitment(block)
  90. block.solve()
  91. return block
  92. # Create 10 more anyone-can-spend utxo's for testing.
  93. def make_utxos(self):
  94. # Doesn't matter which node we use, just use node0.
  95. block = self.build_block_on_tip(self.nodes[0])
  96. self.test_node.send_and_ping(msg_block(block))
  97. assert(int(self.nodes[0].getbestblockhash(), 16) == block.sha256)
  98. self.nodes[0].generate(100)
  99. total_value = block.vtx[0].vout[0].nValue
  100. out_value = total_value // 10
  101. tx = CTransaction()
  102. tx.vin.append(CTxIn(COutPoint(block.vtx[0].sha256, 0), b''))
  103. for i in range(10):
  104. tx.vout.append(CTxOut(out_value, CScript([OP_TRUE])))
  105. tx.rehash()
  106. block2 = self.build_block_on_tip(self.nodes[0])
  107. block2.vtx.append(tx)
  108. block2.hashMerkleRoot = block2.calc_merkle_root()
  109. block2.solve()
  110. self.test_node.send_and_ping(msg_block(block2))
  111. assert_equal(int(self.nodes[0].getbestblockhash(), 16), block2.sha256)
  112. self.utxos.extend([[tx.sha256, i, out_value] for i in range(10)])
  113. return
  114. # Test "sendcmpct" (between peers preferring the same version):
  115. # - No compact block announcements unless sendcmpct is sent.
  116. # - If sendcmpct is sent with version > preferred_version, the message is ignored.
  117. # - If sendcmpct is sent with boolean 0, then block announcements are not
  118. # made with compact blocks.
  119. # - If sendcmpct is then sent with boolean 1, then new block announcements
  120. # are made with compact blocks.
  121. # If old_node is passed in, request compact blocks with version=preferred-1
  122. # and verify that it receives block announcements via compact block.
  123. def test_sendcmpct(self, node, test_node, preferred_version, old_node=None):
  124. # Make sure we get a SENDCMPCT message from our peer
  125. def received_sendcmpct():
  126. return (len(test_node.last_sendcmpct) > 0)
  127. wait_until(received_sendcmpct, timeout=30, lock=mininode_lock)
  128. with mininode_lock:
  129. # Check that the first version received is the preferred one
  130. assert_equal(test_node.last_sendcmpct[0].version, preferred_version)
  131. # And that we receive versions down to 1.
  132. assert_equal(test_node.last_sendcmpct[-1].version, 1)
  133. test_node.last_sendcmpct = []
  134. tip = int(node.getbestblockhash(), 16)
  135. def check_announcement_of_new_block(node, peer, predicate):
  136. peer.clear_block_announcement()
  137. block_hash = int(node.generate(1)[0], 16)
  138. peer.wait_for_block_announcement(block_hash, timeout=30)
  139. assert(peer.block_announced)
  140. with mininode_lock:
  141. assert predicate(peer), (
  142. "block_hash={!r}, cmpctblock={!r}, inv={!r}".format(
  143. block_hash, peer.last_message.get("cmpctblock", None), peer.last_message.get("inv", None)))
  144. # We shouldn't get any block announcements via cmpctblock yet.
  145. check_announcement_of_new_block(node, test_node, lambda p: "cmpctblock" not in p.last_message)
  146. # Try one more time, this time after requesting headers.
  147. test_node.request_headers_and_sync(locator=[tip])
  148. check_announcement_of_new_block(node, test_node, lambda p: "cmpctblock" not in p.last_message and "inv" in p.last_message)
  149. # Test a few ways of using sendcmpct that should NOT
  150. # result in compact block announcements.
  151. # Before each test, sync the headers chain.
  152. test_node.request_headers_and_sync(locator=[tip])
  153. # Now try a SENDCMPCT message with too-high version
  154. sendcmpct = msg_sendcmpct()
  155. sendcmpct.version = preferred_version+1
  156. sendcmpct.announce = True
  157. test_node.send_and_ping(sendcmpct)
  158. check_announcement_of_new_block(node, test_node, lambda p: "cmpctblock" not in p.last_message)
  159. # Headers sync before next test.
  160. test_node.request_headers_and_sync(locator=[tip])
  161. # Now try a SENDCMPCT message with valid version, but announce=False
  162. sendcmpct.version = preferred_version
  163. sendcmpct.announce = False
  164. test_node.send_and_ping(sendcmpct)
  165. check_announcement_of_new_block(node, test_node, lambda p: "cmpctblock" not in p.last_message)
  166. # Headers sync before next test.
  167. test_node.request_headers_and_sync(locator=[tip])
  168. # Finally, try a SENDCMPCT message with announce=True
  169. sendcmpct.version = preferred_version
  170. sendcmpct.announce = True
  171. test_node.send_and_ping(sendcmpct)
  172. check_announcement_of_new_block(node, test_node, lambda p: "cmpctblock" in p.last_message)
  173. # Try one more time (no headers sync should be needed!)
  174. check_announcement_of_new_block(node, test_node, lambda p: "cmpctblock" in p.last_message)
  175. # Try one more time, after turning on sendheaders
  176. test_node.send_and_ping(msg_sendheaders())
  177. check_announcement_of_new_block(node, test_node, lambda p: "cmpctblock" in p.last_message)
  178. # Try one more time, after sending a version-1, announce=false message.
  179. sendcmpct.version = preferred_version-1
  180. sendcmpct.announce = False
  181. test_node.send_and_ping(sendcmpct)
  182. check_announcement_of_new_block(node, test_node, lambda p: "cmpctblock" in p.last_message)
  183. # Now turn off announcements
  184. sendcmpct.version = preferred_version
  185. sendcmpct.announce = False
  186. test_node.send_and_ping(sendcmpct)
  187. check_announcement_of_new_block(node, test_node, lambda p: "cmpctblock" not in p.last_message and "headers" in p.last_message)
  188. if old_node is not None:
  189. # Verify that a peer using an older protocol version can receive
  190. # announcements from this node.
  191. sendcmpct.version = preferred_version-1
  192. sendcmpct.announce = True
  193. old_node.send_and_ping(sendcmpct)
  194. # Header sync
  195. old_node.request_headers_and_sync(locator=[tip])
  196. check_announcement_of_new_block(node, old_node, lambda p: "cmpctblock" in p.last_message)
  197. # This test actually causes bitcoind to (reasonably!) disconnect us, so do this last.
  198. def test_invalid_cmpctblock_message(self):
  199. self.nodes[0].generate(101)
  200. block = self.build_block_on_tip(self.nodes[0])
  201. cmpct_block = P2PHeaderAndShortIDs()
  202. cmpct_block.header = CBlockHeader(block)
  203. cmpct_block.prefilled_txn_length = 1
  204. # This index will be too high
  205. prefilled_txn = PrefilledTransaction(1, block.vtx[0])
  206. cmpct_block.prefilled_txn = [prefilled_txn]
  207. self.test_node.send_await_disconnect(msg_cmpctblock(cmpct_block))
  208. assert_equal(int(self.nodes[0].getbestblockhash(), 16), block.hashPrevBlock)
  209. # Compare the generated shortids to what we expect based on BIP 152, given
  210. # bitcoind's choice of nonce.
  211. def test_compactblock_construction(self, node, test_node, version, use_witness_address):
  212. # Generate a bunch of transactions.
  213. node.generate(101)
  214. num_transactions = 25
  215. address = node.getnewaddress()
  216. if use_witness_address:
  217. # Want at least one segwit spend, so move all funds to
  218. # a witness address.
  219. address = node.addwitnessaddress(address)
  220. value_to_send = node.getbalance()
  221. node.sendtoaddress(address, satoshi_round(value_to_send-Decimal(0.1)))
  222. node.generate(1)
  223. segwit_tx_generated = False
  224. for i in range(num_transactions):
  225. txid = node.sendtoaddress(address, 0.1)
  226. hex_tx = node.gettransaction(txid)["hex"]
  227. tx = FromHex(CTransaction(), hex_tx)
  228. if not tx.wit.is_null():
  229. segwit_tx_generated = True
  230. if use_witness_address:
  231. assert(segwit_tx_generated) # check that our test is not broken
  232. # Wait until we've seen the block announcement for the resulting tip
  233. tip = int(node.getbestblockhash(), 16)
  234. test_node.wait_for_block_announcement(tip)
  235. # Make sure we will receive a fast-announce compact block
  236. self.request_cb_announcements(test_node, node, version)
  237. # Now mine a block, and look at the resulting compact block.
  238. test_node.clear_block_announcement()
  239. block_hash = int(node.generate(1)[0], 16)
  240. # Store the raw block in our internal format.
  241. block = FromHex(CBlock(), node.getblock("%02x" % block_hash, False))
  242. [tx.calc_sha256() for tx in block.vtx]
  243. block.rehash()
  244. # Wait until the block was announced (via compact blocks)
  245. wait_until(test_node.received_block_announcement, timeout=30, lock=mininode_lock)
  246. # Now fetch and check the compact block
  247. header_and_shortids = None
  248. with mininode_lock:
  249. assert("cmpctblock" in test_node.last_message)
  250. # Convert the on-the-wire representation to absolute indexes
  251. header_and_shortids = HeaderAndShortIDs(test_node.last_message["cmpctblock"].header_and_shortids)
  252. self.check_compactblock_construction_from_block(version, header_and_shortids, block_hash, block)
  253. # Now fetch the compact block using a normal non-announce getdata
  254. with mininode_lock:
  255. test_node.clear_block_announcement()
  256. inv = CInv(4, block_hash) # 4 == "CompactBlock"
  257. test_node.send_message(msg_getdata([inv]))
  258. wait_until(test_node.received_block_announcement, timeout=30, lock=mininode_lock)
  259. # Now fetch and check the compact block
  260. header_and_shortids = None
  261. with mininode_lock:
  262. assert("cmpctblock" in test_node.last_message)
  263. # Convert the on-the-wire representation to absolute indexes
  264. header_and_shortids = HeaderAndShortIDs(test_node.last_message["cmpctblock"].header_and_shortids)
  265. self.check_compactblock_construction_from_block(version, header_and_shortids, block_hash, block)
  266. def check_compactblock_construction_from_block(self, version, header_and_shortids, block_hash, block):
  267. # Check that we got the right block!
  268. header_and_shortids.header.calc_sha256()
  269. assert_equal(header_and_shortids.header.sha256, block_hash)
  270. # Make sure the prefilled_txn appears to have included the coinbase
  271. assert(len(header_and_shortids.prefilled_txn) >= 1)
  272. assert_equal(header_and_shortids.prefilled_txn[0].index, 0)
  273. # Check that all prefilled_txn entries match what's in the block.
  274. for entry in header_and_shortids.prefilled_txn:
  275. entry.tx.calc_sha256()
  276. # This checks the non-witness parts of the tx agree
  277. assert_equal(entry.tx.sha256, block.vtx[entry.index].sha256)
  278. # And this checks the witness
  279. wtxid = entry.tx.calc_sha256(True)
  280. if version == 2:
  281. assert_equal(wtxid, block.vtx[entry.index].calc_sha256(True))
  282. else:
  283. # Shouldn't have received a witness
  284. assert(entry.tx.wit.is_null())
  285. # Check that the cmpctblock message announced all the transactions.
  286. assert_equal(len(header_and_shortids.prefilled_txn) + len(header_and_shortids.shortids), len(block.vtx))
  287. # And now check that all the shortids are as expected as well.
  288. # Determine the siphash keys to use.
  289. [k0, k1] = header_and_shortids.get_siphash_keys()
  290. index = 0
  291. while index < len(block.vtx):
  292. if (len(header_and_shortids.prefilled_txn) > 0 and
  293. header_and_shortids.prefilled_txn[0].index == index):
  294. # Already checked prefilled transactions above
  295. header_and_shortids.prefilled_txn.pop(0)
  296. else:
  297. tx_hash = block.vtx[index].sha256
  298. if version == 2:
  299. tx_hash = block.vtx[index].calc_sha256(True)
  300. shortid = calculate_shortid(k0, k1, tx_hash)
  301. assert_equal(shortid, header_and_shortids.shortids[0])
  302. header_and_shortids.shortids.pop(0)
  303. index += 1
  304. # Test that bitcoind requests compact blocks when we announce new blocks
  305. # via header or inv, and that responding to getblocktxn causes the block
  306. # to be successfully reconstructed.
  307. # Post-segwit: upgraded nodes would only make this request of cb-version-2,
  308. # NODE_WITNESS peers. Unupgraded nodes would still make this request of
  309. # any cb-version-1-supporting peer.
  310. def test_compactblock_requests(self, node, test_node, version, segwit):
  311. # Try announcing a block with an inv or header, expect a compactblock
  312. # request
  313. for announce in ["inv", "header"]:
  314. block = self.build_block_on_tip(node, segwit=segwit)
  315. with mininode_lock:
  316. test_node.last_message.pop("getdata", None)
  317. if announce == "inv":
  318. test_node.send_message(msg_inv([CInv(2, block.sha256)]))
  319. wait_until(lambda: "getheaders" in test_node.last_message, timeout=30, lock=mininode_lock)
  320. test_node.send_header_for_blocks([block])
  321. else:
  322. test_node.send_header_for_blocks([block])
  323. wait_until(lambda: "getdata" in test_node.last_message, timeout=30, lock=mininode_lock)
  324. assert_equal(len(test_node.last_message["getdata"].inv), 1)
  325. assert_equal(test_node.last_message["getdata"].inv[0].type, 4)
  326. assert_equal(test_node.last_message["getdata"].inv[0].hash, block.sha256)
  327. # Send back a compactblock message that omits the coinbase
  328. comp_block = HeaderAndShortIDs()
  329. comp_block.header = CBlockHeader(block)
  330. comp_block.nonce = 0
  331. [k0, k1] = comp_block.get_siphash_keys()
  332. coinbase_hash = block.vtx[0].sha256
  333. if version == 2:
  334. coinbase_hash = block.vtx[0].calc_sha256(True)
  335. comp_block.shortids = [
  336. calculate_shortid(k0, k1, coinbase_hash) ]
  337. test_node.send_and_ping(msg_cmpctblock(comp_block.to_p2p()))
  338. assert_equal(int(node.getbestblockhash(), 16), block.hashPrevBlock)
  339. # Expect a getblocktxn message.
  340. with mininode_lock:
  341. assert("getblocktxn" in test_node.last_message)
  342. absolute_indexes = test_node.last_message["getblocktxn"].block_txn_request.to_absolute()
  343. assert_equal(absolute_indexes, [0]) # should be a coinbase request
  344. # Send the coinbase, and verify that the tip advances.
  345. if version == 2:
  346. msg = msg_witness_blocktxn()
  347. else:
  348. msg = msg_blocktxn()
  349. msg.block_transactions.blockhash = block.sha256
  350. msg.block_transactions.transactions = [block.vtx[0]]
  351. test_node.send_and_ping(msg)
  352. assert_equal(int(node.getbestblockhash(), 16), block.sha256)
  353. # Create a chain of transactions from given utxo, and add to a new block.
  354. def build_block_with_transactions(self, node, utxo, num_transactions):
  355. block = self.build_block_on_tip(node)
  356. for i in range(num_transactions):
  357. tx = CTransaction()
  358. tx.vin.append(CTxIn(COutPoint(utxo[0], utxo[1]), b''))
  359. tx.vout.append(CTxOut(utxo[2] - 1000, CScript([OP_TRUE])))
  360. tx.rehash()
  361. utxo = [tx.sha256, 0, tx.vout[0].nValue]
  362. block.vtx.append(tx)
  363. block.hashMerkleRoot = block.calc_merkle_root()
  364. block.solve()
  365. return block
  366. # Test that we only receive getblocktxn requests for transactions that the
  367. # node needs, and that responding to them causes the block to be
  368. # reconstructed.
  369. def test_getblocktxn_requests(self, node, test_node, version):
  370. with_witness = (version==2)
  371. def test_getblocktxn_response(compact_block, peer, expected_result):
  372. msg = msg_cmpctblock(compact_block.to_p2p())
  373. peer.send_and_ping(msg)
  374. with mininode_lock:
  375. assert("getblocktxn" in peer.last_message)
  376. absolute_indexes = peer.last_message["getblocktxn"].block_txn_request.to_absolute()
  377. assert_equal(absolute_indexes, expected_result)
  378. def test_tip_after_message(node, peer, msg, tip):
  379. peer.send_and_ping(msg)
  380. assert_equal(int(node.getbestblockhash(), 16), tip)
  381. # First try announcing compactblocks that won't reconstruct, and verify
  382. # that we receive getblocktxn messages back.
  383. utxo = self.utxos.pop(0)
  384. block = self.build_block_with_transactions(node, utxo, 5)
  385. self.utxos.append([block.vtx[-1].sha256, 0, block.vtx[-1].vout[0].nValue])
  386. comp_block = HeaderAndShortIDs()
  387. comp_block.initialize_from_block(block, use_witness=with_witness)
  388. test_getblocktxn_response(comp_block, test_node, [1, 2, 3, 4, 5])
  389. msg_bt = msg_blocktxn()
  390. if with_witness:
  391. msg_bt = msg_witness_blocktxn() # serialize with witnesses
  392. msg_bt.block_transactions = BlockTransactions(block.sha256, block.vtx[1:])
  393. test_tip_after_message(node, test_node, msg_bt, block.sha256)
  394. utxo = self.utxos.pop(0)
  395. block = self.build_block_with_transactions(node, utxo, 5)
  396. self.utxos.append([block.vtx[-1].sha256, 0, block.vtx[-1].vout[0].nValue])
  397. # Now try interspersing the prefilled transactions
  398. comp_block.initialize_from_block(block, prefill_list=[0, 1, 5], use_witness=with_witness)
  399. test_getblocktxn_response(comp_block, test_node, [2, 3, 4])
  400. msg_bt.block_transactions = BlockTransactions(block.sha256, block.vtx[2:5])
  401. test_tip_after_message(node, test_node, msg_bt, block.sha256)
  402. # Now try giving one transaction ahead of time.
  403. utxo = self.utxos.pop(0)
  404. block = self.build_block_with_transactions(node, utxo, 5)
  405. self.utxos.append([block.vtx[-1].sha256, 0, block.vtx[-1].vout[0].nValue])
  406. test_node.send_and_ping(msg_tx(block.vtx[1]))
  407. assert(block.vtx[1].hash in node.getrawmempool())
  408. # Prefill 4 out of the 6 transactions, and verify that only the one
  409. # that was not in the mempool is requested.
  410. comp_block.initialize_from_block(block, prefill_list=[0, 2, 3, 4], use_witness=with_witness)
  411. test_getblocktxn_response(comp_block, test_node, [5])
  412. msg_bt.block_transactions = BlockTransactions(block.sha256, [block.vtx[5]])
  413. test_tip_after_message(node, test_node, msg_bt, block.sha256)
  414. # Now provide all transactions to the node before the block is
  415. # announced and verify reconstruction happens immediately.
  416. utxo = self.utxos.pop(0)
  417. block = self.build_block_with_transactions(node, utxo, 10)
  418. self.utxos.append([block.vtx[-1].sha256, 0, block.vtx[-1].vout[0].nValue])
  419. for tx in block.vtx[1:]:
  420. test_node.send_message(msg_tx(tx))
  421. test_node.sync_with_ping()
  422. # Make sure all transactions were accepted.
  423. mempool = node.getrawmempool()
  424. for tx in block.vtx[1:]:
  425. assert(tx.hash in mempool)
  426. # Clear out last request.
  427. with mininode_lock:
  428. test_node.last_message.pop("getblocktxn", None)
  429. # Send compact block
  430. comp_block.initialize_from_block(block, prefill_list=[0], use_witness=with_witness)
  431. test_tip_after_message(node, test_node, msg_cmpctblock(comp_block.to_p2p()), block.sha256)
  432. with mininode_lock:
  433. # Shouldn't have gotten a request for any transaction
  434. assert("getblocktxn" not in test_node.last_message)
  435. # Incorrectly responding to a getblocktxn shouldn't cause the block to be
  436. # permanently failed.
  437. def test_incorrect_blocktxn_response(self, node, test_node, version):
  438. if (len(self.utxos) == 0):
  439. self.make_utxos()
  440. utxo = self.utxos.pop(0)
  441. block = self.build_block_with_transactions(node, utxo, 10)
  442. self.utxos.append([block.vtx[-1].sha256, 0, block.vtx[-1].vout[0].nValue])
  443. # Relay the first 5 transactions from the block in advance
  444. for tx in block.vtx[1:6]:
  445. test_node.send_message(msg_tx(tx))
  446. test_node.sync_with_ping()
  447. # Make sure all transactions were accepted.
  448. mempool = node.getrawmempool()
  449. for tx in block.vtx[1:6]:
  450. assert(tx.hash in mempool)
  451. # Send compact block
  452. comp_block = HeaderAndShortIDs()
  453. comp_block.initialize_from_block(block, prefill_list=[0], use_witness=(version == 2))
  454. test_node.send_and_ping(msg_cmpctblock(comp_block.to_p2p()))
  455. absolute_indexes = []
  456. with mininode_lock:
  457. assert("getblocktxn" in test_node.last_message)
  458. absolute_indexes = test_node.last_message["getblocktxn"].block_txn_request.to_absolute()
  459. assert_equal(absolute_indexes, [6, 7, 8, 9, 10])
  460. # Now give an incorrect response.
  461. # Note that it's possible for bitcoind to be smart enough to know we're
  462. # lying, since it could check to see if the shortid matches what we're
  463. # sending, and eg disconnect us for misbehavior. If that behavior
  464. # change were made, we could just modify this test by having a
  465. # different peer provide the block further down, so that we're still
  466. # verifying that the block isn't marked bad permanently. This is good
  467. # enough for now.
  468. msg = msg_blocktxn()
  469. if version==2:
  470. msg = msg_witness_blocktxn()
  471. msg.block_transactions = BlockTransactions(block.sha256, [block.vtx[5]] + block.vtx[7:])
  472. test_node.send_and_ping(msg)
  473. # Tip should not have updated
  474. assert_equal(int(node.getbestblockhash(), 16), block.hashPrevBlock)
  475. # We should receive a getdata request
  476. wait_until(lambda: "getdata" in test_node.last_message, timeout=10, lock=mininode_lock)
  477. assert_equal(len(test_node.last_message["getdata"].inv), 1)
  478. assert(test_node.last_message["getdata"].inv[0].type == 2 or test_node.last_message["getdata"].inv[0].type == 2|MSG_WITNESS_FLAG)
  479. assert_equal(test_node.last_message["getdata"].inv[0].hash, block.sha256)
  480. # Deliver the block
  481. if version==2:
  482. test_node.send_and_ping(msg_witness_block(block))
  483. else:
  484. test_node.send_and_ping(msg_block(block))
  485. assert_equal(int(node.getbestblockhash(), 16), block.sha256)
  486. def test_getblocktxn_handler(self, node, test_node, version):
  487. # bitcoind will not send blocktxn responses for blocks whose height is
  488. # more than 10 blocks deep.
  489. MAX_GETBLOCKTXN_DEPTH = 10
  490. chain_height = node.getblockcount()
  491. current_height = chain_height
  492. while (current_height >= chain_height - MAX_GETBLOCKTXN_DEPTH):
  493. block_hash = node.getblockhash(current_height)
  494. block = FromHex(CBlock(), node.getblock(block_hash, False))
  495. msg = msg_getblocktxn()
  496. msg.block_txn_request = BlockTransactionsRequest(int(block_hash, 16), [])
  497. num_to_request = random.randint(1, len(block.vtx))
  498. msg.block_txn_request.from_absolute(sorted(random.sample(range(len(block.vtx)), num_to_request)))
  499. test_node.send_message(msg)
  500. wait_until(lambda: "blocktxn" in test_node.last_message, timeout=10, lock=mininode_lock)
  501. [tx.calc_sha256() for tx in block.vtx]
  502. with mininode_lock:
  503. assert_equal(test_node.last_message["blocktxn"].block_transactions.blockhash, int(block_hash, 16))
  504. all_indices = msg.block_txn_request.to_absolute()
  505. for index in all_indices:
  506. tx = test_node.last_message["blocktxn"].block_transactions.transactions.pop(0)
  507. tx.calc_sha256()
  508. assert_equal(tx.sha256, block.vtx[index].sha256)
  509. if version == 1:
  510. # Witnesses should have been stripped
  511. assert(tx.wit.is_null())
  512. else:
  513. # Check that the witness matches
  514. assert_equal(tx.calc_sha256(True), block.vtx[index].calc_sha256(True))
  515. test_node.last_message.pop("blocktxn", None)
  516. current_height -= 1
  517. # Next request should send a full block response, as we're past the
  518. # allowed depth for a blocktxn response.
  519. block_hash = node.getblockhash(current_height)
  520. msg.block_txn_request = BlockTransactionsRequest(int(block_hash, 16), [0])
  521. with mininode_lock:
  522. test_node.last_message.pop("block", None)
  523. test_node.last_message.pop("blocktxn", None)
  524. test_node.send_and_ping(msg)
  525. with mininode_lock:
  526. test_node.last_message["block"].block.calc_sha256()
  527. assert_equal(test_node.last_message["block"].block.sha256, int(block_hash, 16))
  528. assert "blocktxn" not in test_node.last_message
  529. def test_compactblocks_not_at_tip(self, node, test_node):
  530. # Test that requesting old compactblocks doesn't work.
  531. MAX_CMPCTBLOCK_DEPTH = 5
  532. new_blocks = []
  533. for i in range(MAX_CMPCTBLOCK_DEPTH + 1):
  534. test_node.clear_block_announcement()
  535. new_blocks.append(node.generate(1)[0])
  536. wait_until(test_node.received_block_announcement, timeout=30, lock=mininode_lock)
  537. test_node.clear_block_announcement()
  538. test_node.send_message(msg_getdata([CInv(4, int(new_blocks[0], 16))]))
  539. wait_until(lambda: "cmpctblock" in test_node.last_message, timeout=30, lock=mininode_lock)
  540. test_node.clear_block_announcement()
  541. node.generate(1)
  542. wait_until(test_node.received_block_announcement, timeout=30, lock=mininode_lock)
  543. test_node.clear_block_announcement()
  544. with mininode_lock:
  545. test_node.last_message.pop("block", None)
  546. test_node.send_message(msg_getdata([CInv(4, int(new_blocks[0], 16))]))
  547. wait_until(lambda: "block" in test_node.last_message, timeout=30, lock=mininode_lock)
  548. with mininode_lock:
  549. test_node.last_message["block"].block.calc_sha256()
  550. assert_equal(test_node.last_message["block"].block.sha256, int(new_blocks[0], 16))
  551. # Generate an old compactblock, and verify that it's not accepted.
  552. cur_height = node.getblockcount()
  553. hashPrevBlock = int(node.getblockhash(cur_height-5), 16)
  554. block = self.build_block_on_tip(node)
  555. block.hashPrevBlock = hashPrevBlock
  556. block.solve()
  557. comp_block = HeaderAndShortIDs()
  558. comp_block.initialize_from_block(block)
  559. test_node.send_and_ping(msg_cmpctblock(comp_block.to_p2p()))
  560. tips = node.getchaintips()
  561. found = False
  562. for x in tips:
  563. if x["hash"] == block.hash:
  564. assert_equal(x["status"], "headers-only")
  565. found = True
  566. break
  567. assert(found)
  568. # Requesting this block via getblocktxn should silently fail
  569. # (to avoid fingerprinting attacks).
  570. msg = msg_getblocktxn()
  571. msg.block_txn_request = BlockTransactionsRequest(block.sha256, [0])
  572. with mininode_lock:
  573. test_node.last_message.pop("blocktxn", None)
  574. test_node.send_and_ping(msg)
  575. with mininode_lock:
  576. assert "blocktxn" not in test_node.last_message
  577. def activate_segwit(self, node):
  578. node.generate(144*3)
  579. assert_equal(get_bip9_status(node, "segwit")["status"], 'active')
  580. def test_end_to_end_block_relay(self, node, listeners):
  581. utxo = self.utxos.pop(0)
  582. block = self.build_block_with_transactions(node, utxo, 10)
  583. [l.clear_block_announcement() for l in listeners]
  584. # ToHex() won't serialize with witness, but this block has no witnesses
  585. # anyway. TODO: repeat this test with witness tx's to a segwit node.
  586. node.submitblock(ToHex(block))
  587. for l in listeners:
  588. wait_until(lambda: l.received_block_announcement(), timeout=30, lock=mininode_lock)
  589. with mininode_lock:
  590. for l in listeners:
  591. assert "cmpctblock" in l.last_message
  592. l.last_message["cmpctblock"].header_and_shortids.header.calc_sha256()
  593. assert_equal(l.last_message["cmpctblock"].header_and_shortids.header.sha256, block.sha256)
  594. # Test that we don't get disconnected if we relay a compact block with valid header,
  595. # but invalid transactions.
  596. def test_invalid_tx_in_compactblock(self, node, test_node, use_segwit):
  597. assert(len(self.utxos))
  598. utxo = self.utxos[0]
  599. block = self.build_block_with_transactions(node, utxo, 5)
  600. del block.vtx[3]
  601. block.hashMerkleRoot = block.calc_merkle_root()
  602. if use_segwit:
  603. # If we're testing with segwit, also drop the coinbase witness,
  604. # but include the witness commitment.
  605. add_witness_commitment(block)
  606. block.vtx[0].wit.vtxinwit = []
  607. block.solve()
  608. # Now send the compact block with all transactions prefilled, and
  609. # verify that we don't get disconnected.
  610. comp_block = HeaderAndShortIDs()
  611. comp_block.initialize_from_block(block, prefill_list=[0, 1, 2, 3, 4], use_witness=use_segwit)
  612. msg = msg_cmpctblock(comp_block.to_p2p())
  613. test_node.send_and_ping(msg)
  614. # Check that the tip didn't advance
  615. assert(int(node.getbestblockhash(), 16) is not block.sha256)
  616. test_node.sync_with_ping()
  617. # Helper for enabling cb announcements
  618. # Send the sendcmpct request and sync headers
  619. def request_cb_announcements(self, peer, node, version):
  620. tip = node.getbestblockhash()
  621. peer.get_headers(locator=[int(tip, 16)], hashstop=0)
  622. msg = msg_sendcmpct()
  623. msg.version = version
  624. msg.announce = True
  625. peer.send_and_ping(msg)
  626. def test_compactblock_reconstruction_multiple_peers(self, node, stalling_peer, delivery_peer):
  627. assert(len(self.utxos))
  628. def announce_cmpct_block(node, peer):
  629. utxo = self.utxos.pop(0)
  630. block = self.build_block_with_transactions(node, utxo, 5)
  631. cmpct_block = HeaderAndShortIDs()
  632. cmpct_block.initialize_from_block(block)
  633. msg = msg_cmpctblock(cmpct_block.to_p2p())
  634. peer.send_and_ping(msg)
  635. with mininode_lock:
  636. assert "getblocktxn" in peer.last_message
  637. return block, cmpct_block
  638. block, cmpct_block = announce_cmpct_block(node, stalling_peer)
  639. for tx in block.vtx[1:]:
  640. delivery_peer.send_message(msg_tx(tx))
  641. delivery_peer.sync_with_ping()
  642. mempool = node.getrawmempool()
  643. for tx in block.vtx[1:]:
  644. assert(tx.hash in mempool)
  645. delivery_peer.send_and_ping(msg_cmpctblock(cmpct_block.to_p2p()))
  646. assert_equal(int(node.getbestblockhash(), 16), block.sha256)
  647. self.utxos.append([block.vtx[-1].sha256, 0, block.vtx[-1].vout[0].nValue])
  648. # Now test that delivering an invalid compact block won't break relay
  649. block, cmpct_block = announce_cmpct_block(node, stalling_peer)
  650. for tx in block.vtx[1:]:
  651. delivery_peer.send_message(msg_tx(tx))
  652. delivery_peer.sync_with_ping()
  653. cmpct_block.prefilled_txn[0].tx.wit.vtxinwit = [ CTxInWitness() ]
  654. cmpct_block.prefilled_txn[0].tx.wit.vtxinwit[0].scriptWitness.stack = [ser_uint256(0)]
  655. cmpct_block.use_witness = True
  656. delivery_peer.send_and_ping(msg_cmpctblock(cmpct_block.to_p2p()))
  657. assert(int(node.getbestblockhash(), 16) != block.sha256)
  658. msg = msg_blocktxn()
  659. msg.block_transactions.blockhash = block.sha256
  660. msg.block_transactions.transactions = block.vtx[1:]
  661. stalling_peer.send_and_ping(msg)
  662. assert_equal(int(node.getbestblockhash(), 16), block.sha256)
  663. def run_test(self):
  664. # Setup the p2p connections and start up the network thread.
  665. self.test_node = TestNode()
  666. self.segwit_node = TestNode()
  667. self.old_node = TestNode() # version 1 peer <--> segwit node
  668. connections = []
  669. connections.append(NodeConn('127.0.0.1', p2p_port(0), self.nodes[0], self.test_node))
  670. connections.append(NodeConn('127.0.0.1', p2p_port(1), self.nodes[1],
  671. self.segwit_node, services=NODE_NETWORK|NODE_WITNESS))
  672. connections.append(NodeConn('127.0.0.1', p2p_port(1), self.nodes[1],
  673. self.old_node, services=NODE_NETWORK))
  674. self.test_node.add_connection(connections[0])
  675. self.segwit_node.add_connection(connections[1])
  676. self.old_node.add_connection(connections[2])
  677. NetworkThread().start() # Start up network handling in another thread
  678. # Test logic begins here
  679. self.test_node.wait_for_verack()
  680. # We will need UTXOs to construct transactions in later tests.
  681. self.make_utxos()
  682. self.log.info("Running tests, pre-segwit activation:")
  683. self.log.info("Testing SENDCMPCT p2p message... ")
  684. self.test_sendcmpct(self.nodes[0], self.test_node, 1)
  685. sync_blocks(self.nodes)
  686. self.test_sendcmpct(self.nodes[1], self.segwit_node, 2, old_node=self.old_node)
  687. sync_blocks(self.nodes)
  688. self.log.info("Testing compactblock construction...")
  689. self.test_compactblock_construction(self.nodes[0], self.test_node, 1, False)
  690. sync_blocks(self.nodes)
  691. self.test_compactblock_construction(self.nodes[1], self.segwit_node, 2, False)
  692. sync_blocks(self.nodes)
  693. self.log.info("Testing compactblock requests... ")
  694. self.test_compactblock_requests(self.nodes[0], self.test_node, 1, False)
  695. sync_blocks(self.nodes)
  696. self.test_compactblock_requests(self.nodes[1], self.segwit_node, 2, False)
  697. sync_blocks(self.nodes)
  698. self.log.info("Testing getblocktxn requests...")
  699. self.test_getblocktxn_requests(self.nodes[0], self.test_node, 1)
  700. sync_blocks(self.nodes)
  701. self.test_getblocktxn_requests(self.nodes[1], self.segwit_node, 2)
  702. sync_blocks(self.nodes)
  703. self.log.info("Testing getblocktxn handler...")
  704. self.test_getblocktxn_handler(self.nodes[0], self.test_node, 1)
  705. sync_blocks(self.nodes)
  706. self.test_getblocktxn_handler(self.nodes[1], self.segwit_node, 2)
  707. self.test_getblocktxn_handler(self.nodes[1], self.old_node, 1)
  708. sync_blocks(self.nodes)
  709. self.log.info("Testing compactblock requests/announcements not at chain tip...")
  710. self.test_compactblocks_not_at_tip(self.nodes[0], self.test_node)
  711. sync_blocks(self.nodes)
  712. self.test_compactblocks_not_at_tip(self.nodes[1], self.segwit_node)
  713. self.test_compactblocks_not_at_tip(self.nodes[1], self.old_node)
  714. sync_blocks(self.nodes)
  715. self.log.info("Testing handling of incorrect blocktxn responses...")
  716. self.test_incorrect_blocktxn_response(self.nodes[0], self.test_node, 1)
  717. sync_blocks(self.nodes)
  718. self.test_incorrect_blocktxn_response(self.nodes[1], self.segwit_node, 2)
  719. sync_blocks(self.nodes)
  720. # End-to-end block relay tests
  721. self.log.info("Testing end-to-end block relay...")
  722. self.request_cb_announcements(self.test_node, self.nodes[0], 1)
  723. self.request_cb_announcements(self.old_node, self.nodes[1], 1)
  724. self.request_cb_announcements(self.segwit_node, self.nodes[1], 2)
  725. self.test_end_to_end_block_relay(self.nodes[0], [self.segwit_node, self.test_node, self.old_node])
  726. self.test_end_to_end_block_relay(self.nodes[1], [self.segwit_node, self.test_node, self.old_node])
  727. self.log.info("Testing handling of invalid compact blocks...")
  728. self.test_invalid_tx_in_compactblock(self.nodes[0], self.test_node, False)
  729. self.test_invalid_tx_in_compactblock(self.nodes[1], self.segwit_node, False)
  730. self.test_invalid_tx_in_compactblock(self.nodes[1], self.old_node, False)
  731. self.log.info("Testing reconstructing compact blocks from all peers...")
  732. self.test_compactblock_reconstruction_multiple_peers(self.nodes[1], self.segwit_node, self.old_node)
  733. sync_blocks(self.nodes)
  734. # Advance to segwit activation
  735. self.log.info("Advancing to segwit activation")
  736. self.activate_segwit(self.nodes[1])
  737. self.log.info("Running tests, post-segwit activation...")
  738. self.log.info("Testing compactblock construction...")
  739. self.test_compactblock_construction(self.nodes[1], self.old_node, 1, True)
  740. self.test_compactblock_construction(self.nodes[1], self.segwit_node, 2, True)
  741. sync_blocks(self.nodes)
  742. self.log.info("Testing compactblock requests (unupgraded node)... ")
  743. self.test_compactblock_requests(self.nodes[0], self.test_node, 1, True)
  744. self.log.info("Testing getblocktxn requests (unupgraded node)...")
  745. self.test_getblocktxn_requests(self.nodes[0], self.test_node, 1)
  746. # Need to manually sync node0 and node1, because post-segwit activation,
  747. # node1 will not download blocks from node0.
  748. self.log.info("Syncing nodes...")
  749. assert(self.nodes[0].getbestblockhash() != self.nodes[1].getbestblockhash())
  750. while (self.nodes[0].getblockcount() > self.nodes[1].getblockcount()):
  751. block_hash = self.nodes[0].getblockhash(self.nodes[1].getblockcount()+1)
  752. self.nodes[1].submitblock(self.nodes[0].getblock(block_hash, False))
  753. assert_equal(self.nodes[0].getbestblockhash(), self.nodes[1].getbestblockhash())
  754. self.log.info("Testing compactblock requests (segwit node)... ")
  755. self.test_compactblock_requests(self.nodes[1], self.segwit_node, 2, True)
  756. self.log.info("Testing getblocktxn requests (segwit node)...")
  757. self.test_getblocktxn_requests(self.nodes[1], self.segwit_node, 2)
  758. sync_blocks(self.nodes)
  759. self.log.info("Testing getblocktxn handler (segwit node should return witnesses)...")
  760. self.test_getblocktxn_handler(self.nodes[1], self.segwit_node, 2)
  761. self.test_getblocktxn_handler(self.nodes[1], self.old_node, 1)
  762. # Test that if we submitblock to node1, we'll get a compact block
  763. # announcement to all peers.
  764. # (Post-segwit activation, blocks won't propagate from node0 to node1
  765. # automatically, so don't bother testing a block announced to node0.)
  766. self.log.info("Testing end-to-end block relay...")
  767. self.request_cb_announcements(self.test_node, self.nodes[0], 1)
  768. self.request_cb_announcements(self.old_node, self.nodes[1], 1)
  769. self.request_cb_announcements(self.segwit_node, self.nodes[1], 2)
  770. self.test_end_to_end_block_relay(self.nodes[1], [self.segwit_node, self.test_node, self.old_node])
  771. self.log.info("Testing handling of invalid compact blocks...")
  772. self.test_invalid_tx_in_compactblock(self.nodes[0], self.test_node, False)
  773. self.test_invalid_tx_in_compactblock(self.nodes[1], self.segwit_node, True)
  774. self.test_invalid_tx_in_compactblock(self.nodes[1], self.old_node, True)
  775. self.log.info("Testing invalid index in cmpctblock message...")
  776. self.test_invalid_cmpctblock_message()
  777. if __name__ == '__main__':
  778. CompactBlocksTest().main()