You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

smartfees.py 12KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266
  1. #!/usr/bin/env python3
  2. # Copyright (c) 2014-2016 The Bitcoin Core developers
  3. # Distributed under the MIT software license, see the accompanying
  4. # file COPYING or http://www.opensource.org/licenses/mit-license.php.
  5. """Test fee estimation code."""
  6. from collections import OrderedDict
  7. from test_framework.test_framework import BitcoinTestFramework
  8. from test_framework.util import *
  9. from test_framework.script import CScript, OP_1, OP_DROP, OP_2, OP_HASH160, OP_EQUAL, hash160, OP_TRUE
  10. from test_framework.mininode import CTransaction, CTxIn, CTxOut, COutPoint, ToHex, FromHex, COIN
  11. # Construct 2 trivial P2SH's and the ScriptSigs that spend them
  12. # So we can create many many transactions without needing to spend
  13. # time signing.
  14. redeem_script_1 = CScript([OP_1, OP_DROP])
  15. redeem_script_2 = CScript([OP_2, OP_DROP])
  16. P2SH_1 = CScript([OP_HASH160, hash160(redeem_script_1), OP_EQUAL])
  17. P2SH_2 = CScript([OP_HASH160, hash160(redeem_script_2), OP_EQUAL])
  18. # Associated ScriptSig's to spend satisfy P2SH_1 and P2SH_2
  19. SCRIPT_SIG = [CScript([OP_TRUE, redeem_script_1]), CScript([OP_TRUE, redeem_script_2])]
  20. global log
  21. def small_txpuzzle_randfee(from_node, conflist, unconflist, amount, min_fee, fee_increment):
  22. """
  23. Create and send a transaction with a random fee.
  24. The transaction pays to a trivial P2SH script, and assumes that its inputs
  25. are of the same form.
  26. The function takes a list of confirmed outputs and unconfirmed outputs
  27. and attempts to use the confirmed list first for its inputs.
  28. It adds the newly created outputs to the unconfirmed list.
  29. Returns (raw transaction, fee)
  30. """
  31. # It's best to exponentially distribute our random fees
  32. # because the buckets are exponentially spaced.
  33. # Exponentially distributed from 1-128 * fee_increment
  34. rand_fee = float(fee_increment)*(1.1892**random.randint(0,28))
  35. # Total fee ranges from min_fee to min_fee + 127*fee_increment
  36. fee = min_fee - fee_increment + satoshi_round(rand_fee)
  37. tx = CTransaction()
  38. total_in = Decimal("0.00000000")
  39. while total_in <= (amount + fee) and len(conflist) > 0:
  40. t = conflist.pop(0)
  41. total_in += t["amount"]
  42. tx.vin.append(CTxIn(COutPoint(int(t["txid"], 16), t["vout"]), b""))
  43. if total_in <= amount + fee:
  44. while total_in <= (amount + fee) and len(unconflist) > 0:
  45. t = unconflist.pop(0)
  46. total_in += t["amount"]
  47. tx.vin.append(CTxIn(COutPoint(int(t["txid"], 16), t["vout"]), b""))
  48. if total_in <= amount + fee:
  49. raise RuntimeError("Insufficient funds: need %d, have %d"%(amount+fee, total_in))
  50. tx.vout.append(CTxOut(int((total_in - amount - fee)*COIN), P2SH_1))
  51. tx.vout.append(CTxOut(int(amount*COIN), P2SH_2))
  52. # These transactions don't need to be signed, but we still have to insert
  53. # the ScriptSig that will satisfy the ScriptPubKey.
  54. for inp in tx.vin:
  55. inp.scriptSig = SCRIPT_SIG[inp.prevout.n]
  56. txid = from_node.sendrawtransaction(ToHex(tx), True)
  57. unconflist.append({ "txid" : txid, "vout" : 0 , "amount" : total_in - amount - fee})
  58. unconflist.append({ "txid" : txid, "vout" : 1 , "amount" : amount})
  59. return (ToHex(tx), fee)
  60. def split_inputs(from_node, txins, txouts, initial_split = False):
  61. """
  62. We need to generate a lot of inputs so we can generate a ton of transactions.
  63. This function takes an input from txins, and creates and sends a transaction
  64. which splits the value into 2 outputs which are appended to txouts.
  65. Previously this was designed to be small inputs so they wouldn't have
  66. a high coin age when the notion of priority still existed.
  67. """
  68. prevtxout = txins.pop()
  69. tx = CTransaction()
  70. tx.vin.append(CTxIn(COutPoint(int(prevtxout["txid"], 16), prevtxout["vout"]), b""))
  71. half_change = satoshi_round(prevtxout["amount"]/2)
  72. rem_change = prevtxout["amount"] - half_change - Decimal("0.00001000")
  73. tx.vout.append(CTxOut(int(half_change*COIN), P2SH_1))
  74. tx.vout.append(CTxOut(int(rem_change*COIN), P2SH_2))
  75. # If this is the initial split we actually need to sign the transaction
  76. # Otherwise we just need to insert the proper ScriptSig
  77. if (initial_split) :
  78. completetx = from_node.signrawtransaction(ToHex(tx))["hex"]
  79. else :
  80. tx.vin[0].scriptSig = SCRIPT_SIG[prevtxout["vout"]]
  81. completetx = ToHex(tx)
  82. txid = from_node.sendrawtransaction(completetx, True)
  83. txouts.append({ "txid" : txid, "vout" : 0 , "amount" : half_change})
  84. txouts.append({ "txid" : txid, "vout" : 1 , "amount" : rem_change})
  85. def check_estimates(node, fees_seen, max_invalid, print_estimates = True):
  86. """
  87. This function calls estimatefee and verifies that the estimates
  88. meet certain invariants.
  89. """
  90. all_estimates = [ node.estimatefee(i) for i in range(1,26) ]
  91. if print_estimates:
  92. log.info([str(all_estimates[e-1]) for e in [1,2,3,6,15,25]])
  93. delta = 1.0e-6 # account for rounding error
  94. last_e = max(fees_seen)
  95. for e in [x for x in all_estimates if x >= 0]:
  96. # Estimates should be within the bounds of what transactions fees actually were:
  97. if float(e)+delta < min(fees_seen) or float(e)-delta > max(fees_seen):
  98. raise AssertionError("Estimated fee (%f) out of range (%f,%f)"
  99. %(float(e), min(fees_seen), max(fees_seen)))
  100. # Estimates should be monotonically decreasing
  101. if float(e)-delta > last_e:
  102. raise AssertionError("Estimated fee (%f) larger than last fee (%f) for lower number of confirms"
  103. %(float(e),float(last_e)))
  104. last_e = e
  105. valid_estimate = False
  106. invalid_estimates = 0
  107. for i,e in enumerate(all_estimates): # estimate is for i+1
  108. if e >= 0:
  109. valid_estimate = True
  110. # estimatesmartfee should return the same result
  111. assert_equal(node.estimatesmartfee(i+1)["feerate"], e)
  112. else:
  113. invalid_estimates += 1
  114. # estimatesmartfee should still be valid
  115. approx_estimate = node.estimatesmartfee(i+1)["feerate"]
  116. answer_found = node.estimatesmartfee(i+1)["blocks"]
  117. assert(approx_estimate > 0)
  118. assert(answer_found > i+1)
  119. # Once we're at a high enough confirmation count that we can give an estimate
  120. # We should have estimates for all higher confirmation counts
  121. if valid_estimate:
  122. raise AssertionError("Invalid estimate appears at higher confirm count than valid estimate")
  123. # Check on the expected number of different confirmation counts
  124. # that we might not have valid estimates for
  125. if invalid_estimates > max_invalid:
  126. raise AssertionError("More than (%d) invalid estimates"%(max_invalid))
  127. return all_estimates
  128. class EstimateFeeTest(BitcoinTestFramework):
  129. def __init__(self):
  130. super().__init__()
  131. self.num_nodes = 3
  132. self.setup_clean_chain = False
  133. def setup_network(self):
  134. """
  135. We'll setup the network to have 3 nodes that all mine with different parameters.
  136. But first we need to use one node to create a lot of outputs
  137. which we will use to generate our transactions.
  138. """
  139. self.nodes = []
  140. # Use node0 to mine blocks for input splitting
  141. self.nodes.append(start_node(0, self.options.tmpdir, ["-maxorphantx=1000",
  142. "-whitelist=127.0.0.1"]))
  143. self.log.info("This test is time consuming, please be patient")
  144. self.log.info("Splitting inputs so we can generate tx's")
  145. self.txouts = []
  146. self.txouts2 = []
  147. # Split a coinbase into two transaction puzzle outputs
  148. split_inputs(self.nodes[0], self.nodes[0].listunspent(0), self.txouts, True)
  149. # Mine
  150. while (len(self.nodes[0].getrawmempool()) > 0):
  151. self.nodes[0].generate(1)
  152. # Repeatedly split those 2 outputs, doubling twice for each rep
  153. # Use txouts to monitor the available utxo, since these won't be tracked in wallet
  154. reps = 0
  155. while (reps < 5):
  156. #Double txouts to txouts2
  157. while (len(self.txouts)>0):
  158. split_inputs(self.nodes[0], self.txouts, self.txouts2)
  159. while (len(self.nodes[0].getrawmempool()) > 0):
  160. self.nodes[0].generate(1)
  161. #Double txouts2 to txouts
  162. while (len(self.txouts2)>0):
  163. split_inputs(self.nodes[0], self.txouts2, self.txouts)
  164. while (len(self.nodes[0].getrawmempool()) > 0):
  165. self.nodes[0].generate(1)
  166. reps += 1
  167. self.log.info("Finished splitting")
  168. # Now we can connect the other nodes, didn't want to connect them earlier
  169. # so the estimates would not be affected by the splitting transactions
  170. # Node1 mines small blocks but that are bigger than the expected transaction rate.
  171. # NOTE: the CreateNewBlock code starts counting block size at 1,000 bytes,
  172. # (17k is room enough for 110 or so transactions)
  173. self.nodes.append(start_node(1, self.options.tmpdir,
  174. ["-blockmaxsize=17000", "-maxorphantx=1000"]))
  175. connect_nodes(self.nodes[1], 0)
  176. # Node2 is a stingy miner, that
  177. # produces too small blocks (room for only 55 or so transactions)
  178. node2args = ["-blockmaxsize=8000", "-maxorphantx=1000"]
  179. self.nodes.append(start_node(2, self.options.tmpdir, node2args))
  180. connect_nodes(self.nodes[0], 2)
  181. connect_nodes(self.nodes[2], 1)
  182. self.is_network_split = False
  183. self.sync_all()
  184. def transact_and_mine(self, numblocks, mining_node):
  185. min_fee = Decimal("0.00001")
  186. # We will now mine numblocks blocks generating on average 100 transactions between each block
  187. # We shuffle our confirmed txout set before each set of transactions
  188. # small_txpuzzle_randfee will use the transactions that have inputs already in the chain when possible
  189. # resorting to tx's that depend on the mempool when those run out
  190. for i in range(numblocks):
  191. random.shuffle(self.confutxo)
  192. for j in range(random.randrange(100-50,100+50)):
  193. from_index = random.randint(1,2)
  194. (txhex, fee) = small_txpuzzle_randfee(self.nodes[from_index], self.confutxo,
  195. self.memutxo, Decimal("0.005"), min_fee, min_fee)
  196. tx_kbytes = (len(txhex) // 2) / 1000.0
  197. self.fees_per_kb.append(float(fee)/tx_kbytes)
  198. sync_mempools(self.nodes[0:3], wait=.1)
  199. mined = mining_node.getblock(mining_node.generate(1)[0],True)["tx"]
  200. sync_blocks(self.nodes[0:3], wait=.1)
  201. # update which txouts are confirmed
  202. newmem = []
  203. for utx in self.memutxo:
  204. if utx["txid"] in mined:
  205. self.confutxo.append(utx)
  206. else:
  207. newmem.append(utx)
  208. self.memutxo = newmem
  209. def run_test(self):
  210. # Make log handler available to helper functions
  211. global log
  212. log = self.log
  213. self.fees_per_kb = []
  214. self.memutxo = []
  215. self.confutxo = self.txouts # Start with the set of confirmed txouts after splitting
  216. self.log.info("Will output estimates for 1/2/3/6/15/25 blocks")
  217. for i in range(2):
  218. self.log.info("Creating transactions and mining them with a block size that can't keep up")
  219. # Create transactions and mine 10 small blocks with node 2, but create txs faster than we can mine
  220. self.transact_and_mine(10, self.nodes[2])
  221. check_estimates(self.nodes[1], self.fees_per_kb, 14)
  222. self.log.info("Creating transactions and mining them at a block size that is just big enough")
  223. # Generate transactions while mining 10 more blocks, this time with node1
  224. # which mines blocks with capacity just above the rate that transactions are being created
  225. self.transact_and_mine(10, self.nodes[1])
  226. check_estimates(self.nodes[1], self.fees_per_kb, 2)
  227. # Finish by mining a normal-sized block:
  228. while len(self.nodes[1].getrawmempool()) > 0:
  229. self.nodes[1].generate(1)
  230. sync_blocks(self.nodes[0:3], wait=.1)
  231. self.log.info("Final estimates after emptying mempools")
  232. check_estimates(self.nodes[1], self.fees_per_kb, 2)
  233. if __name__ == '__main__':
  234. EstimateFeeTest().main()