You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

util.py 14KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410
  1. # Copyright (c) 2014-2015 The Bitcoin Core developers
  2. # Distributed under the MIT software license, see the accompanying
  3. # file COPYING or http://www.opensource.org/licenses/mit-license.php.
  4. #
  5. # Helpful routines for regression testing
  6. #
  7. # Add python-bitcoinrpc to module search path:
  8. import os
  9. import sys
  10. from decimal import Decimal, ROUND_DOWN
  11. import json
  12. import random
  13. import shutil
  14. import subprocess
  15. import time
  16. import re
  17. from . import coverage
  18. from .authproxy import AuthServiceProxy, JSONRPCException
  19. COVERAGE_DIR = None
  20. def enable_coverage(dirname):
  21. """Maintain a log of which RPC calls are made during testing."""
  22. global COVERAGE_DIR
  23. COVERAGE_DIR = dirname
  24. def get_rpc_proxy(url, node_number, timeout=None):
  25. """
  26. Args:
  27. url (str): URL of the RPC server to call
  28. node_number (int): the node number (or id) that this calls to
  29. Kwargs:
  30. timeout (int): HTTP timeout in seconds
  31. Returns:
  32. AuthServiceProxy. convenience object for making RPC calls.
  33. """
  34. proxy_kwargs = {}
  35. if timeout is not None:
  36. proxy_kwargs['timeout'] = timeout
  37. proxy = AuthServiceProxy(url, **proxy_kwargs)
  38. proxy.url = url # store URL on proxy for info
  39. coverage_logfile = coverage.get_filename(
  40. COVERAGE_DIR, node_number) if COVERAGE_DIR else None
  41. return coverage.AuthServiceProxyWrapper(proxy, coverage_logfile)
  42. def p2p_port(n):
  43. return 11000 + n + os.getpid()%999
  44. def rpc_port(n):
  45. return 12000 + n + os.getpid()%999
  46. def check_json_precision():
  47. """Make sure json library being used does not lose precision converting BTC values"""
  48. n = Decimal("20000000.00000003")
  49. satoshis = int(json.loads(json.dumps(float(n)))*1.0e8)
  50. if satoshis != 2000000000000003:
  51. raise RuntimeError("JSON encode/decode loses precision")
  52. def count_bytes(hex_string):
  53. return len(bytearray.fromhex(hex_string))
  54. def sync_blocks(rpc_connections, wait=1):
  55. """
  56. Wait until everybody has the same block count
  57. """
  58. while True:
  59. counts = [ x.getblockcount() for x in rpc_connections ]
  60. if counts == [ counts[0] ]*len(counts):
  61. break
  62. time.sleep(wait)
  63. def sync_mempools(rpc_connections, wait=1):
  64. """
  65. Wait until everybody has the same transactions in their memory
  66. pools
  67. """
  68. while True:
  69. pool = set(rpc_connections[0].getrawmempool())
  70. num_match = 1
  71. for i in range(1, len(rpc_connections)):
  72. if set(rpc_connections[i].getrawmempool()) == pool:
  73. num_match = num_match+1
  74. if num_match == len(rpc_connections):
  75. break
  76. time.sleep(wait)
  77. bitcoind_processes = {}
  78. def initialize_datadir(dirname, n):
  79. datadir = os.path.join(dirname, "node"+str(n))
  80. if not os.path.isdir(datadir):
  81. os.makedirs(datadir)
  82. with open(os.path.join(datadir, "bitcoin.conf"), 'w') as f:
  83. f.write("regtest=1\n");
  84. f.write("rpcuser=rt\n");
  85. f.write("rpcpassword=rt\n");
  86. f.write("port="+str(p2p_port(n))+"\n");
  87. f.write("rpcport="+str(rpc_port(n))+"\n");
  88. return datadir
  89. def initialize_chain(test_dir):
  90. """
  91. Create (or copy from cache) a 200-block-long chain and
  92. 4 wallets.
  93. bitcoind and bitcoin-cli must be in search path.
  94. """
  95. if (not os.path.isdir(os.path.join("cache","node0"))
  96. or not os.path.isdir(os.path.join("cache","node1"))
  97. or not os.path.isdir(os.path.join("cache","node2"))
  98. or not os.path.isdir(os.path.join("cache","node3"))):
  99. #find and delete old cache directories if any exist
  100. for i in range(4):
  101. if os.path.isdir(os.path.join("cache","node"+str(i))):
  102. shutil.rmtree(os.path.join("cache","node"+str(i)))
  103. devnull = open(os.devnull, "w")
  104. # Create cache directories, run bitcoinds:
  105. for i in range(4):
  106. datadir=initialize_datadir("cache", i)
  107. args = [ os.getenv("BITCOIND", "bitcoind"), "-keypool=1", "-datadir="+datadir, "-discover=0" ]
  108. if i > 0:
  109. args.append("-connect=127.0.0.1:"+str(p2p_port(0)))
  110. bitcoind_processes[i] = subprocess.Popen(args)
  111. if os.getenv("PYTHON_DEBUG", ""):
  112. print "initialize_chain: bitcoind started, calling bitcoin-cli -rpcwait getblockcount"
  113. subprocess.check_call([ os.getenv("BITCOINCLI", "bitcoin-cli"), "-datadir="+datadir,
  114. "-rpcwait", "getblockcount"], stdout=devnull)
  115. if os.getenv("PYTHON_DEBUG", ""):
  116. print "initialize_chain: bitcoin-cli -rpcwait getblockcount completed"
  117. devnull.close()
  118. rpcs = []
  119. for i in range(4):
  120. try:
  121. url = "http://rt:rt@127.0.0.1:%d" % (rpc_port(i),)
  122. rpcs.append(get_rpc_proxy(url, i))
  123. except:
  124. sys.stderr.write("Error connecting to "+url+"\n")
  125. sys.exit(1)
  126. # Create a 200-block-long chain; each of the 4 nodes
  127. # gets 25 mature blocks and 25 immature.
  128. # blocks are created with timestamps 10 minutes apart, starting
  129. # at 1 Jan 2014
  130. block_time = 1388534400
  131. for i in range(2):
  132. for peer in range(4):
  133. for j in range(25):
  134. set_node_times(rpcs, block_time)
  135. rpcs[peer].generate(1)
  136. block_time += 10*60
  137. # Must sync before next peer starts generating blocks
  138. sync_blocks(rpcs)
  139. # Shut them down, and clean up cache directories:
  140. stop_nodes(rpcs)
  141. wait_bitcoinds()
  142. for i in range(4):
  143. os.remove(log_filename("cache", i, "debug.log"))
  144. os.remove(log_filename("cache", i, "db.log"))
  145. os.remove(log_filename("cache", i, "peers.dat"))
  146. os.remove(log_filename("cache", i, "fee_estimates.dat"))
  147. for i in range(4):
  148. from_dir = os.path.join("cache", "node"+str(i))
  149. to_dir = os.path.join(test_dir, "node"+str(i))
  150. shutil.copytree(from_dir, to_dir)
  151. initialize_datadir(test_dir, i) # Overwrite port/rpcport in bitcoin.conf
  152. def initialize_chain_clean(test_dir, num_nodes):
  153. """
  154. Create an empty blockchain and num_nodes wallets.
  155. Useful if a test case wants complete control over initialization.
  156. """
  157. for i in range(num_nodes):
  158. datadir=initialize_datadir(test_dir, i)
  159. def _rpchost_to_args(rpchost):
  160. '''Convert optional IP:port spec to rpcconnect/rpcport args'''
  161. if rpchost is None:
  162. return []
  163. match = re.match('(\[[0-9a-fA-f:]+\]|[^:]+)(?::([0-9]+))?$', rpchost)
  164. if not match:
  165. raise ValueError('Invalid RPC host spec ' + rpchost)
  166. rpcconnect = match.group(1)
  167. rpcport = match.group(2)
  168. if rpcconnect.startswith('['): # remove IPv6 [...] wrapping
  169. rpcconnect = rpcconnect[1:-1]
  170. rv = ['-rpcconnect=' + rpcconnect]
  171. if rpcport:
  172. rv += ['-rpcport=' + rpcport]
  173. return rv
  174. def start_node(i, dirname, extra_args=None, rpchost=None, timewait=None, binary=None):
  175. """
  176. Start a bitcoind and return RPC connection to it
  177. """
  178. datadir = os.path.join(dirname, "node"+str(i))
  179. if binary is None:
  180. binary = os.getenv("BITCOIND", "bitcoind")
  181. # RPC tests still depend on free transactions
  182. args = [ binary, "-datadir="+datadir, "-keypool=1", "-discover=0", "-rest", "-blockprioritysize=50000" ]
  183. if extra_args is not None: args.extend(extra_args)
  184. bitcoind_processes[i] = subprocess.Popen(args)
  185. devnull = open(os.devnull, "w")
  186. if os.getenv("PYTHON_DEBUG", ""):
  187. print "start_node: bitcoind started, calling bitcoin-cli -rpcwait getblockcount"
  188. subprocess.check_call([ os.getenv("BITCOINCLI", "bitcoin-cli"), "-datadir="+datadir] +
  189. _rpchost_to_args(rpchost) +
  190. ["-rpcwait", "getblockcount"], stdout=devnull)
  191. if os.getenv("PYTHON_DEBUG", ""):
  192. print "start_node: calling bitcoin-cli -rpcwait getblockcount returned"
  193. devnull.close()
  194. url = "http://rt:rt@%s:%d" % (rpchost or '127.0.0.1', rpc_port(i))
  195. proxy = get_rpc_proxy(url, i, timeout=timewait)
  196. if COVERAGE_DIR:
  197. coverage.write_all_rpc_commands(COVERAGE_DIR, proxy)
  198. return proxy
  199. def start_nodes(num_nodes, dirname, extra_args=None, rpchost=None, binary=None):
  200. """
  201. Start multiple bitcoinds, return RPC connections to them
  202. """
  203. if extra_args is None: extra_args = [ None for i in range(num_nodes) ]
  204. if binary is None: binary = [ None for i in range(num_nodes) ]
  205. return [ start_node(i, dirname, extra_args[i], rpchost, binary=binary[i]) for i in range(num_nodes) ]
  206. def log_filename(dirname, n_node, logname):
  207. return os.path.join(dirname, "node"+str(n_node), "regtest", logname)
  208. def stop_node(node, i):
  209. node.stop()
  210. bitcoind_processes[i].wait()
  211. del bitcoind_processes[i]
  212. def stop_nodes(nodes):
  213. for node in nodes:
  214. node.stop()
  215. del nodes[:] # Emptying array closes connections as a side effect
  216. def set_node_times(nodes, t):
  217. for node in nodes:
  218. node.setmocktime(t)
  219. def wait_bitcoinds():
  220. # Wait for all bitcoinds to cleanly exit
  221. for bitcoind in bitcoind_processes.values():
  222. bitcoind.wait()
  223. bitcoind_processes.clear()
  224. def connect_nodes(from_connection, node_num):
  225. ip_port = "127.0.0.1:"+str(p2p_port(node_num))
  226. from_connection.addnode(ip_port, "onetry")
  227. # poll until version handshake complete to avoid race conditions
  228. # with transaction relaying
  229. while any(peer['version'] == 0 for peer in from_connection.getpeerinfo()):
  230. time.sleep(0.1)
  231. def connect_nodes_bi(nodes, a, b):
  232. connect_nodes(nodes[a], b)
  233. connect_nodes(nodes[b], a)
  234. def find_output(node, txid, amount):
  235. """
  236. Return index to output of txid with value amount
  237. Raises exception if there is none.
  238. """
  239. txdata = node.getrawtransaction(txid, 1)
  240. for i in range(len(txdata["vout"])):
  241. if txdata["vout"][i]["value"] == amount:
  242. return i
  243. raise RuntimeError("find_output txid %s : %s not found"%(txid,str(amount)))
  244. def gather_inputs(from_node, amount_needed, confirmations_required=1):
  245. """
  246. Return a random set of unspent txouts that are enough to pay amount_needed
  247. """
  248. assert(confirmations_required >=0)
  249. utxo = from_node.listunspent(confirmations_required)
  250. random.shuffle(utxo)
  251. inputs = []
  252. total_in = Decimal("0.00000000")
  253. while total_in < amount_needed and len(utxo) > 0:
  254. t = utxo.pop()
  255. total_in += t["amount"]
  256. inputs.append({ "txid" : t["txid"], "vout" : t["vout"], "address" : t["address"] } )
  257. if total_in < amount_needed:
  258. raise RuntimeError("Insufficient funds: need %d, have %d"%(amount_needed, total_in))
  259. return (total_in, inputs)
  260. def make_change(from_node, amount_in, amount_out, fee):
  261. """
  262. Create change output(s), return them
  263. """
  264. outputs = {}
  265. amount = amount_out+fee
  266. change = amount_in - amount
  267. if change > amount*2:
  268. # Create an extra change output to break up big inputs
  269. change_address = from_node.getnewaddress()
  270. # Split change in two, being careful of rounding:
  271. outputs[change_address] = Decimal(change/2).quantize(Decimal('0.00000001'), rounding=ROUND_DOWN)
  272. change = amount_in - amount - outputs[change_address]
  273. if change > 0:
  274. outputs[from_node.getnewaddress()] = change
  275. return outputs
  276. def send_zeropri_transaction(from_node, to_node, amount, fee):
  277. """
  278. Create&broadcast a zero-priority transaction.
  279. Returns (txid, hex-encoded-txdata)
  280. Ensures transaction is zero-priority by first creating a send-to-self,
  281. then using its output
  282. """
  283. # Create a send-to-self with confirmed inputs:
  284. self_address = from_node.getnewaddress()
  285. (total_in, inputs) = gather_inputs(from_node, amount+fee*2)
  286. outputs = make_change(from_node, total_in, amount+fee, fee)
  287. outputs[self_address] = float(amount+fee)
  288. self_rawtx = from_node.createrawtransaction(inputs, outputs)
  289. self_signresult = from_node.signrawtransaction(self_rawtx)
  290. self_txid = from_node.sendrawtransaction(self_signresult["hex"], True)
  291. vout = find_output(from_node, self_txid, amount+fee)
  292. # Now immediately spend the output to create a 1-input, 1-output
  293. # zero-priority transaction:
  294. inputs = [ { "txid" : self_txid, "vout" : vout } ]
  295. outputs = { to_node.getnewaddress() : float(amount) }
  296. rawtx = from_node.createrawtransaction(inputs, outputs)
  297. signresult = from_node.signrawtransaction(rawtx)
  298. txid = from_node.sendrawtransaction(signresult["hex"], True)
  299. return (txid, signresult["hex"])
  300. def random_zeropri_transaction(nodes, amount, min_fee, fee_increment, fee_variants):
  301. """
  302. Create a random zero-priority transaction.
  303. Returns (txid, hex-encoded-transaction-data, fee)
  304. """
  305. from_node = random.choice(nodes)
  306. to_node = random.choice(nodes)
  307. fee = min_fee + fee_increment*random.randint(0,fee_variants)
  308. (txid, txhex) = send_zeropri_transaction(from_node, to_node, amount, fee)
  309. return (txid, txhex, fee)
  310. def random_transaction(nodes, amount, min_fee, fee_increment, fee_variants):
  311. """
  312. Create a random transaction.
  313. Returns (txid, hex-encoded-transaction-data, fee)
  314. """
  315. from_node = random.choice(nodes)
  316. to_node = random.choice(nodes)
  317. fee = min_fee + fee_increment*random.randint(0,fee_variants)
  318. (total_in, inputs) = gather_inputs(from_node, amount+fee)
  319. outputs = make_change(from_node, total_in, amount, fee)
  320. outputs[to_node.getnewaddress()] = float(amount)
  321. rawtx = from_node.createrawtransaction(inputs, outputs)
  322. signresult = from_node.signrawtransaction(rawtx)
  323. txid = from_node.sendrawtransaction(signresult["hex"], True)
  324. return (txid, signresult["hex"], fee)
  325. def assert_equal(thing1, thing2):
  326. if thing1 != thing2:
  327. raise AssertionError("%s != %s"%(str(thing1),str(thing2)))
  328. def assert_greater_than(thing1, thing2):
  329. if thing1 <= thing2:
  330. raise AssertionError("%s <= %s"%(str(thing1),str(thing2)))
  331. def assert_raises(exc, fun, *args, **kwds):
  332. try:
  333. fun(*args, **kwds)
  334. except exc:
  335. pass
  336. except Exception as e:
  337. raise AssertionError("Unexpected exception raised: "+type(e).__name__)
  338. else:
  339. raise AssertionError("No exception raised")
  340. def satoshi_round(amount):
  341. return Decimal(amount).quantize(Decimal('0.00000001'), rounding=ROUND_DOWN)