Ви не можете вибрати більше 25 тем Теми мають розпочинатися з літери або цифри, можуть містити дефіси (-) і не повинні перевищувати 35 символів.

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337
  1. #!/usr/bin/env python3
  2. # Copyright (c) 2014-2016 The Bitcoin Core developers
  3. # Distributed under the MIT software license, see the accompanying
  4. # file COPYING or http://www.opensource.org/licenses/mit-license.php.
  5. """Base class for RPC testing."""
  6. from collections import deque
  7. import logging
  8. import optparse
  9. import os
  10. import shutil
  11. import subprocess
  12. import sys
  13. import tempfile
  14. import time
  15. from .util import (
  16. PortSeed,
  17. MAX_NODES,
  18. bitcoind_processes,
  19. check_json_precision,
  20. connect_nodes_bi,
  21. disable_mocktime,
  22. disconnect_nodes,
  23. enable_coverage,
  24. enable_mocktime,
  25. get_mocktime,
  26. get_rpc_proxy,
  27. initialize_datadir,
  28. log_filename,
  29. p2p_port,
  30. rpc_url,
  31. set_node_times,
  32. start_nodes,
  33. stop_node,
  34. stop_nodes,
  35. sync_blocks,
  36. sync_mempools,
  37. wait_for_bitcoind_start,
  38. )
  39. from .authproxy import JSONRPCException
  40. class BitcoinTestFramework(object):
  41. TEST_EXIT_PASSED = 0
  42. TEST_EXIT_FAILED = 1
  43. TEST_EXIT_SKIPPED = 77
  44. def __init__(self):
  45. self.num_nodes = 4
  46. self.setup_clean_chain = False
  47. self.nodes = None
  48. def run_test(self):
  49. raise NotImplementedError
  50. def add_options(self, parser):
  51. pass
  52. def setup_chain(self):
  53. self.log.info("Initializing test directory "+self.options.tmpdir)
  54. if self.setup_clean_chain:
  55. self._initialize_chain_clean(self.options.tmpdir, self.num_nodes)
  56. else:
  57. self._initialize_chain(self.options.tmpdir, self.num_nodes, self.options.cachedir)
  58. def stop_node(self, num_node):
  59. stop_node(self.nodes[num_node], num_node)
  60. def setup_nodes(self):
  61. extra_args = None
  62. if hasattr(self, "extra_args"):
  63. extra_args = self.extra_args
  64. self.nodes = start_nodes(self.num_nodes, self.options.tmpdir, extra_args)
  65. def setup_network(self):
  66. self.setup_nodes()
  67. # Connect the nodes as a "chain". This allows us
  68. # to split the network between nodes 1 and 2 to get
  69. # two halves that can work on competing chains.
  70. for i in range(self.num_nodes - 1):
  71. connect_nodes_bi(self.nodes, i, i + 1)
  72. self.sync_all()
  73. def split_network(self):
  74. """
  75. Split the network of four nodes into nodes 0/1 and 2/3.
  76. """
  77. disconnect_nodes(self.nodes[1], 2)
  78. disconnect_nodes(self.nodes[2], 1)
  79. self.sync_all([self.nodes[:2], self.nodes[2:]])
  80. def sync_all(self, node_groups=None):
  81. if not node_groups:
  82. node_groups = [self.nodes]
  83. [sync_blocks(group) for group in node_groups]
  84. [sync_mempools(group) for group in node_groups]
  85. def join_network(self):
  86. """
  87. Join the (previously split) network halves together.
  88. """
  89. connect_nodes_bi(self.nodes, 1, 2)
  90. self.sync_all()
  91. def main(self):
  92. parser = optparse.OptionParser(usage="%prog [options]")
  93. parser.add_option("--nocleanup", dest="nocleanup", default=False, action="store_true",
  94. help="Leave bitcoinds and test.* datadir on exit or error")
  95. parser.add_option("--noshutdown", dest="noshutdown", default=False, action="store_true",
  96. help="Don't stop bitcoinds after the test execution")
  97. parser.add_option("--srcdir", dest="srcdir", default=os.path.normpath(os.path.dirname(os.path.realpath(__file__))+"/../../../src"),
  98. help="Source directory containing bitcoind/bitcoin-cli (default: %default)")
  99. parser.add_option("--cachedir", dest="cachedir", default=os.path.normpath(os.path.dirname(os.path.realpath(__file__))+"/../../cache"),
  100. help="Directory for caching pregenerated datadirs")
  101. parser.add_option("--tmpdir", dest="tmpdir", default=tempfile.mkdtemp(prefix="test"),
  102. help="Root directory for datadirs")
  103. parser.add_option("-l", "--loglevel", dest="loglevel", default="INFO",
  104. help="log events at this level and higher to the console. Can be set to DEBUG, INFO, WARNING, ERROR or CRITICAL. Passing --loglevel DEBUG will output all logs to console. Note that logs at all levels are always written to the test_framework.log file in the temporary test directory.")
  105. parser.add_option("--tracerpc", dest="trace_rpc", default=False, action="store_true",
  106. help="Print out all RPC calls as they are made")
  107. parser.add_option("--portseed", dest="port_seed", default=os.getpid(), type='int',
  108. help="The seed to use for assigning port numbers (default: current process id)")
  109. parser.add_option("--coveragedir", dest="coveragedir",
  110. help="Write tested RPC commands into this directory")
  111. self.add_options(parser)
  112. (self.options, self.args) = parser.parse_args()
  113. # backup dir variable for removal at cleanup
  114. self.options.root, self.options.tmpdir = self.options.tmpdir, self.options.tmpdir + '/' + str(self.options.port_seed)
  115. if self.options.coveragedir:
  116. enable_coverage(self.options.coveragedir)
  117. PortSeed.n = self.options.port_seed
  118. os.environ['PATH'] = self.options.srcdir+":"+self.options.srcdir+"/qt:"+os.environ['PATH']
  119. check_json_precision()
  120. # Set up temp directory and start logging
  121. os.makedirs(self.options.tmpdir, exist_ok=False)
  122. self._start_logging()
  123. success = False
  124. try:
  125. self.setup_chain()
  126. self.setup_network()
  127. self.run_test()
  128. success = True
  129. except JSONRPCException as e:
  130. self.log.exception("JSONRPC error")
  131. except AssertionError as e:
  132. self.log.exception("Assertion failed")
  133. except KeyError as e:
  134. self.log.exception("Key error")
  135. except Exception as e:
  136. self.log.exception("Unexpected exception caught during testing")
  137. except KeyboardInterrupt as e:
  138. self.log.warning("Exiting after keyboard interrupt")
  139. if not self.options.noshutdown:
  140. self.log.info("Stopping nodes")
  141. stop_nodes(self.nodes)
  142. else:
  143. self.log.info("Note: bitcoinds were not stopped and may still be running")
  144. if not self.options.nocleanup and not self.options.noshutdown and success:
  145. self.log.info("Cleaning up")
  146. shutil.rmtree(self.options.tmpdir)
  147. if not os.listdir(self.options.root):
  148. os.rmdir(self.options.root)
  149. else:
  150. self.log.warning("Not cleaning up dir %s" % self.options.tmpdir)
  151. if os.getenv("PYTHON_DEBUG", ""):
  152. # Dump the end of the debug logs, to aid in debugging rare
  153. # travis failures.
  154. import glob
  155. filenames = [self.options.tmpdir + "/test_framework.log"]
  156. filenames += glob.glob(self.options.tmpdir + "/node*/regtest/debug.log")
  157. MAX_LINES_TO_PRINT = 1000
  158. for fn in filenames:
  159. try:
  160. with open(fn, 'r') as f:
  161. print("From" , fn, ":")
  162. print("".join(deque(f, MAX_LINES_TO_PRINT)))
  163. except OSError:
  164. print("Opening file %s failed." % fn)
  165. traceback.print_exc()
  166. if success:
  167. self.log.info("Tests successful")
  168. sys.exit(self.TEST_EXIT_PASSED)
  169. else:
  170. self.log.error("Test failed. Test logging available at %s/test_framework.log", self.options.tmpdir)
  171. logging.shutdown()
  172. sys.exit(self.TEST_EXIT_FAILED)
  173. def _start_logging(self):
  174. # Add logger and logging handlers
  175. self.log = logging.getLogger('TestFramework')
  176. self.log.setLevel(logging.DEBUG)
  177. # Create file handler to log all messages
  178. fh = logging.FileHandler(self.options.tmpdir + '/test_framework.log')
  179. fh.setLevel(logging.DEBUG)
  180. # Create console handler to log messages to stderr. By default this logs only error messages, but can be configured with --loglevel.
  181. ch = logging.StreamHandler(sys.stdout)
  182. # User can provide log level as a number or string (eg DEBUG). loglevel was caught as a string, so try to convert it to an int
  183. ll = int(self.options.loglevel) if self.options.loglevel.isdigit() else self.options.loglevel.upper()
  184. ch.setLevel(ll)
  185. # Format logs the same as bitcoind's debug.log with microprecision (so log files can be concatenated and sorted)
  186. formatter = logging.Formatter(fmt = '%(asctime)s.%(msecs)03d000 %(name)s (%(levelname)s): %(message)s', datefmt='%Y-%m-%d %H:%M:%S')
  187. formatter.converter = time.gmtime
  188. fh.setFormatter(formatter)
  189. ch.setFormatter(formatter)
  190. # add the handlers to the logger
  191. self.log.addHandler(fh)
  192. self.log.addHandler(ch)
  193. if self.options.trace_rpc:
  194. rpc_logger = logging.getLogger("BitcoinRPC")
  195. rpc_logger.setLevel(logging.DEBUG)
  196. rpc_handler = logging.StreamHandler(sys.stdout)
  197. rpc_handler.setLevel(logging.DEBUG)
  198. rpc_logger.addHandler(rpc_handler)
  199. def _initialize_chain(self, test_dir, num_nodes, cachedir):
  200. """Initialize a pre-mined blockchain for use by the test.
  201. Create a cache of a 200-block-long chain (with wallet) for MAX_NODES
  202. Afterward, create num_nodes copies from the cache."""
  203. assert num_nodes <= MAX_NODES
  204. create_cache = False
  205. for i in range(MAX_NODES):
  206. if not os.path.isdir(os.path.join(cachedir, 'node' + str(i))):
  207. create_cache = True
  208. break
  209. if create_cache:
  210. self.log.debug("Creating data directories from cached datadir")
  211. # find and delete old cache directories if any exist
  212. for i in range(MAX_NODES):
  213. if os.path.isdir(os.path.join(cachedir, "node" + str(i))):
  214. shutil.rmtree(os.path.join(cachedir, "node" + str(i)))
  215. # Create cache directories, run bitcoinds:
  216. for i in range(MAX_NODES):
  217. datadir = initialize_datadir(cachedir, i)
  218. args = [os.getenv("BITCOIND", "bitcoind"), "-server", "-keypool=1", "-datadir=" + datadir, "-discover=0"]
  219. if i > 0:
  220. args.append("-connect=127.0.0.1:" + str(p2p_port(0)))
  221. bitcoind_processes[i] = subprocess.Popen(args)
  222. self.log.debug("initialize_chain: bitcoind started, waiting for RPC to come up")
  223. wait_for_bitcoind_start(bitcoind_processes[i], rpc_url(i), i)
  224. self.log.debug("initialize_chain: RPC successfully started")
  225. rpcs = []
  226. for i in range(MAX_NODES):
  227. try:
  228. rpcs.append(get_rpc_proxy(rpc_url(i), i))
  229. except:
  230. self.log.exception("Error connecting to node %d" % i)
  231. sys.exit(1)
  232. # Create a 200-block-long chain; each of the 4 first nodes
  233. # gets 25 mature blocks and 25 immature.
  234. # Note: To preserve compatibility with older versions of
  235. # initialize_chain, only 4 nodes will generate coins.
  236. #
  237. # blocks are created with timestamps 10 minutes apart
  238. # starting from 2010 minutes in the past
  239. enable_mocktime()
  240. block_time = get_mocktime() - (201 * 10 * 60)
  241. for i in range(2):
  242. for peer in range(4):
  243. for j in range(25):
  244. set_node_times(rpcs, block_time)
  245. rpcs[peer].generate(1)
  246. block_time += 10 * 60
  247. # Must sync before next peer starts generating blocks
  248. sync_blocks(rpcs)
  249. # Shut them down, and clean up cache directories:
  250. stop_nodes(rpcs)
  251. disable_mocktime()
  252. for i in range(MAX_NODES):
  253. os.remove(log_filename(cachedir, i, "debug.log"))
  254. os.remove(log_filename(cachedir, i, "db.log"))
  255. os.remove(log_filename(cachedir, i, "peers.dat"))
  256. os.remove(log_filename(cachedir, i, "fee_estimates.dat"))
  257. for i in range(num_nodes):
  258. from_dir = os.path.join(cachedir, "node" + str(i))
  259. to_dir = os.path.join(test_dir, "node" + str(i))
  260. shutil.copytree(from_dir, to_dir)
  261. initialize_datadir(test_dir, i) # Overwrite port/rpcport in bitcoin.conf
  262. def _initialize_chain_clean(self, test_dir, num_nodes):
  263. """Initialize empty blockchain for use by the test.
  264. Create an empty blockchain and num_nodes wallets.
  265. Useful if a test case wants complete control over initialization."""
  266. for i in range(num_nodes):
  267. initialize_datadir(test_dir, i)
  268. # Test framework for doing p2p comparison testing, which sets up some bitcoind
  269. # binaries:
  270. # 1 binary: test binary
  271. # 2 binaries: 1 test binary, 1 ref binary
  272. # n>2 binaries: 1 test binary, n-1 ref binaries
  273. class ComparisonTestFramework(BitcoinTestFramework):
  274. def __init__(self):
  275. super().__init__()
  276. self.num_nodes = 2
  277. self.setup_clean_chain = True
  278. def add_options(self, parser):
  279. parser.add_option("--testbinary", dest="testbinary",
  280. default=os.getenv("BITCOIND", "bitcoind"),
  281. help="bitcoind binary to test")
  282. parser.add_option("--refbinary", dest="refbinary",
  283. default=os.getenv("BITCOIND", "bitcoind"),
  284. help="bitcoind binary to use for reference nodes (if any)")
  285. def setup_network(self):
  286. self.nodes = start_nodes(
  287. self.num_nodes, self.options.tmpdir,
  288. extra_args=[['-whitelist=127.0.0.1']] * self.num_nodes,
  289. binary=[self.options.testbinary] +
  290. [self.options.refbinary]*(self.num_nodes-1))