You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

rpc-tests.py 7.5KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255
  1. #!/usr/bin/env python2
  2. # Copyright (c) 2014-2015 The Bitcoin Core developers
  3. # Distributed under the MIT software license, see the accompanying
  4. # file COPYING or http://www.opensource.org/licenses/mit-license.php.
  5. """
  6. Run Regression Test Suite
  7. This module calls down into individual test cases via subprocess. It will
  8. forward all unrecognized arguments onto the individual test scripts, other
  9. than:
  10. - `-extended`: run the "extended" test suite in addition to the basic one.
  11. - `-win`: signal that this is running in a Windows environment, and we
  12. should run the tests.
  13. - `--coverage`: this generates a basic coverage report for the RPC
  14. interface.
  15. For a description of arguments recognized by test scripts, see
  16. `qa/pull-tester/test_framework/test_framework.py:BitcoinTestFramework.main`.
  17. """
  18. import os
  19. import time
  20. import shutil
  21. import sys
  22. import subprocess
  23. import tempfile
  24. import re
  25. from tests_config import *
  26. #If imported values are not defined then set to zero (or disabled)
  27. if not vars().has_key('ENABLE_WALLET'):
  28. ENABLE_WALLET=0
  29. if not vars().has_key('ENABLE_BITCOIND'):
  30. ENABLE_BITCOIND=0
  31. if not vars().has_key('ENABLE_UTILS'):
  32. ENABLE_UTILS=0
  33. if not vars().has_key('ENABLE_ZMQ'):
  34. ENABLE_ZMQ=0
  35. ENABLE_COVERAGE=0
  36. #Create a set to store arguments and create the passOn string
  37. opts = set()
  38. passOn = ""
  39. p = re.compile("^--")
  40. bold = ("","")
  41. if (os.name == 'posix'):
  42. bold = ('\033[0m', '\033[1m')
  43. for arg in sys.argv[1:]:
  44. if arg == '--coverage':
  45. ENABLE_COVERAGE = 1
  46. elif (p.match(arg) or arg == "-h"):
  47. passOn += " " + arg
  48. else:
  49. opts.add(arg)
  50. #Set env vars
  51. buildDir = BUILDDIR
  52. os.environ["BITCOIND"] = buildDir + '/src/bitcoind' + EXEEXT
  53. os.environ["BITCOINCLI"] = buildDir + '/src/bitcoin-cli' + EXEEXT
  54. #Disable Windows tests by default
  55. if EXEEXT == ".exe" and "-win" not in opts:
  56. print "Win tests currently disabled. Use -win option to enable"
  57. sys.exit(0)
  58. #Tests
  59. testScripts = [
  60. 'wallet.py',
  61. 'listtransactions.py',
  62. 'receivedby.py',
  63. 'mempool_resurrect_test.py',
  64. 'txn_doublespend.py --mineblock',
  65. 'txn_clone.py',
  66. 'getchaintips.py',
  67. 'rawtransactions.py',
  68. 'rest.py',
  69. 'mempool_spendcoinbase.py',
  70. 'mempool_reorg.py',
  71. 'httpbasics.py',
  72. 'multi_rpc.py',
  73. 'zapwallettxes.py',
  74. 'proxy_test.py',
  75. 'merkle_blocks.py',
  76. 'fundrawtransaction.py',
  77. 'signrawtransactions.py',
  78. 'walletbackup.py',
  79. 'nodehandling.py',
  80. 'reindex.py',
  81. 'decodescript.py',
  82. 'p2p-fullblocktest.py',
  83. 'blockchain.py',
  84. 'disablewallet.py',
  85. 'sendheaders.py',
  86. 'keypool.py',
  87. 'prioritise_transaction.py',
  88. ]
  89. testScriptsExt = [
  90. 'bip65-cltv.py',
  91. 'bip65-cltv-p2p.py',
  92. 'bipdersig-p2p.py',
  93. 'bipdersig.py',
  94. 'getblocktemplate_longpoll.py',
  95. 'getblocktemplate_proposals.py',
  96. 'txn_doublespend.py',
  97. 'txn_clone.py --mineblock',
  98. 'pruning.py',
  99. 'forknotify.py',
  100. 'invalidateblock.py',
  101. # 'rpcbind_test.py', #temporary, bug in libevent, see #6655
  102. 'smartfees.py',
  103. 'maxblocksinflight.py',
  104. 'invalidblockrequest.py',
  105. 'p2p-acceptblock.py',
  106. 'mempool_packages.py',
  107. 'maxuploadtarget.py',
  108. 'replace-by-fee.py',
  109. ]
  110. #Enable ZMQ tests
  111. if ENABLE_ZMQ == 1:
  112. testScripts.append('zmq_test.py')
  113. def runtests():
  114. coverage = None
  115. if ENABLE_COVERAGE:
  116. coverage = RPCCoverage()
  117. print("Initializing coverage directory at %s\n" % coverage.dir)
  118. if(ENABLE_WALLET == 1 and ENABLE_UTILS == 1 and ENABLE_BITCOIND == 1):
  119. rpcTestDir = buildDir + '/qa/rpc-tests/'
  120. run_extended = '-extended' in opts
  121. cov_flag = coverage.flag if coverage else ''
  122. flags = " --srcdir %s/src %s %s" % (buildDir, cov_flag, passOn)
  123. #Run Tests
  124. for i in range(len(testScripts)):
  125. if (len(opts) == 0
  126. or (len(opts) == 1 and "-win" in opts )
  127. or run_extended
  128. or testScripts[i] in opts
  129. or re.sub(".py$", "", testScripts[i]) in opts ):
  130. print("Running testscript %s%s%s ..." % (bold[1], testScripts[i], bold[0]))
  131. time0 = time.time()
  132. subprocess.check_call(
  133. rpcTestDir + testScripts[i] + flags, shell=True)
  134. print("Duration: %s s\n" % (int(time.time() - time0)))
  135. # exit if help is called so we print just one set of
  136. # instructions
  137. p = re.compile(" -h| --help")
  138. if p.match(passOn):
  139. sys.exit(0)
  140. # Run Extended Tests
  141. for i in range(len(testScriptsExt)):
  142. if (run_extended or testScriptsExt[i] in opts
  143. or re.sub(".py$", "", testScriptsExt[i]) in opts):
  144. print(
  145. "Running 2nd level testscript "
  146. + "%s%s%s ..." % (bold[1], testScriptsExt[i], bold[0]))
  147. time0 = time.time()
  148. subprocess.check_call(
  149. rpcTestDir + testScriptsExt[i] + flags, shell=True)
  150. print("Duration: %s s\n" % (int(time.time() - time0)))
  151. if coverage:
  152. coverage.report_rpc_coverage()
  153. print("Cleaning up coverage data")
  154. coverage.cleanup()
  155. else:
  156. print "No rpc tests to run. Wallet, utils, and bitcoind must all be enabled"
  157. class RPCCoverage(object):
  158. """
  159. Coverage reporting utilities for pull-tester.
  160. Coverage calculation works by having each test script subprocess write
  161. coverage files into a particular directory. These files contain the RPC
  162. commands invoked during testing, as well as a complete listing of RPC
  163. commands per `bitcoin-cli help` (`rpc_interface.txt`).
  164. After all tests complete, the commands run are combined and diff'd against
  165. the complete list to calculate uncovered RPC commands.
  166. See also: qa/rpc-tests/test_framework/coverage.py
  167. """
  168. def __init__(self):
  169. self.dir = tempfile.mkdtemp(prefix="coverage")
  170. self.flag = '--coveragedir %s' % self.dir
  171. def report_rpc_coverage(self):
  172. """
  173. Print out RPC commands that were unexercised by tests.
  174. """
  175. uncovered = self._get_uncovered_rpc_commands()
  176. if uncovered:
  177. print("Uncovered RPC commands:")
  178. print("".join((" - %s\n" % i) for i in sorted(uncovered)))
  179. else:
  180. print("All RPC commands covered.")
  181. def cleanup(self):
  182. return shutil.rmtree(self.dir)
  183. def _get_uncovered_rpc_commands(self):
  184. """
  185. Return a set of currently untested RPC commands.
  186. """
  187. # This is shared from `qa/rpc-tests/test-framework/coverage.py`
  188. REFERENCE_FILENAME = 'rpc_interface.txt'
  189. COVERAGE_FILE_PREFIX = 'coverage.'
  190. coverage_ref_filename = os.path.join(self.dir, REFERENCE_FILENAME)
  191. coverage_filenames = set()
  192. all_cmds = set()
  193. covered_cmds = set()
  194. if not os.path.isfile(coverage_ref_filename):
  195. raise RuntimeError("No coverage reference found")
  196. with open(coverage_ref_filename, 'r') as f:
  197. all_cmds.update([i.strip() for i in f.readlines()])
  198. for root, dirs, files in os.walk(self.dir):
  199. for filename in files:
  200. if filename.startswith(COVERAGE_FILE_PREFIX):
  201. coverage_filenames.add(os.path.join(root, filename))
  202. for filename in coverage_filenames:
  203. with open(filename, 'r') as f:
  204. covered_cmds.update([i.strip() for i in f.readlines()])
  205. return all_cmds - covered_cmds
  206. if __name__ == '__main__':
  207. runtests()