Browse Source

included-tests: generate binary data from test files for inclusion into test binaries

This change moves test data into the binaries rather than reading them from
the disk at runtime.

Advantages:
- Tests become distributable
- Cross-compile friendly. Build on one machine and execute in an arbitrary
  location on another.
- Easier testing for backports. Users can verify that tests pass without having
  to track down corresponding test data.
- More trustworthy test results and easier quality assurance as tests make
  fewer assumptions about their environment.
- Tests could theoretically run at client/daemon startup and exit on failure.

Disadvantages:
- Required 'hexdump' build-dependency. This is a standard bsd tool that should
  be usable everywhere. It is likely already installed on all build-machines.
- Tests can no longer be fudged after build by altering test-data.
pull/1/head
Cory Fields 9 years ago
parent
commit
152e51c7af
  1. 3
      .gitignore
  2. 7
      configure.ac
  3. 2
      src/Makefile.am
  4. 16
      src/Makefile.include
  5. 27
      src/test/Makefile.am
  6. 23
      src/test/alert_tests.cpp
  7. 17
      src/test/base58_tests.cpp
  8. 8
      src/test/canonical_tests.cpp
  9. 0
      src/test/data/alertTests.raw
  10. 33
      src/test/script_tests.cpp
  11. 8
      src/test/transaction_tests.cpp

3
.gitignore vendored

@ -57,6 +57,9 @@ src/qt/res/.dirstamp @@ -57,6 +57,9 @@ src/qt/res/.dirstamp
*.trs
*.dmg
*.json.h
*.raw.h
# Compilation and Qt preprocessor part
*.qm
Makefile

7
configure.ac

@ -151,6 +151,7 @@ AC_PATH_PROGS([LUPDATE], [lupdate-qt4 lupdate4 lupdate],, $qt_bin_path:$PATH) @@ -151,6 +151,7 @@ AC_PATH_PROGS([LUPDATE], [lupdate-qt4 lupdate4 lupdate],, $qt_bin_path:$PATH)
AC_PATH_PROG([PROTOC], [protoc],, $protoc_bin_path:$PATH)
AC_PATH_PROG(CCACHE,ccache)
AC_PATH_PROG(XGETTEXT,xgettext)
AC_PATH_PROG(HEXDUMP,hexdump)
PKG_PROG_PKG_CONFIG
## TODO: Remove these hard-coded paths and flags. They are here for the sake of
@ -353,6 +354,12 @@ AX_BOOST_THREAD @@ -353,6 +354,12 @@ AX_BOOST_THREAD
AX_BOOST_CHRONO
if test x$use_tests = xyes; then
if test x$HEXDUMP = x; then
AC_MSG_ERROR(hexdump is required for tests)
fi
AX_BOOST_UNIT_TEST_FRAMEWORK
fi

2
src/Makefile.am

@ -1,6 +1,6 @@ @@ -1,6 +1,6 @@
AM_CPPFLAGS = $(INCLUDES) -I$(top_builddir)/src/obj \
-I$(top_srcdir)/src/leveldb/include -I$(top_srcdir)/src/leveldb/helpers \
$(BOOST_INCLUDES)
-I$(builddir) $(BOOST_INCLUDES)
AM_LDFLAGS = $(PTHREAD_CFLAGS)
noinst_LIBRARIES = libbitcoin.a

16
src/Makefile.include

@ -49,3 +49,19 @@ moc_%.cpp: %.h @@ -49,3 +49,19 @@ moc_%.cpp: %.h
%.pb.cc %.pb.h: %.proto
test -f $(PROTOC) && $(PROTOC) --cpp_out=$(@D) --proto_path=$(abspath $(<D) $<) || \
echo error: could not build $@
%.json.h: %.json
@$(MKDIR_P) $(@D)
@echo "namespace json_tests{" > $@
@echo "static unsigned const char $(*F)[] = {" >> $@
@$(HEXDUMP) -v -e '8/1 "0x%02x, "' -e '"\n"' $< | $(SED) -e 's/0x ,//g' >> $@
@echo "};};" >> $@
@echo "Generated $@"
%.raw.h: %.raw
@$(MKDIR_P) $(@D)
@echo "namespace alert_tests{" > $@
@echo "static unsigned const char $(*F)[] = {" >> $@
@$(HEXDUMP) -v -e '8/1 "0x%02x, "' -e '"\n"' $< | $(SED) -e 's/0x ,//g' >> $@
@echo "};};" >> $@
@echo "Generated $@"

27
src/test/Makefile.am

@ -10,19 +10,20 @@ bin_PROGRAMS = test_bitcoin @@ -10,19 +10,20 @@ bin_PROGRAMS = test_bitcoin
TESTS = test_bitcoin
TEST_DATA_DIR=$(srcdir)/data
JSON_TEST_FILES= data/script_valid.json \
data/base58_keys_valid.json data/sig_canonical.json \
data/sig_noncanonical.json \
data/base58_encode_decode.json \
data/base58_keys_invalid.json \
data/script_invalid.json data/tx_invalid.json \
data/tx_valid.json
TEST_DATA_FILES= $(TEST_DATA_DIR)/script_valid.json \
$(TEST_DATA_DIR)/base58_keys_valid.json $(TEST_DATA_DIR)/sig_canonical.json \
$(TEST_DATA_DIR)/sig_noncanonical.json \
$(TEST_DATA_DIR)/base58_encode_decode.json $(TEST_DATA_DIR)/alertTests \
$(TEST_DATA_DIR)/base58_keys_invalid.json \
$(TEST_DATA_DIR)/script_invalid.json $(TEST_DATA_DIR)/tx_invalid.json \
$(TEST_DATA_DIR)/tx_valid.json
RAW_TEST_FILES = data/alertTests.raw
BUILT_SOURCES = $(JSON_TEST_FILES:.json=.json.h) $(RAW_TEST_FILES:.raw=.raw.h)
# test_bitcoin binary #
test_bitcoin_CPPFLAGS = $(AM_CPPFLAGS) $(TESTDEFS) \
-DTEST_DATA_DIR=$(abs_srcdir)/data
test_bitcoin_CPPFLAGS = $(AM_CPPFLAGS) $(TESTDEFS)
test_bitcoin_LDADD = $(LIBBITCOIN) $(LIBLEVELDB) $(LIBMEMENV) \
$(BOOST_LIBS) $(BOOST_UNIT_TEST_FRAMEWORK_LIB)
test_bitcoin_SOURCES = accounting_tests.cpp alert_tests.cpp \
@ -33,6 +34,8 @@ test_bitcoin_SOURCES = accounting_tests.cpp alert_tests.cpp \ @@ -33,6 +34,8 @@ test_bitcoin_SOURCES = accounting_tests.cpp alert_tests.cpp \
netbase_tests.cpp pmt_tests.cpp rpc_tests.cpp script_P2SH_tests.cpp \
script_tests.cpp serialize_tests.cpp sigopcount_tests.cpp test_bitcoin.cpp \
transaction_tests.cpp uint160_tests.cpp uint256_tests.cpp util_tests.cpp \
wallet_tests.cpp $(TEST_DATA_FILES)
wallet_tests.cpp $(JSON_TEST_FILES) $(RAW_TEST_FILES)
nodist_test_bitcoin_SOURCES = $(BUILT_SOURCES)
CLEANFILES = *.gcda *.gcno
CLEANFILES = *.gcda *.gcno $(BUILT_SOURCES)

23
src/test/alert_tests.cpp

@ -9,6 +9,7 @@ @@ -9,6 +9,7 @@
#include "alert.h"
#include "serialize.h"
#include "util.h"
#include "data/alertTests.raw.h"
#if 0
//
@ -71,27 +72,13 @@ struct ReadAlerts @@ -71,27 +72,13 @@ struct ReadAlerts
{
ReadAlerts()
{
std::string filename("alertTests");
namespace fs = boost::filesystem;
fs::path testFile = fs::current_path() / "data" / filename;
#ifdef TEST_DATA_DIR
if (!fs::exists(testFile))
{
testFile = fs::path(BOOST_PP_STRINGIZE(TEST_DATA_DIR)) / filename;
}
#endif
FILE* fp = fopen(testFile.string().c_str(), "rb");
if (!fp) return;
CAutoFile filein = CAutoFile(fp, SER_DISK, CLIENT_VERSION);
if (!filein) return;
std::vector<unsigned char> vch(alert_tests::alertTests, alert_tests::alertTests + sizeof(alert_tests::alertTests));
CDataStream stream(vch, SER_DISK, CLIENT_VERSION);
try {
while (!feof(filein))
while (stream.good())
{
CAlert alert;
filein >> alert;
stream >> alert;
alerts.push_back(alert);
}
}

17
src/test/base58_tests.cpp

@ -2,20 +2,21 @@ @@ -2,20 +2,21 @@
#include "json/json_spirit_reader_template.h"
#include "json/json_spirit_writer_template.h"
#include "json/json_spirit_utils.h"
#include "data/base58_encode_decode.json.h"
#include "data/base58_keys_invalid.json.h"
#include "data/base58_keys_valid.json.h"
#include "base58.h"
#include "util.h"
using namespace json_spirit;
extern Array read_json(const std::string& filename);
extern Array read_json(const std::string& jsondata);
BOOST_AUTO_TEST_SUITE(base58_tests)
// Goal: test low-level base58 encoding functionality
BOOST_AUTO_TEST_CASE(base58_EncodeBase58)
{
Array tests = read_json("base58_encode_decode.json");
Array tests = read_json(std::string(json_tests::base58_encode_decode, json_tests::base58_encode_decode + sizeof(json_tests::base58_encode_decode)));
BOOST_FOREACH(Value& tv, tests)
{
Array test = tv.get_array();
@ -36,7 +37,7 @@ BOOST_AUTO_TEST_CASE(base58_EncodeBase58) @@ -36,7 +37,7 @@ BOOST_AUTO_TEST_CASE(base58_EncodeBase58)
// Goal: test low-level base58 decoding functionality
BOOST_AUTO_TEST_CASE(base58_DecodeBase58)
{
Array tests = read_json("base58_encode_decode.json");
Array tests = read_json(std::string(json_tests::base58_encode_decode, json_tests::base58_encode_decode + sizeof(json_tests::base58_encode_decode)));
std::vector<unsigned char> result;
BOOST_FOREACH(Value& tv, tests)
@ -104,7 +105,7 @@ public: @@ -104,7 +105,7 @@ public:
// Goal: check that parsed keys match test payload
BOOST_AUTO_TEST_CASE(base58_keys_valid_parse)
{
Array tests = read_json("base58_keys_valid.json");
Array tests = read_json(std::string(json_tests::base58_keys_valid, json_tests::base58_keys_valid + sizeof(json_tests::base58_keys_valid)));
std::vector<unsigned char> result;
CBitcoinSecret secret;
CBitcoinAddress addr;
@ -163,7 +164,7 @@ BOOST_AUTO_TEST_CASE(base58_keys_valid_parse) @@ -163,7 +164,7 @@ BOOST_AUTO_TEST_CASE(base58_keys_valid_parse)
// Goal: check that generated keys match test vectors
BOOST_AUTO_TEST_CASE(base58_keys_valid_gen)
{
Array tests = read_json("base58_keys_valid.json");
Array tests = read_json(std::string(json_tests::base58_keys_valid, json_tests::base58_keys_valid + sizeof(json_tests::base58_keys_valid)));
std::vector<unsigned char> result;
BOOST_FOREACH(Value& tv, tests)
{
@ -231,7 +232,7 @@ BOOST_AUTO_TEST_CASE(base58_keys_valid_gen) @@ -231,7 +232,7 @@ BOOST_AUTO_TEST_CASE(base58_keys_valid_gen)
// Goal: check that base58 parsing code is robust against a variety of corrupted data
BOOST_AUTO_TEST_CASE(base58_keys_invalid)
{
Array tests = read_json("base58_keys_invalid.json"); // Negative testcases
Array tests = read_json(std::string(json_tests::base58_keys_invalid, json_tests::base58_keys_invalid + sizeof(json_tests::base58_keys_invalid))); // Negative testcases
std::vector<unsigned char> result;
CBitcoinSecret secret;
CBitcoinAddress addr;

8
src/test/canonical_tests.cpp

@ -8,13 +8,15 @@ @@ -8,13 +8,15 @@
#include "key.h"
#include "script.h"
#include "util.h"
#include "data/sig_noncanonical.json.h"
#include "data/sig_canonical.json.h"
using namespace std;
using namespace json_spirit;
// In script_tests.cpp
extern Array read_json(const std::string& filename);
extern Array read_json(const std::string& jsondata);
BOOST_AUTO_TEST_SUITE(canonical_tests)
@ -58,7 +60,7 @@ bool static IsCanonicalSignature_OpenSSL(const std::vector<unsigned char> &vchSi @@ -58,7 +60,7 @@ bool static IsCanonicalSignature_OpenSSL(const std::vector<unsigned char> &vchSi
BOOST_AUTO_TEST_CASE(script_canon)
{
Array tests = read_json("sig_canonical.json");
Array tests = read_json(std::string(json_tests::sig_canonical, json_tests::sig_canonical + sizeof(json_tests::sig_canonical)));
BOOST_FOREACH(Value &tv, tests) {
string test = tv.get_str();
@ -72,7 +74,7 @@ BOOST_AUTO_TEST_CASE(script_canon) @@ -72,7 +74,7 @@ BOOST_AUTO_TEST_CASE(script_canon)
BOOST_AUTO_TEST_CASE(script_noncanon)
{
Array tests = read_json("sig_noncanonical.json");
Array tests = read_json(std::string(json_tests::sig_noncanonical, json_tests::sig_noncanonical + sizeof(json_tests::sig_noncanonical)));
BOOST_FOREACH(Value &tv, tests) {
string test = tv.get_str();

0
src/test/data/alertTests → src/test/data/alertTests.raw

33
src/test/script_tests.cpp

@ -14,6 +14,8 @@ @@ -14,6 +14,8 @@
#include "main.h"
#include "wallet.h"
#include "data/script_invalid.json.h"
#include "data/script_valid.json.h"
using namespace std;
using namespace json_spirit;
@ -90,34 +92,15 @@ ParseScript(string s) @@ -90,34 +92,15 @@ ParseScript(string s)
}
Array
read_json(const std::string& filename)
read_json(const std::string& jsondata)
{
namespace fs = boost::filesystem;
fs::path testFile = fs::current_path() / "data" / filename;
#ifdef TEST_DATA_DIR
if (!fs::exists(testFile))
{
testFile = fs::path(BOOST_PP_STRINGIZE(TEST_DATA_DIR)) / filename;
}
#endif
ifstream ifs(testFile.string().c_str(), ifstream::in);
Value v;
if (!read_stream(ifs, v))
{
if (ifs.fail())
BOOST_ERROR("Cound not find/open " << filename);
else
BOOST_ERROR("JSON syntax error in " << filename);
return Array();
}
if (v.type() != array_type)
if (!read_string(jsondata, v) || v.type() != array_type)
{
BOOST_ERROR(filename << " does not contain a json array");
BOOST_ERROR("Parse error.");
return Array();
}
return v.get_array();
}
@ -130,7 +113,7 @@ BOOST_AUTO_TEST_CASE(script_valid) @@ -130,7 +113,7 @@ BOOST_AUTO_TEST_CASE(script_valid)
// Inner arrays are [ "scriptSig", "scriptPubKey" ]
// ... where scriptSig and scriptPubKey are stringified
// scripts.
Array tests = read_json("script_valid.json");
Array tests = read_json(std::string(json_tests::script_valid, json_tests::script_valid + sizeof(json_tests::script_valid)));
BOOST_FOREACH(Value& tv, tests)
{
@ -154,7 +137,7 @@ BOOST_AUTO_TEST_CASE(script_valid) @@ -154,7 +137,7 @@ BOOST_AUTO_TEST_CASE(script_valid)
BOOST_AUTO_TEST_CASE(script_invalid)
{
// Scripts that should evaluate as invalid
Array tests = read_json("script_invalid.json");
Array tests = read_json(std::string(json_tests::script_invalid, json_tests::script_invalid + sizeof(json_tests::script_invalid)));
BOOST_FOREACH(Value& tv, tests)
{

8
src/test/transaction_tests.cpp

@ -5,12 +5,14 @@ @@ -5,12 +5,14 @@
#include "main.h"
#include "wallet.h"
#include "data/tx_invalid.json.h"
#include "data/tx_valid.json.h"
using namespace std;
using namespace json_spirit;
// In script_tests.cpp
extern Array read_json(const std::string& filename);
extern Array read_json(const std::string& jsondata);
extern CScript ParseScript(string s);
BOOST_AUTO_TEST_SUITE(transaction_tests)
@ -22,7 +24,7 @@ BOOST_AUTO_TEST_CASE(tx_valid) @@ -22,7 +24,7 @@ BOOST_AUTO_TEST_CASE(tx_valid)
// Inner arrays are either [ "comment" ]
// or [[[prevout hash, prevout index, prevout scriptPubKey], [input 2], ...],"], serializedTransaction, enforceP2SH
// ... where all scripts are stringified scripts.
Array tests = read_json("tx_valid.json");
Array tests = read_json(std::string(json_tests::tx_valid, json_tests::tx_valid + sizeof(json_tests::tx_valid)));
BOOST_FOREACH(Value& tv, tests)
{
@ -91,7 +93,7 @@ BOOST_AUTO_TEST_CASE(tx_invalid) @@ -91,7 +93,7 @@ BOOST_AUTO_TEST_CASE(tx_invalid)
// Inner arrays are either [ "comment" ]
// or [[[prevout hash, prevout index, prevout scriptPubKey], [input 2], ...],"], serializedTransaction, enforceP2SH
// ... where all scripts are stringified scripts.
Array tests = read_json("tx_invalid.json");
Array tests = read_json(std::string(json_tests::tx_invalid, json_tests::tx_invalid + sizeof(json_tests::tx_invalid)));
BOOST_FOREACH(Value& tv, tests)
{

Loading…
Cancel
Save