Browse Source

Add pruneblockchain RPC to enable manual block file pruning.

tags/v0.15.1
mrbandrews 4 years ago
parent
commit
1fc4ec7bf2
8 changed files with 207 additions and 24 deletions
  1. 7
    5
      doc/man/bitcoin-qt.1
  2. 7
    5
      doc/man/bitcoind.1
  3. 110
    3
      qa/rpc-tests/pruning.py
  4. 10
    6
      src/init.cpp
  5. 31
    0
      src/rpc/blockchain.cpp
  6. 1
    0
      src/rpc/client.cpp
  7. 39
    5
      src/validation.cpp
  8. 2
    0
      src/validation.h

+ 7
- 5
doc/man/bitcoin-qt.1 View File

@@ -75,11 +75,13 @@ Specify pid file (default: bitcoind.pid)
.HP
\fB\-prune=\fR<n>
.IP
Reduce storage requirements by pruning (deleting) old blocks. This mode
is incompatible with \fB\-txindex\fR and \fB\-rescan\fR. Warning: Reverting
this setting requires re\-downloading the entire blockchain.
(default: 0 = disable pruning blocks, >550 = target size in MiB
to use for block files)
Reduce storage requirements by enabling pruning (deleting) of old blocks.
This allows the pruneblockchain RPC to be called to delete specific blocks,
and enables automatic pruning of old blocks if a target size in MiB is
provided. This mode is incompatible with \fB\-txindex\fR and \fB\-rescan\fR.
Warning: Reverting this setting requires re\-downloading the entire blockchain.
(default: 0 = disable pruning blocks, 1 = allow manual pruning via RPC, >550 =
automatically prune block files to stay under the specified target size in MiB)
.HP
\fB\-reindex\-chainstate\fR
.IP

+ 7
- 5
doc/man/bitcoind.1 View File

@@ -80,11 +80,13 @@ Specify pid file (default: bitcoind.pid)
.HP
\fB\-prune=\fR<n>
.IP
Reduce storage requirements by pruning (deleting) old blocks. This mode
is incompatible with \fB\-txindex\fR and \fB\-rescan\fR. Warning: Reverting
this setting requires re\-downloading the entire blockchain.
(default: 0 = disable pruning blocks, >550 = target size in MiB
to use for block files)
Reduce storage requirements by enabling pruning (deleting) of old blocks.
This allows the pruneblockchain RPC to be called to delete specific blocks,
and enables automatic pruning of old blocks if a target size in MiB is
provided. This mode is incompatible with \fB\-txindex\fR and \fB\-rescan\fR.
Warning: Reverting this setting requires re\-downloading the entire blockchain.
(default: 0 = disable pruning blocks, 1 = allow manual pruning via RPC, >550 =
automatically prune block files to stay under the specified target size in MiB)
.HP
\fB\-reindex\-chainstate\fR
.IP

+ 110
- 3
qa/rpc-tests/pruning.py View File

@@ -25,7 +25,7 @@ class PruneTest(BitcoinTestFramework):
def __init__(self):
super().__init__()
self.setup_clean_chain = True
self.num_nodes = 3
self.num_nodes = 5

# Cache for utxos, as the listunspent may take a long time later in the test
self.utxo_cache_0 = []
@@ -43,10 +43,21 @@ class PruneTest(BitcoinTestFramework):
self.nodes.append(start_node(2, self.options.tmpdir, ["-debug","-maxreceivebuffer=20000","-prune=550"], timewait=900))
self.prunedir = self.options.tmpdir+"/node2/regtest/blocks/"

# Create node 3 to test manual pruning (it will be re-started with manual pruning later)
self.nodes.append(start_node(3, self.options.tmpdir, ["-debug=0","-maxreceivebuffer=20000","-blockmaxsize=999000"], timewait=900))
self.manualdir = self.options.tmpdir+"/node3/regtest/blocks/"

# Create node 4 to test wallet in prune mode, but do not connect
self.nodes.append(start_node(4, self.options.tmpdir, ["-debug=0", "-prune=550"]))

# Determine default relay fee
self.relayfee = self.nodes[0].getnetworkinfo()["relayfee"]

connect_nodes(self.nodes[0], 1)
connect_nodes(self.nodes[1], 2)
connect_nodes(self.nodes[2], 0)
sync_blocks(self.nodes[0:3])
connect_nodes(self.nodes[0], 3)
sync_blocks(self.nodes[0:4])

def create_big_chain(self):
# Start by creating some coinbases we can spend later
@@ -57,7 +68,7 @@ class PruneTest(BitcoinTestFramework):
for i in range(645):
mine_large_block(self.nodes[0], self.utxo_cache_0)

sync_blocks(self.nodes[0:3])
sync_blocks(self.nodes[0:4])

def test_height_min(self):
if not os.path.isfile(self.prunedir+"blk00000.dat"):
@@ -212,6 +223,93 @@ class PruneTest(BitcoinTestFramework):
# Verify we can now have the data for a block previously pruned
assert(self.nodes[2].getblock(self.forkhash)["height"] == self.forkheight)

def manual_test(self):
# at this point, node3 has 995 blocks and has not yet run in prune mode
self.nodes[3] = start_node(3, self.options.tmpdir, ["-debug=0"], timewait=900)
assert_raises_message(JSONRPCException, "not in prune mode", self.nodes[3].pruneblockchain, 500)
stop_node(self.nodes[3],3)

# now re-start in manual pruning mode
self.nodes[3] = start_node(3, self.options.tmpdir, ["-debug=0","-prune=1"], timewait=900)
assert_equal(self.nodes[3].getblockcount(), 995)

# should not prune because chain tip of node 3 (995) < PruneAfterHeight (1000)
assert_raises_message(JSONRPCException, "Blockchain is too short for pruning", self.nodes[3].pruneblockchain, 500)

# mine 6 blocks so we are at height 1001 (i.e., above PruneAfterHeight)
self.nodes[3].generate(6)

# negative and zero inputs should raise an exception
try:
self.nodes[3].pruneblockchain(-10)
raise AssertionError("pruneblockchain(-10) should have failed.")
except:
pass

try:
self.nodes[3].pruneblockchain(0)
raise AssertionError("pruneblockchain(0) should have failed.")
except:
pass

# height=100 too low to prune first block file so this is a no-op
self.nodes[3].pruneblockchain(100)
if not os.path.isfile(self.manualdir+"blk00000.dat"):
raise AssertionError("blk00000.dat is missing when should still be there")

# height=500 should prune first file
self.nodes[3].pruneblockchain(500)
if os.path.isfile(self.manualdir+"blk00000.dat"):
raise AssertionError("blk00000.dat is still there, should be pruned by now")
if not os.path.isfile(self.manualdir+"blk00001.dat"):
raise AssertionError("blk00001.dat is missing when should still be there")

# height=650 should prune second file
self.nodes[3].pruneblockchain(650)
if os.path.isfile(self.manualdir+"blk00001.dat"):
raise AssertionError("blk00001.dat is still there, should be pruned by now")

# height=1000 should not prune anything more, because tip-288 is in blk00002.dat.
self.nodes[3].pruneblockchain(1000)
if not os.path.isfile(self.manualdir+"blk00002.dat"):
raise AssertionError("blk00002.dat is still there, should be pruned by now")

# advance the tip so blk00002.dat and blk00003.dat can be pruned (the last 288 blocks should now be in blk00004.dat)
self.nodes[3].generate(288)
self.nodes[3].pruneblockchain(1000)
if os.path.isfile(self.manualdir+"blk00002.dat"):
raise AssertionError("blk00002.dat is still there, should be pruned by now")
if os.path.isfile(self.manualdir+"blk00003.dat"):
raise AssertionError("blk00003.dat is still there, should be pruned by now")

# stop node, start back up with auto-prune at 550MB, make sure still runs
stop_node(self.nodes[3],3)
self.nodes[3] = start_node(3, self.options.tmpdir, ["-debug=0","-prune=550"], timewait=900)

print("Success")

def wallet_test(self):
# check that the pruning node's wallet is still in good shape
print("Stop and start pruning node to trigger wallet rescan")
try:
stop_node(self.nodes[2], 2)
start_node(2, self.options.tmpdir, ["-debug=1","-prune=550"])
print("Success")
except Exception as detail:
raise AssertionError("Wallet test: unable to re-start the pruning node")

# check that wallet loads loads successfully when restarting a pruned node after IBD.
# this was reported to fail in #7494.
print ("Syncing node 4 to test wallet")
connect_nodes(self.nodes[0], 4)
nds = [self.nodes[0], self.nodes[4]]
sync_blocks(nds)
try:
stop_node(self.nodes[4],4) #stop and start to trigger rescan
start_node(4, self.options.tmpdir, ["-debug=1","-prune=550"])
print ("Success")
except Exception as detail:
raise AssertionError("Wallet test: unable to re-start node4")

def run_test(self):
print("Warning! This test requires 4GB of disk space and takes over 30 mins (up to 2 hours)")
@@ -226,6 +324,9 @@ class PruneTest(BitcoinTestFramework):
# Start by mining a simple chain that all nodes have
# N0=N1=N2 **...*(995)

# stop manual-pruning node with 995 blocks
stop_node(self.nodes[3],3)

print("Check that we haven't started pruning yet because we're below PruneAfterHeight")
self.test_height_min()
# Extend this chain past the PruneAfterHeight
@@ -308,6 +409,12 @@ class PruneTest(BitcoinTestFramework):
#
# N1 doesn't change because 1033 on main chain (*) is invalid

print("Test manual pruning")
self.manual_test()

print("Test wallet re-scan")
self.wallet_test()

print("Done")

if __name__ == '__main__':

+ 10
- 6
src/init.cpp View File

@@ -351,9 +351,9 @@ std::string HelpMessage(HelpMessageMode mode)
#ifndef WIN32
strUsage += HelpMessageOpt("-pid=<file>", strprintf(_("Specify pid file (default: %s)"), BITCOIN_PID_FILENAME));
#endif
strUsage += HelpMessageOpt("-prune=<n>", strprintf(_("Reduce storage requirements by pruning (deleting) old blocks. This mode is incompatible with -txindex and -rescan. "
strUsage += HelpMessageOpt("-prune=<n>", strprintf(_("Reduce storage requirements by enabling pruning (deleting) of old blocks. This allows the pruneblockchain RPC to be called to delete specific blocks, and enables automatic pruning of old blocks if a target size in MiB is provided. This mode is incompatible with -txindex and -rescan. "
"Warning: Reverting this setting requires re-downloading the entire blockchain. "
"(default: 0 = disable pruning blocks, >%u = target size in MiB to use for block files)"), MIN_DISK_SPACE_FOR_BLOCK_FILES / 1024 / 1024));
"(default: 0 = disable pruning blocks, 1 = allow manual pruning via RPC, >%u = automatically prune block files to stay under the specified target size in MiB)"), MIN_DISK_SPACE_FOR_BLOCK_FILES / 1024 / 1024));
strUsage += HelpMessageOpt("-reindex-chainstate", _("Rebuild chain state from the currently indexed blocks"));
strUsage += HelpMessageOpt("-reindex", _("Rebuild chain state and block index from the blk*.dat files on disk"));
#ifndef WIN32
@@ -936,12 +936,16 @@ bool AppInitParameterInteraction()
nScriptCheckThreads = MAX_SCRIPTCHECK_THREADS;

// block pruning; get the amount of disk space (in MiB) to allot for block & undo files
int64_t nSignedPruneTarget = GetArg("-prune", 0) * 1024 * 1024;
if (nSignedPruneTarget < 0) {
int64_t nPruneArg = GetArg("-prune", 0);
if (nPruneArg < 0) {
return InitError(_("Prune cannot be configured with a negative value."));
}
nPruneTarget = (uint64_t) nSignedPruneTarget;
if (nPruneTarget) {
nPruneTarget = (uint64_t) nPruneArg * 1024 * 1024;
if (nPruneArg == 1) { // manual pruning: -prune=1
LogPrintf("Block pruning enabled. Use RPC call pruneblockchain(height) to manually prune block and undo files.\n");
nPruneTarget = std::numeric_limits<uint64_t>::max();
fPruneMode = true;
} else if (nPruneTarget) {
if (nPruneTarget < MIN_DISK_SPACE_FOR_BLOCK_FILES) {
return InitError(strprintf(_("Prune configured below the minimum of %d MiB. Please use a higher number."), MIN_DISK_SPACE_FOR_BLOCK_FILES / 1024 / 1024));
}

+ 31
- 0
src/rpc/blockchain.cpp View File

@@ -814,6 +814,36 @@ static bool GetUTXOStats(CCoinsView *view, CCoinsStats &stats)
return true;
}

UniValue pruneblockchain(const JSONRPCRequest& request)
{
if (request.fHelp || request.params.size() != 1)
throw runtime_error(
"pruneblockchain\n"
"\nArguments:\n"
"1. \"height\" (int, required) The block height to prune up to.\n");

if (!fPruneMode)
throw JSONRPCError(RPC_METHOD_NOT_FOUND, "Cannot prune blocks because node is not in prune mode.");

LOCK(cs_main);

int heightParam = request.params[0].get_int();
if (heightParam < 0)
throw JSONRPCError(RPC_INVALID_PARAMETER, "Negative block height.");

unsigned int height = (unsigned int) heightParam;
unsigned int chainHeight = (unsigned int) chainActive.Height();
if (chainHeight < Params().PruneAfterHeight())
throw JSONRPCError(RPC_INTERNAL_ERROR, "Blockchain is too short for pruning.");
else if (height > chainHeight)
throw JSONRPCError(RPC_INVALID_PARAMETER, "Blockchain is shorter than the attempted prune height.");
else if (height > chainHeight - MIN_BLOCKS_TO_KEEP)
LogPrint("rpc", "Attempt to prune blocks close to the tip. Retaining the minimum number of blocks.");

PruneBlockFilesManual(height);
return NullUniValue;
}

UniValue gettxoutsetinfo(const JSONRPCRequest& request)
{
if (request.fHelp || request.params.size() != 0)
@@ -1384,6 +1414,7 @@ static const CRPCCommand commands[] =
{ "blockchain", "getrawmempool", &getrawmempool, true, {"verbose"} },
{ "blockchain", "gettxout", &gettxout, true, {"txid","n","include_mempool"} },
{ "blockchain", "gettxoutsetinfo", &gettxoutsetinfo, true, {} },
{ "blockchain", "pruneblockchain", &pruneblockchain, true, {"height"} },
{ "blockchain", "verifychain", &verifychain, true, {"checklevel","nblocks"} },

{ "blockchain", "preciousblock", &preciousblock, true, {"blockhash"} },

+ 1
- 0
src/rpc/client.cpp View File

@@ -103,6 +103,7 @@ static const CRPCConvertParam vRPCConvertParams[] =
{ "importmulti", 1, "options" },
{ "verifychain", 0, "checklevel" },
{ "verifychain", 1, "nblocks" },
{ "pruneblockchain", 0, "height" },
{ "keypoolrefill", 0, "newsize" },
{ "getrawmempool", 0, "verbose" },
{ "estimatefee", 0, "nblocks" },

+ 39
- 5
src/validation.cpp View File

@@ -185,7 +185,8 @@ enum FlushStateMode {
};

// See definition for documentation
bool static FlushStateToDisk(CValidationState &state, FlushStateMode mode);
bool static FlushStateToDisk(CValidationState &state, FlushStateMode mode, int nManualPruneHeight=0);
void FindFilesToPruneManual(std::set<int>& setFilesToPrune, int nManualPruneHeight);

bool IsFinalTx(const CTransaction &tx, int nBlockHeight, int64_t nBlockTime)
{
@@ -1934,7 +1935,7 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin
* if they're too large, if it's been a while since the last write,
* or always and in all cases if we're in prune mode and are deleting files.
*/
bool static FlushStateToDisk(CValidationState &state, FlushStateMode mode) {
bool static FlushStateToDisk(CValidationState &state, FlushStateMode mode, int nManualPruneHeight) {
int64_t nMempoolUsage = mempool.DynamicMemoryUsage();
const CChainParams& chainparams = Params();
LOCK2(cs_main, cs_LastBlockFile);
@@ -1944,9 +1945,13 @@ bool static FlushStateToDisk(CValidationState &state, FlushStateMode mode) {
std::set<int> setFilesToPrune;
bool fFlushForPrune = false;
try {
if (fPruneMode && fCheckForPruning && !fReindex) {
FindFilesToPrune(setFilesToPrune, chainparams.PruneAfterHeight());
fCheckForPruning = false;
if (fPruneMode && (fCheckForPruning || nManualPruneHeight > 0) && !fReindex) {
if (nManualPruneHeight > 0) {
FindFilesToPruneManual(setFilesToPrune, nManualPruneHeight);
} else {
FindFilesToPrune(setFilesToPrune, chainparams.PruneAfterHeight());
fCheckForPruning = false;
}
if (!setFilesToPrune.empty()) {
fFlushForPrune = true;
if (!fHavePruned) {
@@ -3247,6 +3252,35 @@ void UnlinkPrunedFiles(std::set<int>& setFilesToPrune)
}
}

/* Calculate the block/rev files to delete based on height specified by user with RPC command pruneblockchain */
void FindFilesToPruneManual(std::set<int>& setFilesToPrune, int nManualPruneHeight)
{
assert(fPruneMode && nManualPruneHeight > 0);

LOCK2(cs_main, cs_LastBlockFile);
if (chainActive.Tip() == NULL)
return;

// last block to prune is the lesser of (user-specified height, MIN_BLOCKS_TO_KEEP from the tip)
unsigned int nLastBlockWeCanPrune = min((unsigned)nManualPruneHeight, chainActive.Tip()->nHeight - MIN_BLOCKS_TO_KEEP);
int count=0;
for (int fileNumber = 0; fileNumber < nLastBlockFile; fileNumber++) {
if (vinfoBlockFile[fileNumber].nSize == 0 || vinfoBlockFile[fileNumber].nHeightLast > nLastBlockWeCanPrune)
continue;
PruneOneBlockFile(fileNumber);
setFilesToPrune.insert(fileNumber);
count++;
}
LogPrintf("Prune (Manual): prune_height=%d removed %d blk/rev pairs\n", nLastBlockWeCanPrune, count);
}

/* This function is called from the RPC code for pruneblockchain */
void PruneBlockFilesManual(int nManualPruneHeight)
{
CValidationState state;
FlushStateToDisk(state, FLUSH_STATE_NONE, nManualPruneHeight);
}

/* Calculate the block/rev files that should be deleted to remain under target*/
void FindFilesToPrune(std::set<int>& setFilesToPrune, uint64_t nPruneAfterHeight)
{

+ 2
- 0
src/validation.h View File

@@ -309,6 +309,8 @@ CBlockIndex * InsertBlockIndex(uint256 hash);
void FlushStateToDisk();
/** Prune block files and flush state to disk. */
void PruneAndFlush();
/** Prune block files up to a given height */
void PruneBlockFilesManual(int nPruneUpToHeight);

/** (try to) add transaction to memory pool **/
bool AcceptToMemoryPool(CTxMemPool& pool, CValidationState &state, const CTransactionRef &tx, bool fLimitFree,

Loading…
Cancel
Save