mirror of
https://git.wownero.com/wownero/wownero.git
synced 2024-08-15 01:03:23 +00:00
blockchain_prune: faster
on my anecdotal SSD, goes from about 9 hours to 1h20.
This commit is contained in:
moneromooo-monero
wowario
parent
27a9a417dc
commit
bfc28a3bfe
1 changed files with 87 additions and 41 deletions
|
|
@ -52,9 +52,11 @@ using namespace cryptonote;
|
|||
static std::string db_path;
|
||||
|
||||
// default to fast:1
|
||||
static uint64_t records_per_sync = 128;
|
||||
static uint64_t records_per_sync = 16 * 65536;
|
||||
static const size_t slack = 512 * 1024 * 1024;
|
||||
|
||||
static std::vector<bool> is_v1;
|
||||
|
||||
static std::error_code replace_file(const boost::filesystem::path& replacement_name, const boost::filesystem::path& replaced_name)
|
||||
{
|
||||
std::error_code ec = tools::replace_file(replacement_name.string(), replaced_name.string());
|
||||
|
|
@ -89,6 +91,14 @@ static void close(MDB_env *env)
|
|||
mdb_env_close(env);
|
||||
}
|
||||
|
||||
static void mark_v1_tx(const MDB_val &k, const MDB_val &v)
|
||||
{
|
||||
const uint64_t tx_id = *(const uint64_t*)k.mv_data;
|
||||
if (tx_id >= is_v1.size())
|
||||
is_v1.resize(tx_id + 1, false);
|
||||
is_v1[tx_id] = cryptonote::is_v1_tx(cryptonote::blobdata_ref{(const char*)v.mv_data, v.mv_size});
|
||||
}
|
||||
|
||||
static void add_size(MDB_env *env, uint64_t bytes)
|
||||
{
|
||||
try
|
||||
|
|
@ -136,7 +146,7 @@ static void check_resize(MDB_env *env, size_t bytes)
|
|||
add_size(env, size_used + bytes + 2 * slack - mei.me_mapsize);
|
||||
}
|
||||
|
||||
static bool resize_point(size_t nrecords, MDB_env *env, MDB_txn **txn, size_t &bytes)
|
||||
static bool resize_point(size_t &nrecords, MDB_env *env, MDB_txn **txn, size_t &bytes)
|
||||
{
|
||||
if (nrecords % records_per_sync && bytes <= slack / 2)
|
||||
return false;
|
||||
|
|
@ -146,10 +156,11 @@ static bool resize_point(size_t nrecords, MDB_env *env, MDB_txn **txn, size_t &b
|
|||
dbr = mdb_txn_begin(env, NULL, 0, txn);
|
||||
if (dbr) throw std::runtime_error("Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr)));
|
||||
bytes = 0;
|
||||
nrecords = 0;
|
||||
return true;
|
||||
}
|
||||
|
||||
static void copy_table(MDB_env *env0, MDB_env *env1, const char *table, unsigned int flags, unsigned int putflags, int (*cmp)(const MDB_val*, const MDB_val*)=0)
|
||||
static void copy_table(MDB_env *env0, MDB_env *env1, const char *table, unsigned int flags, unsigned int putflags, int (*cmp)(const MDB_val*, const MDB_val*)=0, void (*f)(const MDB_val&, const MDB_val&) = 0)
|
||||
{
|
||||
MDB_dbi dbi0, dbi1;
|
||||
MDB_txn *txn0, *txn1;
|
||||
|
|
@ -200,6 +211,11 @@ static void copy_table(MDB_env *env0, MDB_env *env1, const char *table, unsigned
|
|||
dbr = mdb_cursor_open(txn1, dbi1, &cur1);
|
||||
if (dbr) throw std::runtime_error("Failed to create LMDB cursor: " + std::string(mdb_strerror(dbr)));
|
||||
|
||||
if (flags & MDB_DUPSORT)
|
||||
putflags |= MDB_APPENDDUP;
|
||||
else
|
||||
putflags |= MDB_APPEND;
|
||||
|
||||
MDB_val k;
|
||||
MDB_val v;
|
||||
MDB_cursor_op op = MDB_FIRST;
|
||||
|
|
@ -214,7 +230,8 @@ static void copy_table(MDB_env *env0, MDB_env *env1, const char *table, unsigned
|
|||
throw std::runtime_error("Failed to enumerate " + std::string(table) + " records: " + std::string(mdb_strerror(ret)));
|
||||
|
||||
bytes += k.mv_size + v.mv_size;
|
||||
if (resize_point(++nrecords, env1, &txn1, bytes))
|
||||
++nrecords;
|
||||
if (resize_point(nrecords, env1, &txn1, bytes))
|
||||
{
|
||||
dbr = mdb_cursor_open(txn1, dbi1, &cur1);
|
||||
if (dbr) throw std::runtime_error("Failed to create LMDB cursor: " + std::string(mdb_strerror(dbr)));
|
||||
|
|
@ -223,6 +240,9 @@ static void copy_table(MDB_env *env0, MDB_env *env1, const char *table, unsigned
|
|||
ret = mdb_cursor_put(cur1, &k, &v, putflags);
|
||||
if (ret)
|
||||
throw std::runtime_error("Failed to write " + std::string(table) + " record: " + std::string(mdb_strerror(ret)));
|
||||
|
||||
if (f)
|
||||
(*f)(k, v);
|
||||
}
|
||||
|
||||
mdb_cursor_close(cur1);
|
||||
|
|
@ -235,17 +255,6 @@ static void copy_table(MDB_env *env0, MDB_env *env1, const char *table, unsigned
|
|||
mdb_dbi_close(env0, dbi0);
|
||||
}
|
||||
|
||||
static bool is_v1_tx(MDB_cursor *c_txs_pruned, MDB_val *tx_id)
|
||||
{
|
||||
MDB_val v;
|
||||
int ret = mdb_cursor_get(c_txs_pruned, tx_id, &v, MDB_SET);
|
||||
if (ret)
|
||||
throw std::runtime_error("Failed to find transaction pruned data: " + std::string(mdb_strerror(ret)));
|
||||
if (v.mv_size == 0)
|
||||
throw std::runtime_error("Invalid transaction pruned data");
|
||||
return cryptonote::is_v1_tx(cryptonote::blobdata_ref{(const char*)v.mv_data, v.mv_size});
|
||||
}
|
||||
|
||||
static void prune(MDB_env *env0, MDB_env *env1)
|
||||
{
|
||||
MDB_dbi dbi0_blocks, dbi0_txs_pruned, dbi0_txs_prunable, dbi0_tx_indices, dbi1_txs_prunable, dbi1_txs_prunable_tip, dbi1_properties;
|
||||
|
|
@ -324,7 +333,10 @@ static void prune(MDB_env *env0, MDB_env *env1)
|
|||
mdb_dbi_close(env0, dbi0_blocks);
|
||||
const uint64_t blockchain_height = stats.ms_entries;
|
||||
size_t nrecords = 0, bytes = 0;
|
||||
std::vector<bool> prunable_needed;
|
||||
|
||||
// go through all txes tx indices, recording which ones should have their prunable part retained
|
||||
MINFO("Marking prunable txes");
|
||||
MDB_cursor_op op = MDB_FIRST;
|
||||
while (1)
|
||||
{
|
||||
|
|
@ -336,7 +348,8 @@ static void prune(MDB_env *env0, MDB_env *env1)
|
|||
|
||||
const txindex *ti = (const txindex*)v.mv_data;
|
||||
const uint64_t block_height = ti->data.block_id;
|
||||
MDB_val_set(kk, ti->data.tx_id);
|
||||
const uint64_t tx_id = ti->data.tx_id;
|
||||
MDB_val_set(kk, tx_id);
|
||||
if (block_height + CRYPTONOTE_PRUNING_TIP_BLOCKS >= blockchain_height)
|
||||
{
|
||||
MDEBUG(block_height << "/" << blockchain_height << " is in tip");
|
||||
|
|
@ -344,22 +357,23 @@ static void prune(MDB_env *env0, MDB_env *env1)
|
|||
dbr = mdb_cursor_put(cur1_txs_prunable_tip, &kk, &vv, 0);
|
||||
if (dbr) throw std::runtime_error("Failed to write prunable tx tip data: " + std::string(mdb_strerror(dbr)));
|
||||
bytes += kk.mv_size + vv.mv_size;
|
||||
}
|
||||
if (tools::has_unpruned_block(block_height, blockchain_height, pruning_seed) || is_v1_tx(cur0_txs_pruned, &kk))
|
||||
{
|
||||
MDB_val vv;
|
||||
dbr = mdb_cursor_get(cur0_txs_prunable, &kk, &vv, MDB_SET);
|
||||
if (dbr) throw std::runtime_error("Failed to read prunable tx data: " + std::string(mdb_strerror(dbr)));
|
||||
bytes += kk.mv_size + vv.mv_size;
|
||||
if (resize_point(++nrecords, env1, &txn1, bytes))
|
||||
|
||||
++nrecords;
|
||||
if (resize_point(nrecords, env1, &txn1, bytes))
|
||||
{
|
||||
dbr = mdb_cursor_open(txn1, dbi1_txs_prunable, &cur1_txs_prunable);
|
||||
if (dbr) throw std::runtime_error("Failed to create LMDB cursor: " + std::string(mdb_strerror(dbr)));
|
||||
dbr = mdb_cursor_open(txn1, dbi1_txs_prunable_tip, &cur1_txs_prunable_tip);
|
||||
if (dbr) throw std::runtime_error("Failed to create LMDB cursor: " + std::string(mdb_strerror(dbr)));
|
||||
}
|
||||
dbr = mdb_cursor_put(cur1_txs_prunable, &kk, &vv, 0);
|
||||
if (dbr) throw std::runtime_error("Failed to write prunable tx data: " + std::string(mdb_strerror(dbr)));
|
||||
}
|
||||
if (tx_id >= is_v1.size())
|
||||
throw std::runtime_error("tx_id out of range of is_v1 vector");
|
||||
if (tools::has_unpruned_block(block_height, blockchain_height, pruning_seed) || is_v1[tx_id])
|
||||
{
|
||||
if (tx_id >= prunable_needed.size())
|
||||
prunable_needed.resize(tx_id + 1, false);
|
||||
prunable_needed[tx_id] = true;
|
||||
}
|
||||
else
|
||||
{
|
||||
|
|
@ -367,6 +381,37 @@ static void prune(MDB_env *env0, MDB_env *env1)
|
|||
}
|
||||
}
|
||||
|
||||
// go through prunable parts, carrying over those we need
|
||||
MINFO("Copying retained prunable data");
|
||||
op = MDB_FIRST;
|
||||
while (1)
|
||||
{
|
||||
int ret = mdb_cursor_get(cur0_txs_prunable, &k, &v, op);
|
||||
op = MDB_NEXT;
|
||||
if (ret == MDB_NOTFOUND)
|
||||
break;
|
||||
if (ret) throw std::runtime_error("Failed to enumerate records: " + std::string(mdb_strerror(ret)));
|
||||
|
||||
const uint64_t tx_id = *(const uint64_t*)k.mv_data;
|
||||
if (tx_id >= prunable_needed.size())
|
||||
throw std::runtime_error("tx_id out of range of prunable_needed vector");
|
||||
if (prunable_needed[tx_id])
|
||||
{
|
||||
dbr = mdb_cursor_put(cur1_txs_prunable, &k, &v, MDB_APPEND);
|
||||
if (dbr) throw std::runtime_error("Failed to write prunable tx data: " + std::string(mdb_strerror(dbr)));
|
||||
|
||||
bytes += k.mv_size + v.mv_size;
|
||||
++nrecords;
|
||||
if (resize_point(nrecords, env1, &txn1, bytes))
|
||||
{
|
||||
dbr = mdb_cursor_open(txn1, dbi1_txs_prunable, &cur1_txs_prunable);
|
||||
if (dbr) throw std::runtime_error("Failed to create LMDB cursor: " + std::string(mdb_strerror(dbr)));
|
||||
dbr = mdb_cursor_open(txn1, dbi1_txs_prunable_tip, &cur1_txs_prunable_tip);
|
||||
if (dbr) throw std::runtime_error("Failed to create LMDB cursor: " + std::string(mdb_strerror(dbr)));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
mdb_cursor_close(cur1_txs_prunable_tip);
|
||||
mdb_cursor_close(cur1_txs_prunable);
|
||||
mdb_cursor_close(cur0_txs_prunable);
|
||||
|
|
@ -419,7 +464,7 @@ static bool parse_db_sync_mode(std::string db_sync_mode, uint64_t &db_flags)
|
|||
else if(options[0] == "fastest")
|
||||
{
|
||||
db_flags = DBF_FASTEST;
|
||||
records_per_sync = 1000; // default to fastest:async:1000
|
||||
// default to fastest:async:N
|
||||
}
|
||||
else
|
||||
return false;
|
||||
|
|
@ -455,7 +500,7 @@ int main(int argc, char* argv[])
|
|||
const command_line::arg_descriptor<std::string> arg_db_sync_mode = {
|
||||
"db-sync-mode"
|
||||
, "Specify sync option, using format [safe|fast|fastest]:[nrecords_per_sync]."
|
||||
, "fast:1000"
|
||||
, "fast:" + std::to_string(records_per_sync)
|
||||
};
|
||||
const command_line::arg_descriptor<bool> arg_copy_pruned_database = {"copy-pruned-database", "Copy database anyway if already pruned"};
|
||||
|
||||
|
|
@ -601,26 +646,27 @@ int main(int argc, char* argv[])
|
|||
MDB_env *env0 = NULL, *env1 = NULL;
|
||||
open(env0, paths[0], db_flags, true);
|
||||
open(env1, paths[1], db_flags, false);
|
||||
copy_table(env0, env1, "blocks", MDB_INTEGERKEY, MDB_APPEND);
|
||||
copy_table(env0, env1, "block_info", MDB_INTEGERKEY | MDB_DUPSORT| MDB_DUPFIXED, MDB_APPENDDUP, BlockchainLMDB::compare_uint64);
|
||||
copy_table(env0, env1, "blocks", MDB_INTEGERKEY, 0);
|
||||
copy_table(env0, env1, "block_info", MDB_INTEGERKEY | MDB_DUPSORT| MDB_DUPFIXED, 0, BlockchainLMDB::compare_uint64);
|
||||
copy_table(env0, env1, "block_heights", MDB_INTEGERKEY | MDB_DUPSORT| MDB_DUPFIXED, 0, BlockchainLMDB::compare_hash32);
|
||||
//copy_table(env0, env1, "txs", MDB_INTEGERKEY);
|
||||
copy_table(env0, env1, "txs_pruned", MDB_INTEGERKEY, MDB_APPEND);
|
||||
copy_table(env0, env1, "txs_prunable_hash", MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, MDB_APPEND);
|
||||
copy_table(env0, env1, "txs_pruned", MDB_INTEGERKEY, 0, NULL, &mark_v1_tx);
|
||||
copy_table(env0, env1, "txs_prunable_hash", MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, 0);
|
||||
// not copied: prunable, prunable_tip
|
||||
copy_table(env0, env1, "tx_indices", MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, 0, BlockchainLMDB::compare_hash32);
|
||||
copy_table(env0, env1, "tx_outputs", MDB_INTEGERKEY, MDB_APPEND);
|
||||
copy_table(env0, env1, "output_txs", MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, MDB_APPENDDUP, BlockchainLMDB::compare_uint64);
|
||||
copy_table(env0, env1, "output_amounts", MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, MDB_APPENDDUP, BlockchainLMDB::compare_uint64);
|
||||
copy_table(env0, env1, "spent_keys", MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, MDB_NODUPDATA, BlockchainLMDB::compare_hash32);
|
||||
copy_table(env0, env1, "txpool_meta", 0, MDB_NODUPDATA, BlockchainLMDB::compare_hash32);
|
||||
copy_table(env0, env1, "txpool_blob", 0, MDB_NODUPDATA, BlockchainLMDB::compare_hash32);
|
||||
copy_table(env0, env1, "hf_versions", MDB_INTEGERKEY, MDB_APPEND);
|
||||
copy_table(env0, env1, "tx_outputs", MDB_INTEGERKEY, 0);
|
||||
copy_table(env0, env1, "output_txs", MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, 0, BlockchainLMDB::compare_uint64);
|
||||
copy_table(env0, env1, "output_amounts", MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, 0, BlockchainLMDB::compare_uint64);
|
||||
copy_table(env0, env1, "spent_keys", MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, 0, BlockchainLMDB::compare_hash32);
|
||||
copy_table(env0, env1, "txpool_meta", 0, 0, BlockchainLMDB::compare_hash32);
|
||||
copy_table(env0, env1, "txpool_blob", 0, 0, BlockchainLMDB::compare_hash32);
|
||||
copy_table(env0, env1, "alt_blocks", 0, 0, BlockchainLMDB::compare_hash32);
|
||||
copy_table(env0, env1, "hf_versions", MDB_INTEGERKEY, 0);
|
||||
copy_table(env0, env1, "properties", 0, 0, BlockchainLMDB::compare_string);
|
||||
if (already_pruned)
|
||||
{
|
||||
copy_table(env0, env1, "txs_prunable", MDB_INTEGERKEY, MDB_APPEND, BlockchainLMDB::compare_uint64);
|
||||
copy_table(env0, env1, "txs_prunable_tip", MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, MDB_NODUPDATA, BlockchainLMDB::compare_uint64);
|
||||
copy_table(env0, env1, "txs_prunable", MDB_INTEGERKEY, 0, BlockchainLMDB::compare_uint64);
|
||||
copy_table(env0, env1, "txs_prunable_tip", MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, 0, BlockchainLMDB::compare_uint64);
|
||||
}
|
||||
else
|
||||
{
|
||||
|
|
|
|||
Loading…
Reference in a new issue