d0s3t3ch/wownero
1
0
Fork 0
mirror of https://git.wownero.com/wownero/wownero.git synced 2024-08-15 01:03:23 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #4843

Browse source 
b750fb27 Pruning (moneromooo-monero)
This commit is contained in:
Riccardo Spagni 2019-01-28 13:55:49 +02:00
commit 4a0e4c7d70
No known key found for this signature in database
GPG key ID: 55432DF31CCD4FCD
52 changed files with 3308 additions and 445 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
contrib/epee/include/misc_log_ex.h
View file

@ -85,6 +85,16 @@
#define MGINFO_MAGENTA(x) MCLOG_MAGENTA(el::Level::Info, "global",x)
#define MGINFO_CYAN(x) MCLOG_CYAN(el::Level::Info, "global",x)
#define IFLOG(level, cat, type, init, x) \
do { \
if (ELPP->vRegistry()->allowed(level, cat)) { \
init; \
el::base::Writer(level, __FILE__, __LINE__, ELPP_FUNC, type).construct(cat) << x; \
} \
} while(0)
#define MIDEBUG(init, x) IFLOG(el::Level::Debug, MONERO_DEFAULT_LOG_CATEGORY, el::base::DispatchAction::NormalLog, init, x)
#define LOG_ERROR(x) MERROR(x)
#define LOG_PRINT_L0(x) MWARNING(x)
#define LOG_PRINT_L1(x) MINFO(x)

2
contrib/epee/include/net/abstract_tcp_server2.inl
View file

@ -295,6 +295,7 @@ PRAGMA_WARNING_DISABLE_VS(4355)
CRITICAL_REGION_LOCAL(m_throttle_speed_in_mutex);
m_throttle_speed_in.handle_trafic_exact(bytes_transferred);
context.m_current_speed_down = m_throttle_speed_in.get_current_speed();
context.m_max_speed_down = std::max(context.m_max_speed_down, context.m_current_speed_down);
}
{
@ -497,6 +498,7 @@ PRAGMA_WARNING_DISABLE_VS(4355)
CRITICAL_REGION_LOCAL(m_throttle_speed_out_mutex);
m_throttle_speed_out.handle_trafic_exact(cb);
context.m_current_speed_up = m_throttle_speed_out.get_current_speed();
context.m_max_speed_up = std::max(context.m_max_speed_up, context.m_current_speed_up);
}
//_info("[sock " << socket_.native_handle() << "] SEND " << cb);

10
contrib/epee/include/net/net_utils_base.h
View file

@ -228,6 +228,8 @@ namespace net_utils
uint64_t m_send_cnt;
double m_current_speed_down;
double m_current_speed_up;
double m_max_speed_down;
double m_max_speed_up;
connection_context_base(boost::uuids::uuid connection_id,
const network_address &remote_address, bool is_income,
@ -242,7 +244,9 @@ namespace net_utils
m_recv_cnt(recv_cnt),
m_send_cnt(send_cnt),
m_current_speed_down(0),
m_current_speed_up(0)
m_current_speed_up(0),
m_max_speed_down(0),
m_max_speed_up(0)
{}
connection_context_base(): m_connection_id(),
@ -254,7 +258,9 @@ namespace net_utils
m_recv_cnt(0),
m_send_cnt(0),
m_current_speed_down(0),
m_current_speed_up(0)
m_current_speed_up(0),
m_max_speed_down(0),
m_max_speed_up(0)
{}
connection_context_base& operator=(const connection_context_base& a)

9
contrib/epee/include/string_tools.h
View file

@ -206,6 +206,15 @@ POP_WARNINGS
return boost::lexical_cast<std::string>(val);
}
//----------------------------------------------------------------------------
inline std::string to_string_hex(uint32_t val)
{
std::stringstream ss;
ss << std::hex << val;
std::string s;
ss >> s;
return s;
}
//----------------------------------------------------------------------------
inline bool compare_no_case(const std::string& str1, const std::string& str2)
{

26
src/blockchain_db/blockchain_db.cpp
View file

@ -267,7 +267,10 @@ void BlockchainDB::pop_block(block& blk, std::vector<transaction>& txs)
for (const auto& h : boost::adaptors::reverse(blk.tx_hashes))
{
txs.push_back(get_tx(h));
cryptonote::transaction tx;
if (!get_tx(h, tx) && !get_pruned_tx(h, tx))
throw DB_ERROR("Failed to get pruned or unpruned transaction from the db");
txs.push_back(std::move(tx));
remove_transaction(h);
}
remove_transaction(get_transaction_hash(blk.miner_tx));
@ -280,7 +283,7 @@ bool BlockchainDB::is_open() const
void BlockchainDB::remove_transaction(const crypto::hash& tx_hash)
{
transaction tx = get_tx(tx_hash);
transaction tx = get_pruned_tx(tx_hash);
for (const txin_v& tx_input : tx.vin)
{
@ -325,6 +328,17 @@ bool BlockchainDB::get_tx(const crypto::hash& h, cryptonote::transaction &tx) co
return true;
}
bool BlockchainDB::get_pruned_tx(const crypto::hash& h, cryptonote::transaction &tx) const
{
blobdata bd;
if (!get_pruned_tx_blob(h, bd))
return false;
if (!parse_and_validate_tx_base_from_blob(bd, tx))
throw DB_ERROR("Failed to parse transaction base from blob retrieved from the db");
return true;
}
transaction BlockchainDB::get_tx(const crypto::hash& h) const
{
transaction tx;
@ -333,6 +347,14 @@ transaction BlockchainDB::get_tx(const crypto::hash& h) const
return tx;
}
transaction BlockchainDB::get_pruned_tx(const crypto::hash& h) const
{
transaction tx;
if (!get_pruned_tx(h, tx))
throw TX_DNE(std::string("pruned tx with hash ").append(epee::string_tools::pod_to_hex(h)).append(" not found in db").c_str());
return tx;
}
void BlockchainDB::reset_stats()
{
num_calls = 0;

62
src/blockchain_db/blockchain_db.h
View file

@ -1125,6 +1125,17 @@ public:
*/
virtual transaction get_tx(const crypto::hash& h) const;
/**
* @brief fetches the transaction base with the given hash
*
* If the transaction does not exist, the subclass should throw TX_DNE.
*
* @param h the hash to look for
*
* @return the transaction with the given hash
*/
virtual transaction get_pruned_tx(const crypto::hash& h) const;
/**
* @brief fetches the transaction with the given hash
*
@ -1136,6 +1147,17 @@ public:
*/
virtual bool get_tx(const crypto::hash& h, transaction &tx) const;
/**
* @brief fetches the transaction base with the given hash
*
* If the transaction does not exist, the subclass should return false.
*
* @param h the hash to look for
*
* @return true iff the transaction was found
*/
virtual bool get_pruned_tx(const crypto::hash& h, transaction &tx) const;
/**
* @brief fetches the transaction blob with the given hash
*
@ -1164,6 +1186,21 @@ public:
*/
virtual bool get_pruned_tx_blob(const crypto::hash& h, cryptonote::blobdata &tx) const = 0;
/**
* @brief fetches the prunable transaction blob with the given hash
*
* The subclass should return the prunable transaction stored which has the given
* hash.
*
* If the transaction does not exist, or if we do not have that prunable data,
* the subclass should return false.
*
* @param h the hash to look for
*
* @return true iff the transaction was found and we have its prunable data
*/
virtual bool get_prunable_tx_blob(const crypto::hash& h, cryptonote::blobdata &tx) const = 0;
/**
* @brief fetches the prunable transaction hash
*
@ -1412,6 +1449,31 @@ public:
*/
virtual void prune_outputs(uint64_t amount) = 0;
/**
* @brief get the blockchain pruning seed
* @return the blockchain pruning seed
*/
virtual uint32_t get_blockchain_pruning_seed() const = 0;
/**
* @brief prunes the blockchain
* @param pruning_seed the seed to use, 0 for default (highly recommended)
* @return success iff true
*/
virtual bool prune_blockchain(uint32_t pruning_seed = 0) = 0;
/**
* @brief prunes recent blockchain changes as needed, iff pruning is enabled
* @return success iff true
*/
virtual bool update_pruning() = 0;
/**
* @brief checks pruning was done correctly, iff enabled
* @return success iff true
*/
virtual bool check_pruning() = 0;
/**
* @brief runs a function over all txpool transactions
*

402
src/blockchain_db/lmdb/db_lmdb.cpp
View file

@ -35,6 +35,7 @@
#include "string_tools.h"
#include "file_io_utils.h"
#include "common/util.h"
#include "common/pruning.h"
#include "cryptonote_basic/cryptonote_format_utils.h"
#include "crypto/crypto.h"
#include "profile_tools.h"
@ -130,14 +131,20 @@ private:
std::unique_ptr<char[]> data;
};
int compare_uint64(const MDB_val *a, const MDB_val *b)
}
namespace cryptonote
{
const uint64_t va = *(const uint64_t *)a->mv_data;
const uint64_t vb = *(const uint64_t *)b->mv_data;
int BlockchainLMDB::compare_uint64(const MDB_val *a, const MDB_val *b)
{
uint64_t va, vb;
memcpy(&va, a->mv_data, sizeof(va));
memcpy(&vb, b->mv_data, sizeof(vb));
return (va < vb) ? -1 : va > vb;
}
int compare_hash32(const MDB_val *a, const MDB_val *b)
int BlockchainLMDB::compare_hash32(const MDB_val *a, const MDB_val *b)
{
uint32_t *va = (uint32_t*) a->mv_data;
uint32_t *vb = (uint32_t*) b->mv_data;
@ -151,13 +158,18 @@ int compare_hash32(const MDB_val *a, const MDB_val *b)
return 0;
}
int compare_string(const MDB_val *a, const MDB_val *b)
int BlockchainLMDB::compare_string(const MDB_val *a, const MDB_val *b)
{
const char *va = (const char*) a->mv_data;
const char *vb = (const char*) b->mv_data;
return strcmp(va, vb);
}
}
namespace
{
/* DB schema:
*
* Table Key Data
@ -169,6 +181,7 @@ int compare_string(const MDB_val *a, const MDB_val *b)
* txs_pruned txn ID pruned txn blob
* txs_prunable txn ID prunable txn blob
* txs_prunable_hash txn ID prunable txn hash
* txs_prunable_tip txn ID height
* tx_indices txn hash {txn ID, metadata}
* tx_outputs txn ID [txn amount output indices]
*
@ -196,6 +209,7 @@ const char* const LMDB_TXS = "txs";
const char* const LMDB_TXS_PRUNED = "txs_pruned";
const char* const LMDB_TXS_PRUNABLE = "txs_prunable";
const char* const LMDB_TXS_PRUNABLE_HASH = "txs_prunable_hash";
const char* const LMDB_TXS_PRUNABLE_TIP = "txs_prunable_tip";
const char* const LMDB_TX_INDICES = "tx_indices";
const char* const LMDB_TX_OUTPUTS = "tx_outputs";
@ -279,11 +293,6 @@ typedef struct blk_height {
uint64_t bh_height;
} blk_height;
typedef struct txindex {
crypto::hash key;
tx_data_t data;
} txindex;
typedef struct pre_rct_outkey {
uint64_t amount_index;
uint64_t output_id;
@ -549,18 +558,18 @@ bool BlockchainLMDB::need_resize(uint64_t threshold_size) const
// additional size needed.
uint64_t size_used = mst.ms_psize * mei.me_last_pgno;
LOG_PRINT_L1("DB map size: " << mei.me_mapsize);
LOG_PRINT_L1("Space used: " << size_used);
LOG_PRINT_L1("Space remaining: " << mei.me_mapsize - size_used);
LOG_PRINT_L1("Size threshold: " << threshold_size);
MDEBUG("DB map size: " << mei.me_mapsize);
MDEBUG("Space used: " << size_used);
MDEBUG("Space remaining: " << mei.me_mapsize - size_used);
MDEBUG("Size threshold: " << threshold_size);
float resize_percent = RESIZE_PERCENT;
LOG_PRINT_L1(boost::format("Percent used: %.04f Percent threshold: %.04f") % ((double)size_used/mei.me_mapsize) % resize_percent);
MDEBUG(boost::format("Percent used: %.04f Percent threshold: %.04f") % ((double)size_used/mei.me_mapsize) % resize_percent);
if (threshold_size > 0)
{
if (mei.me_mapsize - size_used < threshold_size)
{
LOG_PRINT_L1("Threshold met (size-based)");
MINFO("Threshold met (size-based)");
return true;
}
else
@ -569,7 +578,7 @@ bool BlockchainLMDB::need_resize(uint64_t threshold_size) const
if ((double)size_used / mei.me_mapsize > resize_percent)
{
LOG_PRINT_L1("Threshold met (percent-based)");
MINFO("Threshold met (percent-based)");
return true;
}
return false;
@ -581,7 +590,7 @@ bool BlockchainLMDB::need_resize(uint64_t threshold_size) const
void BlockchainLMDB::check_and_resize_for_batch(uint64_t batch_num_blocks, uint64_t batch_bytes)
{
LOG_PRINT_L3("BlockchainLMDB::" << __func__);
LOG_PRINT_L1("[" << __func__ << "] " << "checking DB size");
MTRACE("[" << __func__ << "] " << "checking DB size");
const uint64_t min_increase_size = 512 * (1 << 20);
uint64_t threshold_size = 0;
uint64_t increase_size = 0;
@ -811,6 +820,7 @@ uint64_t BlockchainLMDB::add_transaction_data(const crypto::hash& blk_hash, cons
CURSOR(txs_pruned)
CURSOR(txs_prunable)
CURSOR(txs_prunable_hash)
CURSOR(txs_prunable_tip)
CURSOR(tx_indices)
MDB_val_set(val_tx_id, tx_id);
@ -858,6 +868,14 @@ uint64_t BlockchainLMDB::add_transaction_data(const crypto::hash& blk_hash, cons
if (result)
throw0(DB_ERROR(lmdb_error("Failed to add prunable tx blob to db transaction: ", result).c_str()));
if (get_blockchain_pruning_seed())
{
MDB_val_set(val_height, m_height);
result = mdb_cursor_put(m_cur_txs_prunable_tip, &val_tx_id, &val_height, 0);
if (result)
throw0(DB_ERROR(lmdb_error("Failed to add prunable tx id to db transaction: ", result).c_str()));
}
if (tx.version > 1)
{
MDB_val_set(val_prunable_hash, tx_prunable_hash);
@ -883,6 +901,7 @@ void BlockchainLMDB::remove_transaction_data(const crypto::hash& tx_hash, const
CURSOR(txs_pruned)
CURSOR(txs_prunable)
CURSOR(txs_prunable_hash)
CURSOR(txs_prunable_tip)
CURSOR(tx_outputs)
MDB_val_set(val_h, tx_hash);
@ -898,11 +917,25 @@ void BlockchainLMDB::remove_transaction_data(const crypto::hash& tx_hash, const
if (result)
throw1(DB_ERROR(lmdb_error("Failed to add removal of pruned tx to db transaction: ", result).c_str()));
if ((result = mdb_cursor_get(m_cur_txs_prunable, &val_tx_id, NULL, MDB_SET)))
result = mdb_cursor_get(m_cur_txs_prunable, &val_tx_id, NULL, MDB_SET);
if (result == 0)
{
result = mdb_cursor_del(m_cur_txs_prunable, 0);
if (result)
throw1(DB_ERROR(lmdb_error("Failed to add removal of prunable tx to db transaction: ", result).c_str()));
}
else if (result != MDB_NOTFOUND)
throw1(DB_ERROR(lmdb_error("Failed to locate prunable tx for removal: ", result).c_str()));
result = mdb_cursor_del(m_cur_txs_prunable, 0);
if (result)
throw1(DB_ERROR(lmdb_error("Failed to add removal of prunable tx to db transaction: ", result).c_str()));
result = mdb_cursor_get(m_cur_txs_prunable_tip, &val_tx_id, NULL, MDB_SET);
if (result && result != MDB_NOTFOUND)
throw1(DB_ERROR(lmdb_error("Failed to locate tx id for removal: ", result).c_str()));
if (result == 0)
{
result = mdb_cursor_del(m_cur_txs_prunable_tip, 0);
if (result)
throw1(DB_ERROR(lmdb_error("Error adding removal of tx id to db transaction", result).c_str()));
}
if (tx.version > 1)
{
@ -1308,6 +1341,7 @@ void BlockchainLMDB::open(const std::string& filename, const int db_flags)
// open necessary databases, and set properties as needed
// uses macros to avoid having to change things too many places
// also change blockchain_prune.cpp to match
lmdb_db_open(txn, LMDB_BLOCKS, MDB_INTEGERKEY | MDB_CREATE, m_blocks, "Failed to open db handle for m_blocks");
lmdb_db_open(txn, LMDB_BLOCK_INFO, MDB_INTEGERKEY | MDB_CREATE | MDB_DUPSORT | MDB_DUPFIXED, m_block_info, "Failed to open db handle for m_block_info");
@ -1316,7 +1350,9 @@ void BlockchainLMDB::open(const std::string& filename, const int db_flags)
lmdb_db_open(txn, LMDB_TXS, MDB_INTEGERKEY | MDB_CREATE, m_txs, "Failed to open db handle for m_txs");
lmdb_db_open(txn, LMDB_TXS_PRUNED, MDB_INTEGERKEY | MDB_CREATE, m_txs_pruned, "Failed to open db handle for m_txs_pruned");
lmdb_db_open(txn, LMDB_TXS_PRUNABLE, MDB_INTEGERKEY | MDB_CREATE, m_txs_prunable, "Failed to open db handle for m_txs_prunable");
lmdb_db_open(txn, LMDB_TXS_PRUNABLE_HASH, MDB_INTEGERKEY | MDB_CREATE, m_txs_prunable_hash, "Failed to open db handle for m_txs_prunable_hash");
lmdb_db_open(txn, LMDB_TXS_PRUNABLE_HASH, MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED | MDB_CREATE, m_txs_prunable_hash, "Failed to open db handle for m_txs_prunable_hash");
if (!(mdb_flags & MDB_RDONLY))
lmdb_db_open(txn, LMDB_TXS_PRUNABLE_TIP, MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED | MDB_CREATE, m_txs_prunable_tip, "Failed to open db handle for m_txs_prunable_tip");
lmdb_db_open(txn, LMDB_TX_INDICES, MDB_INTEGERKEY | MDB_CREATE | MDB_DUPSORT | MDB_DUPFIXED, m_tx_indices, "Failed to open db handle for m_tx_indices");
lmdb_db_open(txn, LMDB_TX_OUTPUTS, MDB_INTEGERKEY | MDB_CREATE, m_tx_outputs, "Failed to open db handle for m_tx_outputs");
@ -1344,6 +1380,10 @@ void BlockchainLMDB::open(const std::string& filename, const int db_flags)
mdb_set_dupsort(txn, m_output_amounts, compare_uint64);
mdb_set_dupsort(txn, m_output_txs, compare_uint64);
mdb_set_dupsort(txn, m_block_info, compare_uint64);
if (!(mdb_flags & MDB_RDONLY))
mdb_set_dupsort(txn, m_txs_prunable_tip, compare_uint64);
mdb_set_compare(txn, m_txs_prunable, compare_uint64);
mdb_set_dupsort(txn, m_txs_prunable_hash, compare_uint64);
mdb_set_compare(txn, m_txpool_meta, compare_hash32);
mdb_set_compare(txn, m_txpool_blob, compare_hash32);
@ -1502,6 +1542,8 @@ void BlockchainLMDB::reset()
throw0(DB_ERROR(lmdb_error("Failed to drop m_txs_prunable: ", result).c_str()));
if (auto result = mdb_drop(txn, m_txs_prunable_hash, 0))
throw0(DB_ERROR(lmdb_error("Failed to drop m_txs_prunable_hash: ", result).c_str()));
if (auto result = mdb_drop(txn, m_txs_prunable_tip, 0))
throw0(DB_ERROR(lmdb_error("Failed to drop m_txs_prunable_tip: ", result).c_str()));
if (auto result = mdb_drop(txn, m_tx_indices, 0))
throw0(DB_ERROR(lmdb_error("Failed to drop m_tx_indices: ", result).c_str()));
if (auto result = mdb_drop(txn, m_tx_outputs, 0))
@ -1827,6 +1869,290 @@ cryptonote::blobdata BlockchainLMDB::get_txpool_tx_blob(const crypto::hash& txid
return bd;
}
uint32_t BlockchainLMDB::get_blockchain_pruning_seed() const
{
LOG_PRINT_L3("BlockchainLMDB::" << __func__);
check_open();
TXN_PREFIX_RDONLY();
RCURSOR(properties)
MDB_val_str(k, "pruning_seed");
MDB_val v;
int result = mdb_cursor_get(m_cur_properties, &k, &v, MDB_SET);
if (result == MDB_NOTFOUND)
return 0;
if (result)
throw0(DB_ERROR(lmdb_error("Failed to retrieve pruning seed: ", result).c_str()));
if (v.mv_size != sizeof(uint32_t))
throw0(DB_ERROR("Failed to retrieve or create pruning seed: unexpected value size"));
uint32_t pruning_seed;
memcpy(&pruning_seed, v.mv_data, sizeof(pruning_seed));
TXN_POSTFIX_RDONLY();
return pruning_seed;
}
static bool is_v1_tx(MDB_cursor *c_txs_pruned, MDB_val *tx_id)
{
MDB_val v;
int ret = mdb_cursor_get(c_txs_pruned, tx_id, &v, MDB_SET);
if (ret)
throw0(DB_ERROR(lmdb_error("Failed to find transaction pruned data: ", ret).c_str()));
if (v.mv_size == 0)
throw0(DB_ERROR("Invalid transaction pruned data"));
return cryptonote::is_v1_tx(cryptonote::blobdata_ref{(const char*)v.mv_data, v.mv_size});
}
enum { prune_mode_prune, prune_mode_update, prune_mode_check };
bool BlockchainLMDB::prune_worker(int mode, uint32_t pruning_seed)
{
LOG_PRINT_L3("BlockchainLMDB::" << __func__);
const uint32_t log_stripes = tools::get_pruning_log_stripes(pruning_seed);
if (log_stripes && log_stripes != CRYPTONOTE_PRUNING_LOG_STRIPES)
throw0(DB_ERROR("Pruning seed not in range"));
pruning_seed = tools::get_pruning_stripe(pruning_seed);;
if (pruning_seed > (1ul << CRYPTONOTE_PRUNING_LOG_STRIPES))
throw0(DB_ERROR("Pruning seed not in range"));
check_open();
TIME_MEASURE_START(t);
size_t n_total_records = 0, n_prunable_records = 0, n_pruned_records = 0;
uint64_t n_bytes = 0;
mdb_txn_safe txn;
auto result = mdb_txn_begin(m_env, NULL, 0, txn);
if (result)
throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str()));
MDB_stat db_stats;
if ((result = mdb_stat(txn, m_txs_prunable, &db_stats)))
throw0(DB_ERROR(lmdb_error("Failed to query m_txs_prunable: ", result).c_str()));
const size_t pages0 = db_stats.ms_branch_pages + db_stats.ms_leaf_pages + db_stats.ms_overflow_pages;
MDB_val_str(k, "pruning_seed");
MDB_val v;
result = mdb_get(txn, m_properties, &k, &v);
bool prune_tip_table = false;
if (result == MDB_NOTFOUND)
{
// not pruned yet
if (mode != prune_mode_prune)
{
txn.abort();
TIME_MEASURE_FINISH(t);
MDEBUG("Pruning not enabled, nothing to do");
return true;
}
if (pruning_seed == 0)
pruning_seed = tools::get_random_stripe();
pruning_seed = tools::make_pruning_seed(pruning_seed, CRYPTONOTE_PRUNING_LOG_STRIPES);
v.mv_data = &pruning_seed;
v.mv_size = sizeof(pruning_seed);
result = mdb_put(txn, m_properties, &k, &v, 0);
if (result)
throw0(DB_ERROR("Failed to save pruning seed"));
prune_tip_table = false;
}
else if (result == 0)
{
// pruned already
if (v.mv_size != sizeof(uint32_t))
throw0(DB_ERROR("Failed to retrieve or create pruning seed: unexpected value size"));
const uint32_t data = *(const uint32_t*)v.mv_data;
if (pruning_seed == 0)
pruning_seed = tools::get_pruning_stripe(data);
if (tools::get_pruning_stripe(data) != pruning_seed)
throw0(DB_ERROR("Blockchain already pruned with different seed"));
if (tools::get_pruning_log_stripes(data) != CRYPTONOTE_PRUNING_LOG_STRIPES)
throw0(DB_ERROR("Blockchain already pruned with different base"));
pruning_seed = tools::make_pruning_seed(pruning_seed, CRYPTONOTE_PRUNING_LOG_STRIPES);
prune_tip_table = (mode == prune_mode_update);
}
else
{
throw0(DB_ERROR(lmdb_error("Failed to retrieve or create pruning seed: ", result).c_str()));
}
if (mode == prune_mode_check)
MINFO("Checking blockchain pruning...");
else
MINFO("Pruning blockchain...");
MDB_cursor *c_txs_pruned, *c_txs_prunable, *c_txs_prunable_tip;
result = mdb_cursor_open(txn, m_txs_pruned, &c_txs_pruned);
if (result)
throw0(DB_ERROR(lmdb_error("Failed to open a cursor for txs_pruned: ", result).c_str()));
result = mdb_cursor_open(txn, m_txs_prunable, &c_txs_prunable);
if (result)
throw0(DB_ERROR(lmdb_error("Failed to open a cursor for txs_prunable: ", result).c_str()));
result = mdb_cursor_open(txn, m_txs_prunable_tip, &c_txs_prunable_tip);
if (result)
throw0(DB_ERROR(lmdb_error("Failed to open a cursor for txs_prunable_tip: ", result).c_str()));
const uint64_t blockchain_height = height();
if (prune_tip_table)
{
MDB_cursor_op op = MDB_FIRST;
while (1)
{
int ret = mdb_cursor_get(c_txs_prunable_tip, &k, &v, op);
op = MDB_NEXT;
if (ret == MDB_NOTFOUND)
break;
if (ret)
throw0(DB_ERROR(lmdb_error("Failed to enumerate transactions: ", ret).c_str()));
uint64_t block_height;
memcpy(&block_height, v.mv_data, sizeof(block_height));
if (block_height + CRYPTONOTE_PRUNING_TIP_BLOCKS < blockchain_height)
{
++n_total_records;
if (!tools::has_unpruned_block(block_height, blockchain_height, pruning_seed) && !is_v1_tx(c_txs_pruned, &k))
{
++n_prunable_records;
result = mdb_cursor_get(c_txs_prunable, &k, &v, MDB_SET);
if (result == MDB_NOTFOUND)
MWARNING("Already pruned at height " << block_height << "/" << blockchain_height);
else if (result)
throw0(DB_ERROR(lmdb_error("Failed to find transaction prunable data: ", result).c_str()));
else
{
MDEBUG("Pruning at height " << block_height << "/" << blockchain_height);
++n_pruned_records;
n_bytes += k.mv_size + v.mv_size;
result = mdb_cursor_del(c_txs_prunable, 0);
if (result)
throw0(DB_ERROR(lmdb_error("Failed to delete transaction prunable data: ", result).c_str()));
}
}
result = mdb_cursor_del(c_txs_prunable_tip, 0);
if (result)
throw0(DB_ERROR(lmdb_error("Failed to delete transaction tip data: ", result).c_str()));
}
}
}
else
{
MDB_cursor *c_tx_indices;
result = mdb_cursor_open(txn, m_tx_indices, &c_tx_indices);
if (result)
throw0(DB_ERROR(lmdb_error("Failed to open a cursor for tx_indices: ", result).c_str()));
MDB_cursor_op op = MDB_FIRST;
while (1)
{
int ret = mdb_cursor_get(c_tx_indices, &k, &v, op);
op = MDB_NEXT;
if (ret == MDB_NOTFOUND)
break;
if (ret)
throw0(DB_ERROR(lmdb_error("Failed to enumerate transactions: ", ret).c_str()));
++n_total_records;
//const txindex *ti = (const txindex *)v.mv_data;
txindex ti;
memcpy(&ti, v.mv_data, sizeof(ti));
const uint64_t block_height = ti.data.block_id;
if (block_height + CRYPTONOTE_PRUNING_TIP_BLOCKS >= blockchain_height)
{
MDB_val_set(kp, ti.data.tx_id);
MDB_val_set(vp, block_height);
if (mode == prune_mode_check)
{
result = mdb_cursor_get(c_txs_prunable_tip, &kp, &vp, MDB_SET);
if (result && result != MDB_NOTFOUND)
throw0(DB_ERROR(lmdb_error("Error looking for transaction prunable data: ", result).c_str()));
if (result == MDB_NOTFOUND)
MERROR("Transaction not found in prunable tip table for height " << block_height << "/" << blockchain_height <<
", seed " << epee::string_tools::to_string_hex(pruning_seed));
}
else
{
result = mdb_cursor_put(c_txs_prunable_tip, &kp, &vp, 0);
if (result && result != MDB_NOTFOUND)
throw0(DB_ERROR(lmdb_error("Error looking for transaction prunable data: ", result).c_str()));
}
}
MDB_val_set(kp, ti.data.tx_id);
if (!tools::has_unpruned_block(block_height, blockchain_height, pruning_seed) && !is_v1_tx(c_txs_pruned, &kp))
{
result = mdb_cursor_get(c_txs_prunable, &kp, &v, MDB_SET);
if (result && result != MDB_NOTFOUND)
throw0(DB_ERROR(lmdb_error("Error looking for transaction prunable data: ", result).c_str()));
if (mode == prune_mode_check)
{
if (result != MDB_NOTFOUND)
MERROR("Prunable data found for pruned height " << block_height << "/" << blockchain_height <<
", seed " << epee::string_tools::to_string_hex(pruning_seed));
}
else
{
++n_prunable_records;
if (result == MDB_NOTFOUND)
MWARNING("Already pruned at height " << block_height << "/" << blockchain_height);
else
{
MDEBUG("Pruning at height " << block_height << "/" << blockchain_height);
++n_pruned_records;
n_bytes += kp.mv_size + v.mv_size;
result = mdb_cursor_del(c_txs_prunable, 0);
if (result)
throw0(DB_ERROR(lmdb_error("Failed to delete transaction prunable data: ", result).c_str()));
}
}
}
else
{
if (mode == prune_mode_check)
{
MDB_val_set(kp, ti.data.tx_id);
result = mdb_cursor_get(c_txs_prunable, &kp, &v, MDB_SET);
if (result && result != MDB_NOTFOUND)
throw0(DB_ERROR(lmdb_error("Error looking for transaction prunable data: ", result).c_str()));
if (result == MDB_NOTFOUND)
MERROR("Prunable data not found for unpruned height " << block_height << "/" << blockchain_height <<
", seed " << epee::string_tools::to_string_hex(pruning_seed));
}
}
}
mdb_cursor_close(c_tx_indices);
}
if ((result = mdb_stat(txn, m_txs_prunable, &db_stats)))
throw0(DB_ERROR(lmdb_error("Failed to query m_txs_prunable: ", result).c_str()));
const size_t pages1 = db_stats.ms_branch_pages + db_stats.ms_leaf_pages + db_stats.ms_overflow_pages;
const size_t db_bytes = (pages0 - pages1) * db_stats.ms_psize;
mdb_cursor_close(c_txs_prunable_tip);
mdb_cursor_close(c_txs_prunable);
mdb_cursor_close(c_txs_pruned);
txn.commit();
TIME_MEASURE_FINISH(t);
MINFO((mode == prune_mode_check ? "Checked" : "Pruned") << " blockchain in " <<
t << " ms: " << (n_bytes/1024.0f/1024.0f) << " MB (" << db_bytes/1024.0f/1024.0f << " MB) pruned in " <<
n_pruned_records << " records (" << pages0 - pages1 << "/" << pages0 << " " << db_stats.ms_psize << " byte pages), " <<
n_prunable_records << "/" << n_total_records << " pruned records");
return true;
}
bool BlockchainLMDB::prune_blockchain(uint32_t pruning_seed)
{
return prune_worker(prune_mode_prune, pruning_seed);
}
bool BlockchainLMDB::update_pruning()
{
return prune_worker(prune_mode_update, 0);
}
bool BlockchainLMDB::check_pruning()
{
return prune_worker(prune_mode_check, 0);
}
bool BlockchainLMDB::for_all_txpool_txes(std::function<bool(const crypto::hash&, const txpool_tx_meta_t&, const cryptonote::blobdata*)> f, bool include_blob, bool include_unrelayed_txes) const
{
LOG_PRINT_L3("BlockchainLMDB::" << __func__);
@ -2428,6 +2754,36 @@ bool BlockchainLMDB::get_pruned_tx_blob(const crypto::hash& h, cryptonote::blobd
return true;
}
bool BlockchainLMDB::get_prunable_tx_blob(const crypto::hash& h, cryptonote::blobdata &bd) const
{
LOG_PRINT_L3("BlockchainLMDB::" << __func__);
check_open();
TXN_PREFIX_RDONLY();
RCURSOR(tx_indices);
RCURSOR(txs_prunable);
MDB_val_set(v, h);
MDB_val result;
auto get_result = mdb_cursor_get(m_cur_tx_indices, (MDB_val *)&zerokval, &v, MDB_GET_BOTH);
if (get_result == 0)
{
const txindex *tip = (const txindex *)v.mv_data;
MDB_val_set(val_tx_id, tip->data.tx_id);
get_result = mdb_cursor_get(m_cur_txs_prunable, &val_tx_id, &result, MDB_SET);
}
if (get_result == MDB_NOTFOUND)
return false;
else if (get_result)
throw0(DB_ERROR(lmdb_error("DB error attempting to fetch tx from hash", get_result).c_str()));
bd.assign(reinterpret_cast<char*>(result.mv_data), result.mv_size);
TXN_POSTFIX_RDONLY();
return true;
}
bool BlockchainLMDB::get_prunable_tx_hash(const crypto::hash& tx_hash, crypto::hash &prunable_hash) const
{
LOG_PRINT_L3("BlockchainLMDB::" << __func__);

26
src/blockchain_db/lmdb/db_lmdb.h
View file

@ -40,6 +40,11 @@
namespace cryptonote
{
typedef struct txindex {
crypto::hash key;
tx_data_t data;
} txindex;
typedef struct mdb_txn_cursors
{
MDB_cursor *m_txc_blocks;
@ -53,6 +58,7 @@ typedef struct mdb_txn_cursors
MDB_cursor *m_txc_txs_pruned;
MDB_cursor *m_txc_txs_prunable;
MDB_cursor *m_txc_txs_prunable_hash;
MDB_cursor *m_txc_txs_prunable_tip;
MDB_cursor *m_txc_tx_indices;
MDB_cursor *m_txc_tx_outputs;
@ -62,6 +68,8 @@ typedef struct mdb_txn_cursors
MDB_cursor *m_txc_txpool_blob;
MDB_cursor *m_txc_hf_versions;
MDB_cursor *m_txc_properties;
} mdb_txn_cursors;
#define m_cur_blocks m_cursors->m_txc_blocks
@ -73,12 +81,14 @@ typedef struct mdb_txn_cursors
#define m_cur_txs_pruned m_cursors->m_txc_txs_pruned
#define m_cur_txs_prunable m_cursors->m_txc_txs_prunable
#define m_cur_txs_prunable_hash m_cursors->m_txc_txs_prunable_hash
#define m_cur_txs_prunable_tip m_cursors->m_txc_txs_prunable_tip
#define m_cur_tx_indices m_cursors->m_txc_tx_indices
#define m_cur_tx_outputs m_cursors->m_txc_tx_outputs
#define m_cur_spent_keys m_cursors->m_txc_spent_keys
#define m_cur_txpool_meta m_cursors->m_txc_txpool_meta
#define m_cur_txpool_blob m_cursors->m_txc_txpool_blob
#define m_cur_hf_versions m_cursors->m_txc_hf_versions
#define m_cur_properties m_cursors->m_txc_properties
typedef struct mdb_rflags
{
@ -92,12 +102,14 @@ typedef struct mdb_rflags
bool m_rf_txs_pruned;
bool m_rf_txs_prunable;
bool m_rf_txs_prunable_hash;
bool m_rf_txs_prunable_tip;
bool m_rf_tx_indices;
bool m_rf_tx_outputs;
bool m_rf_spent_keys;
bool m_rf_txpool_meta;
bool m_rf_txpool_blob;
bool m_rf_hf_versions;
bool m_rf_properties;
} mdb_rflags;
typedef struct mdb_threadinfo
@ -232,6 +244,7 @@ public:
virtual bool get_tx_blob(const crypto::hash& h, cryptonote::blobdata &tx) const;
virtual bool get_pruned_tx_blob(const crypto::hash& h, cryptonote::blobdata &tx) const;
virtual bool get_prunable_tx_blob(const crypto::hash& h, cryptonote::blobdata &tx) const;
virtual bool get_prunable_tx_hash(const crypto::hash& tx_hash, crypto::hash &prunable_hash) const;
virtual uint64_t get_tx_count() const;
@ -264,6 +277,11 @@ public:
virtual bool get_txpool_tx_meta(const crypto::hash& txid, txpool_tx_meta_t &meta) const;
virtual bool get_txpool_tx_blob(const crypto::hash& txid, cryptonote::blobdata &bd) const;
virtual cryptonote::blobdata get_txpool_tx_blob(const crypto::hash& txid) const;
virtual uint32_t get_blockchain_pruning_seed() const;
virtual bool prune_blockchain(uint32_t pruning_seed = 0);
virtual bool update_pruning();
virtual bool check_pruning();
virtual bool for_all_txpool_txes(std::function<bool(const crypto::hash&, const txpool_tx_meta_t&, const cryptonote::blobdata*)> f, bool include_blob = false, bool include_unrelayed_txes = true) const;
virtual bool for_all_key_images(std::function<bool(const crypto::key_image&)>) const;
@ -309,6 +327,11 @@ public:
bool get_output_distribution(uint64_t amount, uint64_t from_height, uint64_t to_height, std::vector<uint64_t> &distribution, uint64_t &base) const;
// helper functions
static int compare_uint64(const MDB_val *a, const MDB_val *b);
static int compare_hash32(const MDB_val *a, const MDB_val *b);
static int compare_string(const MDB_val *a, const MDB_val *b);
private:
void do_resize(uint64_t size_increase=0);
@ -361,6 +384,8 @@ private:
inline void check_open() const;
bool prune_worker(int mode, uint32_t pruning_seed);
virtual bool is_read_only() const;
virtual uint64_t get_database_size() const;
@ -393,6 +418,7 @@ private:
MDB_dbi m_txs_pruned;
MDB_dbi m_txs_prunable;
MDB_dbi m_txs_prunable_hash;
MDB_dbi m_txs_prunable_tip;
MDB_dbi m_tx_indices;
MDB_dbi m_tx_outputs;

33
src/blockchain_utilities/CMakeLists.txt
View file

@ -92,6 +92,17 @@ monero_private_headers(blockchain_prune_known_spent_data
set(blockchain_prune_sources
blockchain_prune.cpp
)
set(blockchain_prune_private_headers)
monero_private_headers(blockchain_prune
${blockchain_prune_private_headers})
set(blockchain_ancestry_sources
blockchain_ancestry.cpp
)
@ -298,3 +309,25 @@ set_property(TARGET blockchain_prune_known_spent_data
PROPERTY
OUTPUT_NAME "monero-blockchain-prune-known-spent-data")
install(TARGETS blockchain_prune_known_spent_data DESTINATION bin)
monero_add_executable(blockchain_prune
${blockchain_prune_sources}
${blockchain_prune_private_headers})
set_property(TARGET blockchain_prune
PROPERTY
OUTPUT_NAME "monero-blockchain-prune")
install(TARGETS blockchain_prune DESTINATION bin)
target_link_libraries(blockchain_prune
PRIVATE
cryptonote_core
blockchain_db
p2p
version
epee
${Boost_FILESYSTEM_LIBRARY}
${Boost_SYSTEM_LIBRARY}
${Boost_THREAD_LIBRARY}
${CMAKE_THREAD_LIBS_INIT}
${EXTRA_LIBRARIES})

6
src/blockchain_utilities/blockchain_export.cpp
View file

@ -177,6 +177,12 @@ int main(int argc, char* argv[])
}
r = core_storage->init(db, opt_testnet ? cryptonote::TESTNET : opt_stagenet ? cryptonote::STAGENET : cryptonote::MAINNET);
if (core_storage->get_blockchain_pruning_seed())
{
LOG_PRINT_L0("Blockchain is pruned, cannot export");
return 1;
}
CHECK_AND_ASSERT_MES(r, 1, "Failed to initialize source blockchain storage");
LOG_PRINT_L0("Source blockchain storage initialized OK");
LOG_PRINT_L0("Exporting blockchain raw data...");

663
src/blockchain_utilities/blockchain_prune.cpp Normal file
View file

@ -0,0 +1,663 @@
// Copyright (c) 2018, The Monero Project
//
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without modification, are
// permitted provided that the following conditions are met:
//
// 1. Redistributions of source code must retain the above copyright notice, this list of
// conditions and the following disclaimer.
//
// 2. Redistributions in binary form must reproduce the above copyright notice, this list
// of conditions and the following disclaimer in the documentation and/or other
// materials provided with the distribution.
//
// 3. Neither the name of the copyright holder nor the names of its contributors may be
// used to endorse or promote products derived from this software without specific
// prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include <array>
#include <lmdb.h>
#include <boost/algorithm/string.hpp>
#include "common/command_line.h"
#include "common/pruning.h"
#include "cryptonote_core/cryptonote_core.h"
#include "cryptonote_core/blockchain.h"
#include "blockchain_db/blockchain_db.h"
#include "blockchain_db/lmdb/db_lmdb.h"
#include "blockchain_db/db_types.h"
#include "version.h"
#undef MONERO_DEFAULT_LOG_CATEGORY
#define MONERO_DEFAULT_LOG_CATEGORY "bcutil"
#define MDB_val_set(var, val) MDB_val var = {sizeof(val), (void *)&val}
namespace po = boost::program_options;
using namespace epee;
using namespace cryptonote;
static std::string db_path;
// default to fast:1
static uint64_t records_per_sync = 128;
static const size_t slack = 512 * 1024 * 1024;
static std::error_code replace_file(const boost::filesystem::path& replacement_name, const boost::filesystem::path& replaced_name)
{
std::error_code ec = tools::replace_file(replacement_name.string(), replaced_name.string());
if (ec)
MERROR("Error renaming " << replacement_name << " to " << replaced_name << ": " << ec.message());
return ec;
}
static void open(MDB_env *&env, const boost::filesystem::path &path, uint64_t db_flags, bool readonly)
{
int dbr;
int flags = 0;
if (db_flags & DBF_FAST)
flags |= MDB_NOSYNC;
if (db_flags & DBF_FASTEST)
flags |= MDB_NOSYNC | MDB_WRITEMAP | MDB_MAPASYNC;
if (readonly)
flags |= MDB_RDONLY;
dbr = mdb_env_create(&env);
if (dbr) throw std::runtime_error("Failed to create LDMB environment: " + std::string(mdb_strerror(dbr)));
dbr = mdb_env_set_maxdbs(env, 32);
if (dbr) throw std::runtime_error("Failed to set max env dbs: " + std::string(mdb_strerror(dbr)));
dbr = mdb_env_open(env, path.string().c_str(), flags, 0664);
if (dbr) throw std::runtime_error("Failed to open database file '"
+ path.string() + "': " + std::string(mdb_strerror(dbr)));
}
static void close(MDB_env *env)
{
mdb_env_close(env);
}
static void add_size(MDB_env *env, uint64_t bytes)
{
try
{
boost::filesystem::path path(db_path);
boost::filesystem::space_info si = boost::filesystem::space(path);
if(si.available < bytes)
{
MERROR("!! WARNING: Insufficient free space to extend database !!: " <<
(si.available >> 20L) << " MB available, " << (bytes >> 20L) << " MB needed");
return;
}
}
catch(...)
{
// print something but proceed.
MWARNING("Unable to query free disk space.");
}
MDB_envinfo mei;
mdb_env_info(env, &mei);
MDB_stat mst;
mdb_env_stat(env, &mst);
uint64_t new_mapsize = (uint64_t)mei.me_mapsize + bytes;
new_mapsize += (new_mapsize % mst.ms_psize);
int result = mdb_env_set_mapsize(env, new_mapsize);
if (result)
throw std::runtime_error("Failed to set new mapsize to " + std::to_string(new_mapsize) + ": " + std::string(mdb_strerror(result)));
MGINFO("LMDB Mapsize increased." << " Old: " << mei.me_mapsize / (1024 * 1024) << "MiB" << ", New: " << new_mapsize / (1024 * 1024) << "MiB");
}
static void check_resize(MDB_env *env, size_t bytes)
{
MDB_envinfo mei;
MDB_stat mst;
mdb_env_info(env, &mei);
mdb_env_stat(env, &mst);
uint64_t size_used = mst.ms_psize * mei.me_last_pgno;
if (size_used + bytes + slack >= mei.me_mapsize)
add_size(env, size_used + bytes + 2 * slack - mei.me_mapsize);
}
static bool resize_point(size_t nrecords, MDB_env *env, MDB_txn **txn, size_t &bytes)
{
if (nrecords % records_per_sync && bytes <= slack / 2)
return false;
int dbr = mdb_txn_commit(*txn);
if (dbr) throw std::runtime_error("Failed to commit txn: " + std::string(mdb_strerror(dbr)));
check_resize(env, bytes);
dbr = mdb_txn_begin(env, NULL, 0, txn);
if (dbr) throw std::runtime_error("Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr)));
bytes = 0;
return true;
}
static void copy_table(MDB_env *env0, MDB_env *env1, const char *table, unsigned int flags, unsigned int putflags, int (*cmp)(const MDB_val*, const MDB_val*)=0)
{
MDB_dbi dbi0, dbi1;
MDB_txn *txn0, *txn1;
MDB_cursor *cur0, *cur1;
bool tx_active0 = false, tx_active1 = false;
int dbr;
MINFO("Copying " << table);
epee::misc_utils::auto_scope_leave_caller txn_dtor = epee::misc_utils::create_scope_leave_handler([&](){
if (tx_active1) mdb_txn_abort(txn1);
if (tx_active0) mdb_txn_abort(txn0);
});
dbr = mdb_txn_begin(env0, NULL, MDB_RDONLY, &txn0);
if (dbr) throw std::runtime_error("Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr)));
tx_active0 = true;
dbr = mdb_txn_begin(env1, NULL, 0, &txn1);
if (dbr) throw std::runtime_error("Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr)));
tx_active1 = true;
dbr = mdb_dbi_open(txn0, table, flags, &dbi0);
if (dbr) throw std::runtime_error("Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr)));
if (cmp)
((flags & MDB_DUPSORT) ? mdb_set_dupsort : mdb_set_compare)(txn0, dbi0, cmp);
dbr = mdb_dbi_open(txn1, table, flags, &dbi1);
if (dbr) throw std::runtime_error("Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr)));
if (cmp)
((flags & MDB_DUPSORT) ? mdb_set_dupsort : mdb_set_compare)(txn1, dbi1, cmp);
dbr = mdb_txn_commit(txn1);
if (dbr) throw std::runtime_error("Failed to commit txn: " + std::string(mdb_strerror(dbr)));
tx_active1 = false;
MDB_stat stats;
dbr = mdb_env_stat(env0, &stats);
if (dbr) throw std::runtime_error("Failed to stat " + std::string(table) + " LMDB table: " + std::string(mdb_strerror(dbr)));
check_resize(env1, (stats.ms_branch_pages + stats.ms_overflow_pages + stats.ms_leaf_pages) * stats.ms_psize);
dbr = mdb_txn_begin(env1, NULL, 0, &txn1);
if (dbr) throw std::runtime_error("Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr)));
tx_active1 = true;
dbr = mdb_drop(txn1, dbi1, 0);
if (dbr) throw std::runtime_error("Failed to empty " + std::string(table) + " LMDB table: " + std::string(mdb_strerror(dbr)));
dbr = mdb_cursor_open(txn0, dbi0, &cur0);
if (dbr) throw std::runtime_error("Failed to create LMDB cursor: " + std::string(mdb_strerror(dbr)));
dbr = mdb_cursor_open(txn1, dbi1, &cur1);
if (dbr) throw std::runtime_error("Failed to create LMDB cursor: " + std::string(mdb_strerror(dbr)));
MDB_val k;
MDB_val v;
MDB_cursor_op op = MDB_FIRST;
size_t nrecords = 0, bytes = 0;
while (1)
{
int ret = mdb_cursor_get(cur0, &k, &v, op);
op = MDB_NEXT;
if (ret == MDB_NOTFOUND)
break;
if (ret)
throw std::runtime_error("Failed to enumerate " + std::string(table) + " records: " + std::string(mdb_strerror(ret)));
bytes += k.mv_size + v.mv_size;
if (resize_point(++nrecords, env1, &txn1, bytes))
{
dbr = mdb_cursor_open(txn1, dbi1, &cur1);
if (dbr) throw std::runtime_error("Failed to create LMDB cursor: " + std::string(mdb_strerror(dbr)));
}
ret = mdb_cursor_put(cur1, &k, &v, putflags);
if (ret)
throw std::runtime_error("Failed to write " + std::string(table) + " record: " + std::string(mdb_strerror(ret)));
}
mdb_cursor_close(cur1);
mdb_cursor_close(cur0);
mdb_txn_commit(txn1);
tx_active1 = false;
mdb_txn_commit(txn0);
tx_active0 = false;
mdb_dbi_close(env1, dbi1);
mdb_dbi_close(env0, dbi0);
}
static bool is_v1_tx(MDB_cursor *c_txs_pruned, MDB_val *tx_id)
{
MDB_val v;
int ret = mdb_cursor_get(c_txs_pruned, tx_id, &v, MDB_SET);
if (ret)
throw std::runtime_error("Failed to find transaction pruned data: " + std::string(mdb_strerror(ret)));
if (v.mv_size == 0)
throw std::runtime_error("Invalid transaction pruned data");
return cryptonote::is_v1_tx(cryptonote::blobdata_ref{(const char*)v.mv_data, v.mv_size});
}
static void prune(MDB_env *env0, MDB_env *env1)
{
MDB_dbi dbi0_blocks, dbi0_txs_pruned, dbi0_txs_prunable, dbi0_tx_indices, dbi1_txs_prunable, dbi1_txs_prunable_tip, dbi1_properties;
MDB_txn *txn0, *txn1;
MDB_cursor *cur0_txs_pruned, *cur0_txs_prunable, *cur0_tx_indices, *cur1_txs_prunable, *cur1_txs_prunable_tip;
bool tx_active0 = false, tx_active1 = false;
int dbr;
MGINFO("Creating pruned txs_prunable");
epee::misc_utils::auto_scope_leave_caller txn_dtor = epee::misc_utils::create_scope_leave_handler([&](){
if (tx_active1) mdb_txn_abort(txn1);
if (tx_active0) mdb_txn_abort(txn0);
});
dbr = mdb_txn_begin(env0, NULL, MDB_RDONLY, &txn0);
if (dbr) throw std::runtime_error("Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr)));
tx_active0 = true;
dbr = mdb_txn_begin(env1, NULL, 0, &txn1);
if (dbr) throw std::runtime_error("Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr)));
tx_active1 = true;
dbr = mdb_dbi_open(txn0, "txs_pruned", MDB_INTEGERKEY, &dbi0_txs_pruned);
if (dbr) throw std::runtime_error("Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr)));
mdb_set_compare(txn0, dbi0_txs_pruned, BlockchainLMDB::compare_uint64);
dbr = mdb_cursor_open(txn0, dbi0_txs_pruned, &cur0_txs_pruned);
if (dbr) throw std::runtime_error("Failed to create LMDB cursor: " + std::string(mdb_strerror(dbr)));
dbr = mdb_dbi_open(txn0, "txs_prunable", MDB_INTEGERKEY, &dbi0_txs_prunable);
if (dbr) throw std::runtime_error("Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr)));
mdb_set_compare(txn0, dbi0_txs_prunable, BlockchainLMDB::compare_uint64);
dbr = mdb_cursor_open(txn0, dbi0_txs_prunable, &cur0_txs_prunable);
if (dbr) throw std::runtime_error("Failed to create LMDB cursor: " + std::string(mdb_strerror(dbr)));
dbr = mdb_dbi_open(txn0, "tx_indices", MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, &dbi0_tx_indices);
if (dbr) throw std::runtime_error("Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr)));
mdb_set_dupsort(txn0, dbi0_tx_indices, BlockchainLMDB::compare_hash32);
dbr = mdb_cursor_open(txn0, dbi0_tx_indices, &cur0_tx_indices);
if (dbr) throw std::runtime_error("Failed to create LMDB cursor: " + std::string(mdb_strerror(dbr)));
dbr = mdb_dbi_open(txn1, "txs_prunable", MDB_INTEGERKEY, &dbi1_txs_prunable);
if (dbr) throw std::runtime_error("Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr)));
mdb_set_compare(txn1, dbi1_txs_prunable, BlockchainLMDB::compare_uint64);
dbr = mdb_cursor_open(txn1, dbi1_txs_prunable, &cur1_txs_prunable);
if (dbr) throw std::runtime_error("Failed to create LMDB cursor: " + std::string(mdb_strerror(dbr)));
dbr = mdb_dbi_open(txn1, "txs_prunable_tip", MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, &dbi1_txs_prunable_tip);
if (dbr) throw std::runtime_error("Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr)));
mdb_set_dupsort(txn1, dbi1_txs_prunable_tip, BlockchainLMDB::compare_uint64);
dbr = mdb_cursor_open(txn1, dbi1_txs_prunable_tip, &cur1_txs_prunable_tip);
if (dbr) throw std::runtime_error("Failed to create LMDB cursor: " + std::string(mdb_strerror(dbr)));
dbr = mdb_drop(txn1, dbi1_txs_prunable, 0);
if (dbr) throw std::runtime_error("Failed to empty LMDB table: " + std::string(mdb_strerror(dbr)));
dbr = mdb_drop(txn1, dbi1_txs_prunable_tip, 0);
if (dbr) throw std::runtime_error("Failed to empty LMDB table: " + std::string(mdb_strerror(dbr)));
dbr = mdb_dbi_open(txn1, "properties", 0, &dbi1_properties);
if (dbr) throw std::runtime_error("Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr)));
MDB_val k, v;
uint32_t pruning_seed = tools::make_pruning_seed(tools::get_random_stripe(), CRYPTONOTE_PRUNING_LOG_STRIPES);
static char pruning_seed_key[] = "pruning_seed";
k.mv_data = pruning_seed_key;
k.mv_size = strlen("pruning_seed") + 1;
v.mv_data = (void*)&pruning_seed;
v.mv_size = sizeof(pruning_seed);
dbr = mdb_put(txn1, dbi1_properties, &k, &v, 0);
if (dbr) throw std::runtime_error("Failed to save pruning seed: " + std::string(mdb_strerror(dbr)));
MDB_stat stats;
dbr = mdb_dbi_open(txn0, "blocks", 0, &dbi0_blocks);
if (dbr) throw std::runtime_error("Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr)));
dbr = mdb_stat(txn0, dbi0_blocks, &stats);
if (dbr) throw std::runtime_error("Failed to query size of blocks: " + std::string(mdb_strerror(dbr)));
mdb_dbi_close(env0, dbi0_blocks);
const uint64_t blockchain_height = stats.ms_entries;
size_t nrecords = 0, bytes = 0;
MDB_cursor_op op = MDB_FIRST;
while (1)
{
int ret = mdb_cursor_get(cur0_tx_indices, &k, &v, op);
op = MDB_NEXT;
if (ret == MDB_NOTFOUND)
break;
if (ret) throw std::runtime_error("Failed to enumerate records: " + std::string(mdb_strerror(ret)));
const txindex *ti = (const txindex*)v.mv_data;
const uint64_t block_height = ti->data.block_id;
MDB_val_set(kk, ti->data.tx_id);
if (block_height + CRYPTONOTE_PRUNING_TIP_BLOCKS >= blockchain_height)
{
MDEBUG(block_height << "/" << blockchain_height << " is in tip");
MDB_val_set(vv, block_height);
dbr = mdb_cursor_put(cur1_txs_prunable_tip, &kk, &vv, 0);
if (dbr) throw std::runtime_error("Failed to write prunable tx tip data: " + std::string(mdb_strerror(dbr)));
bytes += kk.mv_size + vv.mv_size;
}
if (tools::has_unpruned_block(block_height, blockchain_height, pruning_seed) || is_v1_tx(cur0_txs_pruned, &kk))
{
MDB_val vv;
dbr = mdb_cursor_get(cur0_txs_prunable, &kk, &vv, MDB_SET);
if (dbr) throw std::runtime_error("Failed to read prunable tx data: " + std::string(mdb_strerror(dbr)));
bytes += kk.mv_size + vv.mv_size;
if (resize_point(++nrecords, env1, &txn1, bytes))
{
dbr = mdb_cursor_open(txn1, dbi1_txs_prunable, &cur1_txs_prunable);
if (dbr) throw std::runtime_error("Failed to create LMDB cursor: " + std::string(mdb_strerror(dbr)));
dbr = mdb_cursor_open(txn1, dbi1_txs_prunable_tip, &cur1_txs_prunable_tip);
if (dbr) throw std::runtime_error("Failed to create LMDB cursor: " + std::string(mdb_strerror(dbr)));
}
dbr = mdb_cursor_put(cur1_txs_prunable, &kk, &vv, 0);
if (dbr) throw std::runtime_error("Failed to write prunable tx data: " + std::string(mdb_strerror(dbr)));
}
else
{
MDEBUG("" << block_height << "/" << blockchain_height << " should be pruned, dropping");
}
}
mdb_cursor_close(cur1_txs_prunable_tip);
mdb_cursor_close(cur1_txs_prunable);
mdb_cursor_close(cur0_txs_prunable);
mdb_cursor_close(cur0_txs_pruned);
mdb_cursor_close(cur0_tx_indices);
mdb_txn_commit(txn1);
tx_active1 = false;
mdb_txn_commit(txn0);
tx_active0 = false;
mdb_dbi_close(env1, dbi1_properties);
mdb_dbi_close(env1, dbi1_txs_prunable_tip);
mdb_dbi_close(env1, dbi1_txs_prunable);
mdb_dbi_close(env0, dbi0_txs_prunable);
mdb_dbi_close(env0, dbi0_txs_pruned);
mdb_dbi_close(env0, dbi0_tx_indices);
}
static bool parse_db_sync_mode(std::string db_sync_mode, uint64_t &db_flags)
{
std::vector<std::string> options;
boost::trim(db_sync_mode);
boost::split(options, db_sync_mode, boost::is_any_of(" :"));
for(const auto &option : options)
MDEBUG("option: " << option);
// default to fast:async:1
uint64_t DEFAULT_FLAGS = DBF_FAST;
db_flags = 0;
if(options.size() == 0)
{
// default to fast:async:1
db_flags = DEFAULT_FLAGS;
}
bool safemode = false;
if(options.size() >= 1)
{
if(options[0] == "safe")
{
safemode = true;
db_flags = DBF_SAFE;
}
else if(options[0] == "fast")
{
db_flags = DBF_FAST;
}
else if(options[0] == "fastest")
{
db_flags = DBF_FASTEST;
records_per_sync = 1000; // default to fastest:async:1000
}
else
return false;
}
if(options.size() >= 2 && !safemode)
{
char *endptr;
uint64_t bps = strtoull(options[1].c_str(), &endptr, 0);
if (*endptr != '\0')
return false;
records_per_sync = bps;
}
return true;
}
int main(int argc, char* argv[])
{
TRY_ENTRY();
epee::string_tools::set_module_name_and_folder(argv[0]);
std::string default_db_type = "lmdb";
std::string available_dbs = cryptonote::blockchain_db_types(", ");
available_dbs = "available: " + available_dbs;
uint32_t log_level = 0;
tools::on_startup();
boost::filesystem::path output_file_path;
po::options_description desc_cmd_only("Command line options");
po::options_description desc_cmd_sett("Command line options and settings options");
const command_line::arg_descriptor<std::string> arg_log_level = {"log-level", "0-4 or categories", ""};
const command_line::arg_descriptor<std::string> arg_database = {
"database", available_dbs.c_str(), default_db_type
};
const command_line::arg_descriptor<std::string> arg_db_sync_mode = {
"db-sync-mode"
, "Specify sync option, using format [safe|fast|fastest]:[nrecords_per_sync]."
, "fast:1000"
};
const command_line::arg_descriptor<bool> arg_copy_pruned_database = {"copy-pruned-database", "Copy database anyway if already pruned"};
command_line::add_arg(desc_cmd_sett, cryptonote::arg_data_dir);
command_line::add_arg(desc_cmd_sett, cryptonote::arg_testnet_on);
command_line::add_arg(desc_cmd_sett, cryptonote::arg_stagenet_on);
command_line::add_arg(desc_cmd_sett, arg_log_level);
command_line::add_arg(desc_cmd_sett, arg_database);
command_line::add_arg(desc_cmd_sett, arg_db_sync_mode);
command_line::add_arg(desc_cmd_sett, arg_copy_pruned_database);
command_line::add_arg(desc_cmd_only, command_line::arg_help);
po::options_description desc_options("Allowed options");
desc_options.add(desc_cmd_only).add(desc_cmd_sett);
po::variables_map vm;
bool r = command_line::handle_error_helper(desc_options, [&]()
{
auto parser = po::command_line_parser(argc, argv).options(desc_options);
po::store(parser.run(), vm);
po::notify(vm);
return true;
});
if (! r)
return 1;
if (command_line::get_arg(vm, command_line::arg_help))
{
std::cout << "Monero '" << MONERO_RELEASE_NAME << "' (v" << MONERO_VERSION_FULL << ")" << ENDL << ENDL;
std::cout << desc_options << std::endl;
return 1;
}
mlog_configure(mlog_get_default_log_path("monero-blockchain-prune.log"), true);
if (!command_line::is_arg_defaulted(vm, arg_log_level))
mlog_set_log(command_line::get_arg(vm, arg_log_level).c_str());
else
mlog_set_log(std::string(std::to_string(log_level) + ",bcutil:INFO").c_str());
MINFO("Starting...");
bool opt_testnet = command_line::get_arg(vm, cryptonote::arg_testnet_on);
bool opt_stagenet = command_line::get_arg(vm, cryptonote::arg_stagenet_on);
network_type net_type = opt_testnet ? TESTNET : opt_stagenet ? STAGENET : MAINNET;
bool opt_copy_pruned_database = command_line::get_arg(vm, arg_copy_pruned_database);
std::string data_dir = command_line::get_arg(vm, cryptonote::arg_data_dir);
while (boost::ends_with(data_dir, "/") || boost::ends_with(data_dir, "\\"))
data_dir.pop_back();
std::string db_type = command_line::get_arg(vm, arg_database);
if (!cryptonote::blockchain_valid_db_type(db_type))
{
MERROR("Invalid database type: " << db_type);
return 1;
}
if (db_type != "lmdb")
{
MERROR("Unsupported database type: " << db_type << ". Only lmdb is supported");
return 1;
}
std::string db_sync_mode = command_line::get_arg(vm, arg_db_sync_mode);
uint64_t db_flags = 0;
if (!parse_db_sync_mode(db_sync_mode, db_flags))
{
MERROR("Invalid db sync mode: " << db_sync_mode);
return 1;
}
// If we wanted to use the memory pool, we would set up a fake_core.
// Use Blockchain instead of lower-level BlockchainDB for two reasons:
// 1. Blockchain has the init() method for easy setup
// 2. exporter needs to use get_current_blockchain_height(), get_block_id_by_height(), get_block_by_hash()
//
// cannot match blockchain_storage setup above with just one line,
// e.g.
// Blockchain* core_storage = new Blockchain(NULL);
// because unlike blockchain_storage constructor, which takes a pointer to
// tx_memory_pool, Blockchain's constructor takes tx_memory_pool object.
MINFO("Initializing source blockchain (BlockchainDB)");
std::array<std::unique_ptr<Blockchain>, 2> core_storage;
Blockchain *blockchain = NULL;
tx_memory_pool m_mempool(*blockchain);
boost::filesystem::path paths[2];
bool already_pruned = false;
for (size_t n = 0; n < core_storage.size(); ++n)
{
core_storage[n].reset(new Blockchain(m_mempool));
BlockchainDB* db = new_db(db_type);
if (db == NULL)
{
MERROR("Attempted to use non-existent database type: " << db_type);
throw std::runtime_error("Attempting to use non-existent database type");
}
MDEBUG("database: " << db_type);
if (n == 1)
{
paths[1] = boost::filesystem::path(data_dir) / (db->get_db_name() + "-pruned");
if (boost::filesystem::exists(paths[1]))
{
if (!boost::filesystem::is_directory(paths[1]))
{
MERROR("LMDB needs a directory path, but a file was passed: " << paths[1].string());
return 1;
}
}
else
{
if (!boost::filesystem::create_directories(paths[1]))
{
MERROR("Failed to create directory: " << paths[1].string());
return 1;
}
}
db_path = paths[1].string();
}
else
{
paths[0] = boost::filesystem::path(data_dir) / db->get_db_name();
}
MINFO("Loading blockchain from folder " << paths[n] << " ...");
try
{
db->open(paths[n].string(), n == 0 ? DBF_RDONLY : 0);
}
catch (const std::exception& e)
{
MERROR("Error opening database: " << e.what());
return 1;
}
r = core_storage[n]->init(db, net_type);
std::string source_dest = n == 0 ? "source" : "pruned";
CHECK_AND_ASSERT_MES(r, 1, "Failed to initialize " << source_dest << " blockchain storage");
MINFO(source_dest << " blockchain storage initialized OK");
if (n == 0 && core_storage[0]->get_blockchain_pruning_seed())
{
if (!opt_copy_pruned_database)
{
MERROR("Blockchain is already pruned, use --" << arg_copy_pruned_database.name << " to copy it anyway");
return 1;
}
already_pruned = true;
}
}
core_storage[0]->deinit();
core_storage[0].reset(NULL);
core_storage[1]->deinit();
core_storage[1].reset(NULL);
MINFO("Pruning...");
MDB_env *env0 = NULL, *env1 = NULL;
open(env0, paths[0], db_flags, true);
open(env1, paths[1], db_flags, false);
copy_table(env0, env1, "blocks", MDB_INTEGERKEY, MDB_APPEND);
copy_table(env0, env1, "block_info", MDB_INTEGERKEY | MDB_DUPSORT| MDB_DUPFIXED, MDB_APPENDDUP, BlockchainLMDB::compare_uint64);
copy_table(env0, env1, "block_heights", MDB_INTEGERKEY | MDB_DUPSORT| MDB_DUPFIXED, 0, BlockchainLMDB::compare_hash32);
//copy_table(env0, env1, "txs", MDB_INTEGERKEY);
copy_table(env0, env1, "txs_pruned", MDB_INTEGERKEY, MDB_APPEND);
copy_table(env0, env1, "txs_prunable_hash", MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, MDB_APPEND);
// not copied: prunable, prunable_tip
copy_table(env0, env1, "tx_indices", MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, 0, BlockchainLMDB::compare_hash32);
copy_table(env0, env1, "tx_outputs", MDB_INTEGERKEY, MDB_APPEND);
copy_table(env0, env1, "output_txs", MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, MDB_APPENDDUP, BlockchainLMDB::compare_uint64);
copy_table(env0, env1, "output_amounts", MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, MDB_APPENDDUP, BlockchainLMDB::compare_uint64);
copy_table(env0, env1, "spent_keys", MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, MDB_NODUPDATA, BlockchainLMDB::compare_hash32);
copy_table(env0, env1, "txpool_meta", 0, MDB_NODUPDATA, BlockchainLMDB::compare_hash32);
copy_table(env0, env1, "txpool_blob", 0, MDB_NODUPDATA, BlockchainLMDB::compare_hash32);
copy_table(env0, env1, "hf_versions", MDB_INTEGERKEY, MDB_APPEND);
copy_table(env0, env1, "properties", 0, 0, BlockchainLMDB::compare_string);
if (already_pruned)
{
copy_table(env0, env1, "txs_prunable", MDB_INTEGERKEY, MDB_APPEND, BlockchainLMDB::compare_uint64);
copy_table(env0, env1, "txs_prunable_tip", MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, MDB_NODUPDATA, BlockchainLMDB::compare_uint64);
}
else
{
prune(env0, env1);
}
close(env1);
close(env0);
MINFO("Swapping databases, pre-pruning blockchain will be left in " << paths[0].string() + "-old and can be removed if desired");
if (replace_file(paths[0].string(), paths[0].string() + "-old") || replace_file(paths[1].string(), paths[0].string()))
{
MERROR("Blockchain pruned OK, but renaming failed");
return 1;
}
MINFO("Blockchain pruned OK");
return 0;
CATCH_ENTRY("Pruning error", 1);
}

2
src/common/CMakeLists.txt
View file

@ -40,6 +40,7 @@ set(common_sources
notify.cpp
password.cpp
perf_timer.cpp
pruning.cpp
spawn.cpp
threadpool.cpp
updates.cpp
@ -69,6 +70,7 @@ set(common_private_headers
http_connection.h
notify.h
pod-class.h
pruning.h
rpc_client.h
scoped_message_writer.h
unordered_containers_boost_serialization.h

116
src/common/pruning.cpp Normal file
View file

@ -0,0 +1,116 @@
// Copyright (c) 2018, The Monero Project
//
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without modification, are
// permitted provided that the following conditions are met:
//
// 1. Redistributions of source code must retain the above copyright notice, this list of
// conditions and the following disclaimer.
//
// 2. Redistributions in binary form must reproduce the above copyright notice, this list
// of conditions and the following disclaimer in the documentation and/or other
// materials provided with the distribution.
//
// 3. Neither the name of the copyright holder nor the names of its contributors may be
// used to endorse or promote products derived from this software without specific
// prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include "cryptonote_config.h"
#include "misc_log_ex.h"
#include "crypto/crypto.h"
#include "pruning.h"
namespace tools
{
uint32_t make_pruning_seed(uint32_t stripe, uint32_t log_stripes)
{
CHECK_AND_ASSERT_THROW_MES(log_stripes <= PRUNING_SEED_LOG_STRIPES_MASK, "log_stripes out of range");
CHECK_AND_ASSERT_THROW_MES(stripe > 0 && stripe <= (1ul << log_stripes), "stripe out of range");
return (log_stripes << PRUNING_SEED_LOG_STRIPES_SHIFT) | ((stripe - 1) << PRUNING_SEED_STRIPE_SHIFT);
}
bool has_unpruned_block(uint64_t block_height, uint64_t blockchain_height, uint32_t pruning_seed)
{
const uint32_t stripe = get_pruning_stripe(pruning_seed);
if (stripe == 0)
return true;
const uint32_t log_stripes = get_pruning_log_stripes(pruning_seed);
uint32_t block_stripe = get_pruning_stripe(block_height, blockchain_height, log_stripes);
return block_stripe == 0 || block_stripe == stripe;
}
uint32_t get_pruning_stripe(uint64_t block_height, uint64_t blockchain_height, uint32_t log_stripes)
{
if (block_height + CRYPTONOTE_PRUNING_TIP_BLOCKS >= blockchain_height)
return 0;
return ((block_height / CRYPTONOTE_PRUNING_STRIPE_SIZE) & (uint64_t)((1ul << log_stripes) - 1)) + 1;
}
uint32_t get_pruning_seed(uint64_t block_height, uint64_t blockchain_height, uint32_t log_stripes)
{
const uint32_t stripe = get_pruning_stripe(block_height, blockchain_height, log_stripes);
if (stripe == 0)
return 0;
return make_pruning_seed(stripe, log_stripes);
}
uint64_t get_next_unpruned_block_height(uint64_t block_height, uint64_t blockchain_height, uint32_t pruning_seed)
{
CHECK_AND_ASSERT_MES(block_height <= CRYPTONOTE_MAX_BLOCK_NUMBER+1, block_height, "block_height too large");
CHECK_AND_ASSERT_MES(blockchain_height <= CRYPTONOTE_MAX_BLOCK_NUMBER+1, block_height, "blockchain_height too large");
const uint32_t stripe = get_pruning_stripe(pruning_seed);
if (stripe == 0)
return block_height;
if (block_height + CRYPTONOTE_PRUNING_TIP_BLOCKS >= blockchain_height)
return block_height;
const uint32_t seed_log_stripes = get_pruning_log_stripes(pruning_seed);
const uint64_t log_stripes = seed_log_stripes ? seed_log_stripes : CRYPTONOTE_PRUNING_LOG_STRIPES;
const uint64_t mask = (1ul << log_stripes) - 1;
const uint32_t block_pruning_stripe = ((block_height / CRYPTONOTE_PRUNING_STRIPE_SIZE) & mask) + 1;
if (block_pruning_stripe == stripe)
return block_height;
const uint64_t cycles = ((block_height / CRYPTONOTE_PRUNING_STRIPE_SIZE) >> log_stripes);
const uint64_t cycle_start = cycles + ((stripe > block_pruning_stripe) ? 0 : 1);
const uint64_t h = cycle_start * (CRYPTONOTE_PRUNING_STRIPE_SIZE << log_stripes) + (stripe - 1) * CRYPTONOTE_PRUNING_STRIPE_SIZE;
if (h + CRYPTONOTE_PRUNING_TIP_BLOCKS > blockchain_height)
return blockchain_height < CRYPTONOTE_PRUNING_TIP_BLOCKS ? 0 : blockchain_height - CRYPTONOTE_PRUNING_TIP_BLOCKS;
CHECK_AND_ASSERT_MES(h >= block_height, block_height, "h < block_height, unexpected");
return h;
}
uint64_t get_next_pruned_block_height(uint64_t block_height, uint64_t blockchain_height, uint32_t pruning_seed)
{
const uint32_t stripe = get_pruning_stripe(pruning_seed);
if (stripe == 0)
return blockchain_height;
if (block_height + CRYPTONOTE_PRUNING_TIP_BLOCKS >= blockchain_height)
return blockchain_height;
const uint32_t seed_log_stripes = get_pruning_log_stripes(pruning_seed);
const uint64_t log_stripes = seed_log_stripes ? seed_log_stripes : CRYPTONOTE_PRUNING_LOG_STRIPES;
const uint64_t mask = (1ul << log_stripes) - 1;
const uint32_t block_pruning_seed = ((block_height / CRYPTONOTE_PRUNING_STRIPE_SIZE) & mask) + 1;
if (block_pruning_seed != stripe)
return block_height;
const uint32_t next_stripe = 1 + (block_pruning_seed & mask);
return get_next_unpruned_block_height(block_height, blockchain_height, tools::make_pruning_seed(next_stripe, log_stripes));
}
uint32_t get_random_stripe()
{
return 1 + crypto::rand<uint8_t>() % (1ul << CRYPTONOTE_PRUNING_LOG_STRIPES);
}
}

52
src/common/pruning.h Normal file
View file

@ -0,0 +1,52 @@
// Copyright (c) 2018, The Monero Project
//
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without modification, are
// permitted provided that the following conditions are met:
//
// 1. Redistributions of source code must retain the above copyright notice, this list of
// conditions and the following disclaimer.
//
// 2. Redistributions in binary form must reproduce the above copyright notice, this list
// of conditions and the following disclaimer in the documentation and/or other
// materials provided with the distribution.
//
// 3. Neither the name of the copyright holder nor the names of its contributors may be
// used to endorse or promote products derived from this software without specific
// prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#pragma once
#include <stdint.h>
namespace tools
{
static constexpr uint32_t PRUNING_SEED_LOG_STRIPES_SHIFT = 7;
static constexpr uint32_t PRUNING_SEED_LOG_STRIPES_MASK = 0x7;
static constexpr uint32_t PRUNING_SEED_STRIPE_SHIFT = 0;
static constexpr uint32_t PRUNING_SEED_STRIPE_MASK = 0x7f;
constexpr inline uint32_t get_pruning_log_stripes(uint32_t pruning_seed) { return (pruning_seed >> PRUNING_SEED_LOG_STRIPES_SHIFT) & PRUNING_SEED_LOG_STRIPES_MASK; }
inline uint32_t get_pruning_stripe(uint32_t pruning_seed) { if (pruning_seed == 0) return 0; return 1 + ((pruning_seed >> PRUNING_SEED_STRIPE_SHIFT) & PRUNING_SEED_STRIPE_MASK); }
uint32_t make_pruning_seed(uint32_t stripe, uint32_t log_stripes);
bool has_unpruned_block(uint64_t block_height, uint64_t blockchain_height, uint32_t pruning_seed);
uint32_t get_pruning_stripe(uint64_t block_height, uint64_t blockchain_height, uint32_t log_stripes);
uint32_t get_pruning_seed(uint64_t block_height, uint64_t blockchain_height, uint32_t log_stripes);
uint64_t get_next_unpruned_block_height(uint64_t block_height, uint64_t blockchain_height, uint32_t pruning_seed);
uint64_t get_next_pruned_block_height(uint64_t block_height, uint64_t blockchain_height, uint32_t pruning_seed);
uint32_t get_random_stripe();
}

2
src/common/varint.h
View file

@ -123,6 +123,6 @@ namespace tools {
*/
template<typename InputIt, typename T>
int read_varint(InputIt &&first, InputIt &&last, T &i) {
return read_varint<std::numeric_limits<T>::digits, InputIt, T>(std::move(first), std::move(last), i);
return read_varint<std::numeric_limits<T>::digits>(std::forward<InputIt>(first), std::forward<InputIt>(last), i);
}
}

4
src/cryptonote_basic/blobdatatype.h
View file

@ -30,7 +30,11 @@
#pragma once
#include <string>
#include "span.h"
namespace cryptonote
{
typedef std::string blobdata;
typedef epee::span<const char> blobdata_ref;
}

23
src/cryptonote_basic/connection_context.h
View file

@ -40,7 +40,7 @@ namespace cryptonote
struct cryptonote_connection_context: public epee::net_utils::connection_context_base
{
cryptonote_connection_context(): m_state(state_before_handshake), m_remote_blockchain_height(0), m_last_response_height(0),
m_last_request_time(boost::posix_time::microsec_clock::universal_time()), m_callback_request_count(0), m_last_known_hash(crypto::null_hash) {}
m_last_request_time(boost::date_time::not_a_date_time), m_callback_request_count(0), m_last_known_hash(crypto::null_hash), m_pruning_seed(0), m_anchor(false) {}
enum state
{
@ -59,6 +59,8 @@ namespace cryptonote
boost::posix_time::ptime m_last_request_time;
epee::copyable_atomic m_callback_request_count; //in debug purpose: problem with double callback rise
crypto::hash m_last_known_hash;
uint32_t m_pruning_seed;
bool m_anchor;
//size_t m_score; TODO: add score calculations
};
@ -81,4 +83,23 @@ namespace cryptonote
}
}
inline char get_protocol_state_char(cryptonote_connection_context::state s)
{
switch (s)
{
case cryptonote_connection_context::state_before_handshake:
return 'h';
case cryptonote_connection_context::state_synchronizing:
return 's';
case cryptonote_connection_context::state_standby:
return 'w';
case cryptonote_connection_context::state_idle:
return 'i';
case cryptonote_connection_context::state_normal:
return 'n';
default:
return 'u';
}
}
}

15
src/cryptonote_basic/cryptonote_basic.h
View file

@ -201,9 +201,11 @@ namespace cryptonote
mutable crypto::hash hash;
mutable size_t blob_size;
bool pruned;
transaction();
transaction(const transaction &t): transaction_prefix(t), hash_valid(false), blob_size_valid(false), signatures(t.signatures), rct_signatures(t.rct_signatures) { if (t.is_hash_valid()) { hash = t.hash; set_hash_valid(true); } if (t.is_blob_size_valid()) { blob_size = t.blob_size; set_blob_size_valid(true); } }
transaction &operator=(const transaction &t) { transaction_prefix::operator=(t); set_hash_valid(false); set_blob_size_valid(false); signatures = t.signatures; rct_signatures = t.rct_signatures; if (t.is_hash_valid()) { hash = t.hash; set_hash_valid(true); } if (t.is_blob_size_valid()) { blob_size = t.blob_size; set_blob_size_valid(true); } return *this; }
transaction(const transaction &t): transaction_prefix(t), hash_valid(false), blob_size_valid(false), signatures(t.signatures), rct_signatures(t.rct_signatures), pruned(t.pruned) { if (t.is_hash_valid()) { hash = t.hash; set_hash_valid(true); } if (t.is_blob_size_valid()) { blob_size = t.blob_size; set_blob_size_valid(true); } }
transaction &operator=(const transaction &t) { transaction_prefix::operator=(t); set_hash_valid(false); set_blob_size_valid(false); signatures = t.signatures; rct_signatures = t.rct_signatures; if (t.is_hash_valid()) { hash = t.hash; set_hash_valid(true); } if (t.is_blob_size_valid()) { blob_size = t.blob_size; set_blob_size_valid(true); } pruned = t.pruned; return *this; }
virtual ~transaction();
void set_null();
void invalidate_hashes();
@ -232,7 +234,7 @@ namespace cryptonote
if (!signatures_not_expected && vin.size() != signatures.size())
return false;
for (size_t i = 0; i < vin.size(); ++i)
if (!pruned) for (size_t i = 0; i < vin.size(); ++i)
{
size_t signature_size = get_signature_size(vin[i]);
if (signatures_not_expected)
@ -263,7 +265,7 @@ namespace cryptonote
bool r = rct_signatures.serialize_rctsig_base(ar, vin.size(), vout.size());
if (!r || !ar.stream().good()) return false;
ar.end_object();
if (rct_signatures.type != rct::RCTTypeNull)
if (!pruned && rct_signatures.type != rct::RCTTypeNull)
{
ar.tag("rctsig_prunable");
ar.begin_object();
@ -274,6 +276,8 @@ namespace cryptonote
}
}
}
if (!typename Archive<W>::is_saving())
pruned = false;
END_SERIALIZE()
template<bool W, template <bool> class Archive>
@ -295,6 +299,8 @@ namespace cryptonote
ar.end_object();
}
}
if (!typename Archive<W>::is_saving())
pruned = true;
return true;
}
@ -322,6 +328,7 @@ namespace cryptonote
rct_signatures.type = rct::RCTTypeNull;
set_hash_valid(false);
set_blob_size_valid(false);
pruned = false;
}
inline

17
src/cryptonote_basic/cryptonote_format_utils.cpp
View file

@ -196,6 +196,7 @@ namespace cryptonote
bool r = tx.serialize_base(ba);
CHECK_AND_ASSERT_MES(r, false, "Failed to parse transaction from blob");
CHECK_AND_ASSERT_MES(expand_transaction_1(tx, true), false, "Failed to expand transaction data");
tx.invalidate_hashes();
return true;
}
//---------------------------------------------------------------
@ -225,6 +226,22 @@ namespace cryptonote
return true;
}
//---------------------------------------------------------------
bool is_v1_tx(const blobdata_ref& tx_blob)
{
uint64_t version;
const char* begin = static_cast<const char*>(tx_blob.data());
const char* end = begin + tx_blob.size();
int read = tools::read_varint(begin, end, version);
if (read <= 0)
throw std::runtime_error("Internal error getting transaction version");
return version <= 1;
}
//---------------------------------------------------------------
bool is_v1_tx(const blobdata& tx_blob)
{
return is_v1_tx(blobdata_ref{tx_blob.data(), tx_blob.size()});
}
//---------------------------------------------------------------
bool generate_key_image_helper(const account_keys& ack, const std::unordered_map<crypto::public_key, subaddress_index>& subaddresses, const crypto::public_key& out_key, const crypto::public_key& tx_public_key, const std::vector<crypto::public_key>& additional_tx_public_keys, size_t real_output_index, keypair& in_ephemeral, crypto::key_image& ki, hw::device &hwdev)
{
crypto::key_derivation recv_derivation = AUTO_VAL_INIT(recv_derivation);

2
src/cryptonote_basic/cryptonote_format_utils.h
View file

@ -53,6 +53,8 @@ namespace cryptonote
bool parse_and_validate_tx_from_blob(const blobdata& tx_blob, transaction& tx, crypto::hash& tx_hash, crypto::hash& tx_prefix_hash);
bool parse_and_validate_tx_from_blob(const blobdata& tx_blob, transaction& tx);
bool parse_and_validate_tx_base_from_blob(const blobdata& tx_blob, transaction& tx);
bool is_v1_tx(const blobdata_ref& tx_blob);
bool is_v1_tx(const blobdata& tx_blob);
template<typename T>
bool find_tx_extra_field_by_type(const std::vector<tx_extra_field>& tx_extra_fields, T& field, size_t index = 0)

5
src/cryptonote_config.h
View file

@ -150,6 +150,11 @@
#define BULLETPROOF_MAX_OUTPUTS 16
#define CRYPTONOTE_PRUNING_STRIPE_SIZE 4096 // the smaller, the smoother the increase
#define CRYPTONOTE_PRUNING_LOG_STRIPES 3 // the higher, the more space saved
#define CRYPTONOTE_PRUNING_TIP_BLOCKS 5500 // the smaller, the more space saved
//#define CRYPTONOTE_PRUNING_DEBUG_SPOOF_SEED
// New constants are intended to go here
namespace config
{

96
src/cryptonote_core/blockchain.cpp
View file

@ -53,6 +53,8 @@
#include "ringct/rctSigs.h"
#include "common/perf_timer.h"
#include "common/notify.h"
#include "common/varint.h"
#include "common/pruning.h"
#undef MONERO_DEFAULT_LOG_CATEGORY
#define MONERO_DEFAULT_LOG_CATEGORY "blockchain"
@ -646,8 +648,14 @@ block Blockchain::pop_block_from_blockchain()
m_hardfork->on_block_popped(1);
// return transactions from popped block to the tx_pool
size_t pruned = 0;
for (transaction& tx : popped_txs)
{
if (tx.pruned)
{
++pruned;
continue;
}
if (!is_coinbase(tx))
{
cryptonote::tx_verification_context tvc = AUTO_VAL_INIT(tvc);
@ -669,6 +677,8 @@ block Blockchain::pop_block_from_blockchain()
}
}
}
if (pruned)
MWARNING(pruned << " pruned txes could not be added back to the txpool");
m_blocks_longhash_table.clear();
m_scan_table.clear();
@ -2044,6 +2054,51 @@ bool Blockchain::get_transactions_blobs(const t_ids_container& txs_ids, t_tx_con
return true;
}
//------------------------------------------------------------------
size_t get_transaction_version(const cryptonote::blobdata &bd)
{
size_t version;
const char* begin = static_cast<const char*>(bd.data());
const char* end = begin + bd.size();
int read = tools::read_varint(begin, end, version);
if (read <= 0)
throw std::runtime_error("Internal error getting transaction version");
return version;
}
//------------------------------------------------------------------
template<class t_ids_container, class t_tx_container, class t_missed_container>
bool Blockchain::get_split_transactions_blobs(const t_ids_container& txs_ids, t_tx_container& txs, t_missed_container& missed_txs) const
{
LOG_PRINT_L3("Blockchain::" << __func__);
CRITICAL_REGION_LOCAL(m_blockchain_lock);
reserve_container(txs, txs_ids.size());
for (const auto& tx_hash : txs_ids)
{
try
{
cryptonote::blobdata tx;
if (m_db->get_pruned_tx_blob(tx_hash, tx))
{
txs.push_back(std::make_tuple(tx_hash, std::move(tx), crypto::null_hash, cryptonote::blobdata()));
if (!is_v1_tx(std::get<1>(txs.back())) && !m_db->get_prunable_tx_hash(tx_hash, std::get<2>(txs.back())))
{
MERROR("Prunable data hash not found for " << tx_hash);
return false;
}
if (!m_db->get_prunable_tx_blob(tx_hash, std::get<3>(txs.back())))
std::get<3>(txs.back()).clear();
}
else
missed_txs.push_back(tx_hash);
}
catch (const std::exception& e)
{
return false;
}
}
return true;
}
//------------------------------------------------------------------
template<class t_ids_container, class t_tx_container, class t_missed_container>
bool Blockchain::get_transactions(const t_ids_container& txs_ids, t_tx_container& txs, t_missed_container& missed_txs) const
{
@ -2092,9 +2147,12 @@ bool Blockchain::find_blockchain_supplement(const std::list<crypto::hash>& qbloc
m_db->block_txn_start(true);
current_height = get_current_blockchain_height();
const uint32_t pruning_seed = get_blockchain_pruning_seed();
start_height = tools::get_next_unpruned_block_height(start_height, current_height, pruning_seed);
uint64_t stop_height = tools::get_next_pruned_block_height(start_height, current_height, pruning_seed);
size_t count = 0;
hashes.reserve(std::max((size_t)(current_height - start_height), (size_t)BLOCKS_IDS_SYNCHRONIZING_DEFAULT_COUNT));
for(size_t i = start_height; i < current_height && count < BLOCKS_IDS_SYNCHRONIZING_DEFAULT_COUNT; i++, count++)
hashes.reserve(std::min((size_t)(stop_height - start_height), (size_t)BLOCKS_IDS_SYNCHRONIZING_DEFAULT_COUNT));
for(size_t i = start_height; i < stop_height && count < BLOCKS_IDS_SYNCHRONIZING_DEFAULT_COUNT; i++, count++)
{
hashes.push_back(m_db->get_block_hash_from_height(i));
}
@ -3369,7 +3427,7 @@ leave:
{
if (memcmp(&hash, &expected_hash, sizeof(hash)) != 0)
{
MERROR_VER("Block with id is INVALID: " << id);
MERROR_VER("Block with id is INVALID: " << id << ", expected " << expected_hash);
bvc.m_verifivation_failed = true;
goto leave;
}
@ -3635,6 +3693,35 @@ leave:
return true;
}
//------------------------------------------------------------------
bool Blockchain::prune_blockchain(uint32_t pruning_seed)
{
uint8_t hf_version = m_hardfork->get_current_version();
if (hf_version < 10)
{
MERROR("Most of the network will only be ready for pruned blockchains from v10, not pruning");
return false;
}
return m_db->prune_blockchain(pruning_seed);
}
//------------------------------------------------------------------
bool Blockchain::update_blockchain_pruning()
{
m_tx_pool.lock();
epee::misc_utils::auto_scope_leave_caller unlocker = epee::misc_utils::create_scope_leave_handler([&](){m_tx_pool.unlock();});
CRITICAL_REGION_LOCAL(m_blockchain_lock);
return m_db->update_pruning();
}
//------------------------------------------------------------------
bool Blockchain::check_blockchain_pruning()
{
m_tx_pool.lock();
epee::misc_utils::auto_scope_leave_caller unlocker = epee::misc_utils::create_scope_leave_handler([&](){m_tx_pool.unlock();});
CRITICAL_REGION_LOCAL(m_blockchain_lock);
return m_db->check_pruning();
}
//------------------------------------------------------------------
bool Blockchain::update_next_cumulative_weight_limit()
{
uint64_t full_reward_zone = get_min_block_weight(get_current_hard_fork_version());
@ -3848,6 +3935,8 @@ bool Blockchain::cleanup_handle_incoming_blocks(bool force_sync)
CRITICAL_REGION_END();
m_tx_pool.unlock();
update_blockchain_pruning();
return success;
}
@ -4621,4 +4710,5 @@ void Blockchain::cache_block_template(const block &b, const cryptonote::account_
namespace cryptonote {
template bool Blockchain::get_transactions(const std::vector<crypto::hash>&, std::vector<transaction>&, std::vector<crypto::hash>&) const;
template bool Blockchain::get_transactions_blobs(const std::vector<crypto::hash>&, std::vector<cryptonote::blobdata>&, std::vector<crypto::hash>&, bool) const;
template bool Blockchain::get_split_transactions_blobs(const std::vector<crypto::hash>&, std::vector<std::tuple<crypto::hash, cryptonote::blobdata, crypto::hash, cryptonote::blobdata>>&, std::vector<crypto::hash>&) const;
}

6
src/cryptonote_core/blockchain.h
View file

@ -677,6 +677,8 @@ namespace cryptonote
template<class t_ids_container, class t_tx_container, class t_missed_container>
bool get_transactions_blobs(const t_ids_container& txs_ids, t_tx_container& txs, t_missed_container& missed_txs, bool pruned = false) const;
template<class t_ids_container, class t_tx_container, class t_missed_container>
bool get_split_transactions_blobs(const t_ids_container& txs_ids, t_tx_container& txs, t_missed_container& missed_txs) const;
template<class t_ids_container, class t_tx_container, class t_missed_container>
bool get_transactions(const t_ids_container& txs_ids, t_tx_container& txs, t_missed_container& missed_txs) const;
//debug functions
@ -956,6 +958,10 @@ namespace cryptonote
bool is_within_compiled_block_hash_area(uint64_t height) const;
bool is_within_compiled_block_hash_area() const { return is_within_compiled_block_hash_area(m_db->height()); }
uint64_t prevalidate_block_hashes(uint64_t height, const std::vector<crypto::hash> &hashes);
uint32_t get_blockchain_pruning_seed() const { return m_db->get_blockchain_pruning_seed(); }
bool prune_blockchain(uint32_t pruning_seed = 0);
bool update_blockchain_pruning();
bool check_blockchain_pruning();
void lock();
void unlock();

47
src/cryptonote_core/cryptonote_core.cpp
View file

@ -105,6 +105,11 @@ namespace cryptonote
"disable-dns-checkpoints"
, "Do not retrieve checkpoints from DNS"
};
const command_line::arg_descriptor<size_t> arg_block_download_max_size = {
"block-download-max-size"
, "Set maximum size of block download queue in bytes (0 for default)"
, 0
};
static const command_line::arg_descriptor<bool> arg_test_drop_download = {
"test-drop-download"
@ -175,6 +180,11 @@ namespace cryptonote
, "Run a program for each new block, '%s' will be replaced by the block hash"
, ""
};
static const command_line::arg_descriptor<bool> arg_prune_blockchain = {
"prune-blockchain"
, "Prune blockchain"
, false
};
//-----------------------------------------------------------------------------------------------
core::core(i_cryptonote_protocol* pprotocol):
@ -285,9 +295,11 @@ namespace cryptonote
command_line::add_arg(desc, arg_test_dbg_lock_sleep);
command_line::add_arg(desc, arg_offline);
command_line::add_arg(desc, arg_disable_dns_checkpoints);
command_line::add_arg(desc, arg_block_download_max_size);
command_line::add_arg(desc, arg_max_txpool_weight);
command_line::add_arg(desc, arg_pad_transactions);
command_line::add_arg(desc, arg_block_notify);
command_line::add_arg(desc, arg_prune_blockchain);
miner::init_options(desc);
BlockchainDB::init_options(desc);
@ -374,6 +386,11 @@ namespace cryptonote
return m_blockchain_storage.get_transactions_blobs(txs_ids, txs, missed_txs);
}
//-----------------------------------------------------------------------------------------------
bool core::get_split_transactions_blobs(const std::vector<crypto::hash>& txs_ids, std::vector<std::tuple<crypto::hash, cryptonote::blobdata, crypto::hash, cryptonote::blobdata>>& txs, std::vector<crypto::hash>& missed_txs) const
{
return m_blockchain_storage.get_split_transactions_blobs(txs_ids, txs, missed_txs);
}
//-----------------------------------------------------------------------------------------------
bool core::get_txpool_backlog(std::vector<tx_backlog_entry>& backlog) const
{
m_mempool.get_transaction_backlog(backlog);
@ -413,6 +430,7 @@ namespace cryptonote
uint64_t blocks_threads = command_line::get_arg(vm, arg_prep_blocks_threads);
std::string check_updates_string = command_line::get_arg(vm, arg_check_updates);
size_t max_txpool_weight = command_line::get_arg(vm, arg_max_txpool_weight);
bool prune_blockchain = command_line::get_arg(vm, arg_prune_blockchain);
boost::filesystem::path folder(m_config_folder);
if (m_nettype == FAKECHAIN)
@ -607,6 +625,14 @@ namespace cryptonote
r = m_miner.init(vm, m_nettype);
CHECK_AND_ASSERT_MES(r, false, "Failed to initialize miner instance");
if (prune_blockchain)
{
// display a message if the blockchain is not pruned yet
if (m_blockchain_storage.get_current_blockchain_height() > 1 && !m_blockchain_storage.get_blockchain_pruning_seed())
MGINFO("Pruning blockchain...");
CHECK_AND_ASSERT_MES(m_blockchain_storage.prune_blockchain(), false, "Failed to prune blockchain");
}
return load_state_data();
}
//-----------------------------------------------------------------------------------------------
@ -1501,6 +1527,7 @@ namespace cryptonote
m_check_updates_interval.do_call(boost::bind(&core::check_updates, this));
m_check_disk_space_interval.do_call(boost::bind(&core::check_disk_space, this));
m_block_rate_interval.do_call(boost::bind(&core::check_block_rate, this));
m_blockchain_pruning_interval.do_call(boost::bind(&core::update_blockchain_pruning, this));
m_miner.on_idle();
m_mempool.on_idle();
return true;
@ -1736,6 +1763,16 @@ namespace cryptonote
return true;
}
//-----------------------------------------------------------------------------------------------
bool core::update_blockchain_pruning()
{
return m_blockchain_storage.update_blockchain_pruning();
}
//-----------------------------------------------------------------------------------------------
bool core::check_blockchain_pruning()
{
return m_blockchain_storage.check_blockchain_pruning();
}
//-----------------------------------------------------------------------------------------------
void core::set_target_blockchain_height(uint64_t target_blockchain_height)
{
m_target_blockchain_height = target_blockchain_height;
@ -1758,6 +1795,16 @@ namespace cryptonote
return si.available;
}
//-----------------------------------------------------------------------------------------------
uint32_t core::get_blockchain_pruning_seed() const
{
return get_blockchain_storage().get_blockchain_pruning_seed();
}
//-----------------------------------------------------------------------------------------------
bool core::prune_blockchain(uint32_t pruning_seed)
{
return get_blockchain_storage().prune_blockchain(pruning_seed);
}
//-----------------------------------------------------------------------------------------------
std::time_t core::get_start_time() const
{
return start_time;

39
src/cryptonote_core/cryptonote_core.h
View file

@ -62,6 +62,7 @@ namespace cryptonote
extern const command_line::arg_descriptor<bool, false> arg_regtest_on;
extern const command_line::arg_descriptor<difficulty_type> arg_fixed_difficulty;
extern const command_line::arg_descriptor<bool> arg_offline;
extern const command_line::arg_descriptor<size_t> arg_block_download_max_size;
/************************************************************************/
/* */
@ -354,6 +355,13 @@ namespace cryptonote
*/
bool get_transactions(const std::vector<crypto::hash>& txs_ids, std::vector<cryptonote::blobdata>& txs, std::vector<crypto::hash>& missed_txs) const;
/**
* @copydoc Blockchain::get_transactions
*
* @note see Blockchain::get_transactions
*/
bool get_split_transactions_blobs(const std::vector<crypto::hash>& txs_ids, std::vector<std::tuple<crypto::hash, cryptonote::blobdata, crypto::hash, cryptonote::blobdata>>& txs, std::vector<crypto::hash>& missed_txs) const;
/**
* @copydoc Blockchain::get_transactions
*
@ -783,6 +791,36 @@ namespace cryptonote
*/
bool offline() const { return m_offline; }
/**
* @brief get the blockchain pruning seed
*
* @return the blockchain pruning seed
*/
uint32_t get_blockchain_pruning_seed() const;
/**
* @brief prune the blockchain
*
* @param pruning_seed the seed to use to prune the chain (0 for default, highly recommended)
*
* @return true iff success
*/
bool prune_blockchain(uint32_t pruning_seed = 0);
/**
* @brief incrementally prunes blockchain
*
* @return true on success, false otherwise
*/
bool update_blockchain_pruning();
/**
* @brief checks the blockchain pruning if enabled
*
* @return true on success, false otherwise
*/
bool check_blockchain_pruning();
private:
/**
@ -985,6 +1023,7 @@ namespace cryptonote
epee::math_helper::once_a_time_seconds<60*60*12, true> m_check_updates_interval; //!< interval for checking for new versions
epee::math_helper::once_a_time_seconds<60*10, true> m_check_disk_space_interval; //!< interval for checking for disk space
epee::math_helper::once_a_time_seconds<90, false> m_block_rate_interval; //!< interval for checking block rate
epee::math_helper::once_a_time_seconds<60*60*5, true> m_blockchain_pruning_interval; //!< interval for incremental blockchain pruning
std::atomic<bool> m_starter_message_showed; //!< has the "daemon will sync now" message been shown?

193
src/cryptonote_protocol/block_queue.cpp
View file

@ -34,6 +34,7 @@
#include <boost/uuid/uuid_io.hpp>
#include "string_tools.h"
#include "cryptonote_protocol_defs.h"
#include "common/pruning.h"
#include "block_queue.h"
#undef MONERO_DEFAULT_LOG_CATEGORY
@ -60,7 +61,10 @@ void block_queue::add_blocks(uint64_t height, std::vector<cryptonote::block_comp
if (has_hashes)
{
for (const crypto::hash &h: hashes)
{
requested_hashes.insert(h);
have_blocks.insert(h);
}
set_span_hashes(height, connection_id, hashes);
}
}
@ -90,7 +94,10 @@ void block_queue::erase_block(block_map::iterator j)
{
CHECK_AND_ASSERT_THROW_MES(j != blocks.end(), "Invalid iterator");
for (const crypto::hash &h: j->hashes)
{
requested_hashes.erase(h);
have_blocks.erase(h);
}
blocks.erase(j);
}
@ -98,12 +105,10 @@ void block_queue::flush_stale_spans(const std::set<boost::uuids::uuid> &live_con
{
boost::unique_lock<boost::recursive_mutex> lock(mutex);
block_map::iterator i = blocks.begin();
if (i != blocks.end() && is_blockchain_placeholder(*i))
++i;
while (i != blocks.end())
{
block_map::iterator j = i++;
if (live_connections.find(j->connection_id) == live_connections.end() && j->blocks.size() == 0)
if (j->blocks.empty() && live_connections.find(j->connection_id) == live_connections.end())
{
erase_block(j);
}
@ -152,23 +157,56 @@ uint64_t block_queue::get_max_block_height() const
return height;
}
uint64_t block_queue::get_next_needed_height(uint64_t blockchain_height) const
{
boost::unique_lock<boost::recursive_mutex> lock(mutex);
if (blocks.empty())
return blockchain_height;
uint64_t last_needed_height = blockchain_height;
bool first = true;
for (const auto &span: blocks)
{
if (span.start_block_height + span.nblocks - 1 < blockchain_height)
continue;
if (span.start_block_height != last_needed_height || (first && span.blocks.empty()))
return last_needed_height;
last_needed_height = span.start_block_height + span.nblocks;
first = false;
}
return last_needed_height;
}
void block_queue::print() const
{
boost::unique_lock<boost::recursive_mutex> lock(mutex);
MDEBUG("Block queue has " << blocks.size() << " spans");
for (const auto &span: blocks)
MDEBUG(" " << span.start_block_height << " - " << (span.start_block_height+span.nblocks-1) << " (" << span.nblocks << ") - " << (is_blockchain_placeholder(span) ? "blockchain" : span.blocks.empty() ? "scheduled" : "filled ") << " " << span.connection_id << " (" << ((unsigned)(span.rate*10/1024.f))/10.f << " kB/s)");
MDEBUG(" " << span.start_block_height << " - " << (span.start_block_height+span.nblocks-1) << " (" << span.nblocks << ") - " << (span.blocks.empty() ? "scheduled" : "filled ") << " " << span.connection_id << " (" << ((unsigned)(span.rate*10/1024.f))/10.f << " kB/s)");
}
std::string block_queue::get_overview() const
std::string block_queue::get_overview(uint64_t blockchain_height) const
{
boost::unique_lock<boost::recursive_mutex> lock(mutex);
if (blocks.empty())
return "[]";
block_map::const_iterator i = blocks.begin();
std::string s = std::string("[") + std::to_string(i->start_block_height + i->nblocks - 1) + ":";
while (++i != blocks.end())
s += i->blocks.empty() ? "." : "o";
std::string s = std::string("[");
uint64_t expected = blockchain_height;
while (i != blocks.end())
{
if (expected > i->start_block_height)
{
s += "<";
}
else
{
if (expected < i->start_block_height)
s += std::string(std::max((uint64_t)1, (i->start_block_height - expected) / (i->nblocks ? i->nblocks : 1)), '_');
s += i->blocks.empty() ? "." : i->start_block_height == blockchain_height ? "m" : "o";
expected = i->start_block_height + i->nblocks;
}
++i;
}
s += "]";
return s;
}
@ -184,16 +222,31 @@ bool block_queue::requested(const crypto::hash &hash) const
return requested_internal(hash);
}
std::pair<uint64_t, uint64_t> block_queue::reserve_span(uint64_t first_block_height, uint64_t last_block_height, uint64_t max_blocks, const boost::uuids::uuid &connection_id, const std::vector<crypto::hash> &block_hashes, boost::posix_time::ptime time)
bool block_queue::have(const crypto::hash &hash) const
{
boost::unique_lock<boost::recursive_mutex> lock(mutex);
return have_blocks.find(hash) != have_blocks.end();
}
std::pair<uint64_t, uint64_t> block_queue::reserve_span(uint64_t first_block_height, uint64_t last_block_height, uint64_t max_blocks, const boost::uuids::uuid &connection_id, uint32_t pruning_seed, uint64_t blockchain_height, const std::vector<crypto::hash> &block_hashes, boost::posix_time::ptime time)
{
boost::unique_lock<boost::recursive_mutex> lock(mutex);
MDEBUG("reserve_span: first_block_height " << first_block_height << ", last_block_height " << last_block_height
<< ", max " << max_blocks << ", seed " << epee::string_tools::to_string_hex(pruning_seed) << ", blockchain_height " <<
blockchain_height << ", block hashes size " << block_hashes.size());
if (last_block_height < first_block_height || max_blocks == 0)
{
MDEBUG("reserve_span: early out: first_block_height " << first_block_height << ", last_block_height " << last_block_height << ", max_blocks " << max_blocks);
return std::make_pair(0, 0);
}
if (block_hashes.size() >= last_block_height)
{
MDEBUG("reserve_span: more block hashes than fit within last_block_height: " << block_hashes.size() << " and " << last_block_height);
return std::make_pair(0, 0);
}
// skip everything we've already requested
uint64_t span_start_height = last_block_height - block_hashes.size() + 1;
std::vector<crypto::hash>::const_iterator i = block_hashes.begin();
while (i != block_hashes.end() && requested_internal(*i))
@ -201,55 +254,57 @@ std::pair<uint64_t, uint64_t> block_queue::reserve_span(uint64_t first_block_hei
++i;
++span_start_height;
}
// if the peer's pruned for the starting block and its unpruned stripe comes next, start downloading from there
const uint32_t next_unpruned_height = tools::get_next_unpruned_block_height(span_start_height, blockchain_height, pruning_seed);
MDEBUG("reserve_span: next_unpruned_height " << next_unpruned_height << " from " << span_start_height << " and seed "
<< epee::string_tools::to_string_hex(pruning_seed) << ", limit " << span_start_height + CRYPTONOTE_PRUNING_STRIPE_SIZE);
if (next_unpruned_height > span_start_height && next_unpruned_height < span_start_height + CRYPTONOTE_PRUNING_STRIPE_SIZE)
{
MDEBUG("We can download from next span: ideal height " << span_start_height << ", next unpruned height " << next_unpruned_height <<
"(+" << next_unpruned_height - span_start_height << "), current seed " << pruning_seed);
span_start_height = next_unpruned_height;
}
MDEBUG("span_start_height: " <<span_start_height);
const uint64_t block_hashes_start_height = last_block_height - block_hashes.size() + 1;
if (span_start_height >= block_hashes.size() + block_hashes_start_height)
{
MDEBUG("Out of hashes, cannot reserve");
return std::make_pair(0, 0);
}
i = block_hashes.begin() + span_start_height - block_hashes_start_height;
while (i != block_hashes.end() && requested_internal(*i))
{
++i;
++span_start_height;
}
uint64_t span_length = 0;
std::vector<crypto::hash> hashes;
while (i != block_hashes.end() && span_length < max_blocks)
while (i != block_hashes.end() && span_length < max_blocks && tools::has_unpruned_block(span_start_height + span_length, blockchain_height, pruning_seed))
{
hashes.push_back(*i);
++i;
++span_length;
}
if (span_length == 0)
{
MDEBUG("span_length 0, cannot reserve");
return std::make_pair(0, 0);
}
MDEBUG("Reserving span " << span_start_height << " - " << (span_start_height + span_length - 1) << " for " << connection_id);
add_blocks(span_start_height, span_length, connection_id, time);
set_span_hashes(span_start_height, connection_id, hashes);
return std::make_pair(span_start_height, span_length);
}
bool block_queue::is_blockchain_placeholder(const span &span) const
{
return span.connection_id == boost::uuids::nil_uuid();
}
std::pair<uint64_t, uint64_t> block_queue::get_start_gap_span() const
{
boost::unique_lock<boost::recursive_mutex> lock(mutex);
if (blocks.empty())
return std::make_pair(0, 0);
block_map::const_iterator i = blocks.begin();
if (!is_blockchain_placeholder(*i))
return std::make_pair(0, 0);
uint64_t current_height = i->start_block_height + i->nblocks - 1;
++i;
if (i == blocks.end())
return std::make_pair(0, 0);
uint64_t first_span_height = i->start_block_height;
if (first_span_height <= current_height + 1)
return std::make_pair(0, 0);
MDEBUG("Found gap at start of spans: last blockchain block height " << current_height << ", first span's block height " << first_span_height);
print();
return std::make_pair(current_height + 1, first_span_height - current_height - 1);
}
std::pair<uint64_t, uint64_t> block_queue::get_next_span_if_scheduled(std::vector<crypto::hash> &hashes, boost::uuids::uuid &connection_id, boost::posix_time::ptime &time) const
{
boost::unique_lock<boost::recursive_mutex> lock(mutex);
if (blocks.empty())
return std::make_pair(0, 0);
block_map::const_iterator i = blocks.begin();
if (is_blockchain_placeholder(*i))
++i;
if (i == blocks.end())
return std::make_pair(0, 0);
if (!i->blocks.empty())
@ -260,6 +315,16 @@ std::pair<uint64_t, uint64_t> block_queue::get_next_span_if_scheduled(std::vecto
return std::make_pair(i->start_block_height, i->nblocks);
}
void block_queue::reset_next_span_time(boost::posix_time::ptime t)
{
boost::unique_lock<boost::recursive_mutex> lock(mutex);
CHECK_AND_ASSERT_THROW_MES(!blocks.empty(), "No next span to reset time");
block_map::iterator i = blocks.begin();
CHECK_AND_ASSERT_THROW_MES(i != blocks.end(), "No next span to reset time");
CHECK_AND_ASSERT_THROW_MES(i->blocks.empty(), "Next span is not empty");
(boost::posix_time::ptime&)i->time = t; // sod off, time doesn't influence sorting
}
void block_queue::set_span_hashes(uint64_t start_height, const boost::uuids::uuid &connection_id, std::vector<crypto::hash> hashes)
{
boost::unique_lock<boost::recursive_mutex> lock(mutex);
@ -284,8 +349,6 @@ bool block_queue::get_next_span(uint64_t &height, std::vector<cryptonote::block_
if (blocks.empty())
return false;
block_map::const_iterator i = blocks.begin();
if (is_blockchain_placeholder(*i))
++i;
for (; i != blocks.end(); ++i)
{
if (!filled || !i->blocks.empty())
@ -299,19 +362,34 @@ bool block_queue::get_next_span(uint64_t &height, std::vector<cryptonote::block_
return false;
}
bool block_queue::has_next_span(const boost::uuids::uuid &connection_id, bool &filled) const
bool block_queue::has_next_span(const boost::uuids::uuid &connection_id, bool &filled, boost::posix_time::ptime &time) const
{
boost::unique_lock<boost::recursive_mutex> lock(mutex);
if (blocks.empty())
return false;
block_map::const_iterator i = blocks.begin();
if (is_blockchain_placeholder(*i))
++i;
if (i == blocks.end())
return false;
if (i->connection_id != connection_id)
return false;
filled = !i->blocks.empty();
time = i->time;
return true;
}
bool block_queue::has_next_span(uint64_t height, bool &filled, boost::posix_time::ptime &time, boost::uuids::uuid &connection_id) const
{
boost::unique_lock<boost::recursive_mutex> lock(mutex);
if (blocks.empty())
return false;
block_map::const_iterator i = blocks.begin();
if (i == blocks.end())
return false;
if (i->start_block_height > height)
return false;
filled = !i->blocks.empty();
time = i->time;
connection_id = i->connection_id;
return true;
}
@ -331,8 +409,6 @@ size_t block_queue::get_num_filled_spans_prefix() const
if (blocks.empty())
return 0;
block_map::const_iterator i = blocks.begin();
if (is_blockchain_placeholder(*i))
++i;
size_t size = 0;
while (i != blocks.end() && !i->blocks.empty())
{
@ -417,12 +493,35 @@ float block_queue::get_speed(const boost::uuids::uuid &connection_id) const
return speed;
}
bool block_queue::foreach(std::function<bool(const span&)> f, bool include_blockchain_placeholder) const
float block_queue::get_download_rate(const boost::uuids::uuid &connection_id) const
{
boost::unique_lock<boost::recursive_mutex> lock(mutex);
float conn_rate = -1.f;
for (const auto &span: blocks)
{
if (span.blocks.empty())
continue;
if (span.connection_id != connection_id)
continue;
// note that the average below does not average over the whole set, but over the
// previous pseudo average and the latest rate: this gives much more importance
// to the latest measurements, which is fine here
if (conn_rate < 0.f)
conn_rate = span.rate;
else
conn_rate = (conn_rate + span.rate) / 2;
}
if (conn_rate < 0)
conn_rate = 0.0f;
MTRACE("Download rate for " << connection_id << ": " << conn_rate << " b/s");
return conn_rate;
}
bool block_queue::foreach(std::function<bool(const span&)> f) const
{
boost::unique_lock<boost::recursive_mutex> lock(mutex);
block_map::const_iterator i = blocks.begin();
if (!include_blockchain_placeholder && i != blocks.end() && is_blockchain_placeholder(*i))
++i;
while (i != blocks.end())
if (!f(*i++))
return false;

17
src/cryptonote_protocol/block_queue.h
View file

@ -76,22 +76,26 @@ namespace cryptonote
void remove_spans(const boost::uuids::uuid &connection_id, uint64_t start_block_height);
uint64_t get_max_block_height() const;
void print() const;
std::string get_overview() const;
std::pair<uint64_t, uint64_t> reserve_span(uint64_t first_block_height, uint64_t last_block_height, uint64_t max_blocks, const boost::uuids::uuid &connection_id, const std::vector<crypto::hash> &block_hashes, boost::posix_time::ptime time = boost::posix_time::microsec_clock::universal_time());
bool is_blockchain_placeholder(const span &span) const;
std::pair<uint64_t, uint64_t> get_start_gap_span() const;
std::string get_overview(uint64_t blockchain_height) const;
bool has_unpruned_height(uint64_t block_height, uint64_t blockchain_height, uint32_t pruning_seed) const;
std::pair<uint64_t, uint64_t> reserve_span(uint64_t first_block_height, uint64_t last_block_height, uint64_t max_blocks, const boost::uuids::uuid &connection_id, uint32_t pruning_seed, uint64_t blockchain_height, const std::vector<crypto::hash> &block_hashes, boost::posix_time::ptime time = boost::posix_time::microsec_clock::universal_time());
uint64_t get_next_needed_height(uint64_t blockchain_height) const;
std::pair<uint64_t, uint64_t> get_next_span_if_scheduled(std::vector<crypto::hash> &hashes, boost::uuids::uuid &connection_id, boost::posix_time::ptime &time) const;
void reset_next_span_time(boost::posix_time::ptime t = boost::posix_time::microsec_clock::universal_time());
void set_span_hashes(uint64_t start_height, const boost::uuids::uuid &connection_id, std::vector<crypto::hash> hashes);
bool get_next_span(uint64_t &height, std::vector<cryptonote::block_complete_entry> &bcel, boost::uuids::uuid &connection_id, bool filled = true) const;
bool has_next_span(const boost::uuids::uuid &connection_id, bool &filled) const;
bool has_next_span(const boost::uuids::uuid &connection_id, bool &filled, boost::posix_time::ptime &time) const;
bool has_next_span(uint64_t height, bool &filled, boost::posix_time::ptime &time, boost::uuids::uuid &connection_id) const;
size_t get_data_size() const;
size_t get_num_filled_spans_prefix() const;
size_t get_num_filled_spans() const;
crypto::hash get_last_known_hash(const boost::uuids::uuid &connection_id) const;
bool has_spans(const boost::uuids::uuid &connection_id) const;
float get_speed(const boost::uuids::uuid &connection_id) const;
bool foreach(std::function<bool(const span&)> f, bool include_blockchain_placeholder = false) const;
float get_download_rate(const boost::uuids::uuid &connection_id) const;
bool foreach(std::function<bool(const span&)> f) const;
bool requested(const crypto::hash &hash) const;
bool have(const crypto::hash &hash) const;
private:
void erase_block(block_map::iterator j);
@ -101,5 +105,6 @@ namespace cryptonote
block_map blocks;
mutable boost::recursive_mutex mutex;
std::unordered_set<crypto::hash> requested_hashes;
std::unordered_set<crypto::hash> have_blocks;
};
}

5
src/cryptonote_protocol/cryptonote_protocol_defs.h
View file

@ -78,6 +78,8 @@ namespace cryptonote
uint64_t height;
uint32_t pruning_seed;
BEGIN_KV_SERIALIZE_MAP()
KV_SERIALIZE(incoming)
KV_SERIALIZE(localhost)
@ -100,6 +102,7 @@ namespace cryptonote
KV_SERIALIZE(support_flags)
KV_SERIALIZE(connection_id)
KV_SERIALIZE(height)
KV_SERIALIZE(pruning_seed)
END_KV_SERIALIZE_MAP()
};
@ -200,12 +203,14 @@ namespace cryptonote
uint64_t cumulative_difficulty;
crypto::hash top_id;
uint8_t top_version;
uint32_t pruning_seed;
BEGIN_KV_SERIALIZE_MAP()
KV_SERIALIZE(current_height)
KV_SERIALIZE(cumulative_difficulty)
KV_SERIALIZE_VAL_POD_AS_BLOB(top_id)
KV_SERIALIZE_OPT(top_version, (uint8_t)0)
KV_SERIALIZE_OPT(pruning_seed, (uint32_t)0)
END_KV_SERIALIZE_MAP()
};

16
src/cryptonote_protocol/cryptonote_protocol_handler.h
View file

@ -43,6 +43,7 @@
#include "cryptonote_protocol_defs.h"
#include "cryptonote_protocol_handler_common.h"
#include "block_queue.h"
#include "common/perf_timer.h"
#include "cryptonote_basic/connection_context.h"
#include "cryptonote_basic/cryptonote_stat_info.h"
#include <boost/circular_buffer.hpp>
@ -109,6 +110,10 @@ namespace cryptonote
const block_queue &get_block_queue() const { return m_block_queue; }
void stop();
void on_connection_close(cryptonote_connection_context &context);
void set_max_out_peers(unsigned int max) { m_max_out_peers = max; }
std::string get_peers_overview() const;
std::pair<uint32_t, uint32_t> get_next_needed_pruning_stripe() const;
bool needs_new_sync_connections() const;
private:
//----------------- commands handlers ----------------------------------------------
int handle_notify_new_block(int command, NOTIFY_NEW_BLOCK::request& arg, cryptonote_connection_context& context);
@ -125,14 +130,17 @@ namespace cryptonote
virtual bool relay_transactions(NOTIFY_NEW_TRANSACTIONS::request& arg, cryptonote_connection_context& exclude_context);
//----------------------------------------------------------------------------------
//bool get_payload_sync_data(HANDSHAKE_DATA::request& hshd, cryptonote_connection_context& context);
bool should_drop_connection(cryptonote_connection_context& context, uint32_t next_stripe);
bool request_missing_objects(cryptonote_connection_context& context, bool check_having_blocks, bool force_next_span = false);
size_t get_synchronizing_connections_count();
bool on_connection_synchronized();
bool should_download_next_span(cryptonote_connection_context& context) const;
bool should_download_next_span(cryptonote_connection_context& context, bool standby);
void drop_connection(cryptonote_connection_context &context, bool add_fail, bool flush_all_spans);
bool kick_idle_peers();
bool check_standby_peers();
int try_add_next_blocks(cryptonote_connection_context &context);
void notify_new_stripe(cryptonote_connection_context &context, uint32_t stripe);
void skip_unneeded_hashes(cryptonote_connection_context& context, bool check_block_queue) const;
t_core& m_core;
@ -145,6 +153,12 @@ namespace cryptonote
block_queue m_block_queue;
epee::math_helper::once_a_time_seconds<30> m_idle_peer_kicker;
epee::math_helper::once_a_time_milliseconds<100> m_standby_checker;
std::atomic<unsigned int> m_max_out_peers;
tools::PerformanceTimer m_sync_timer, m_add_timer;
uint64_t m_last_add_end_time;
uint64_t m_sync_spans_downloaded, m_sync_old_spans_downloaded, m_sync_bad_spans_downloaded;
uint64_t m_sync_download_chain_size, m_sync_download_objects_size;
size_t m_block_download_max_size;
boost::mutex m_buffer_mutex;
double get_avg_block_size();

802
src/cryptonote_protocol/cryptonote_protocol_handler.inl
View file

File diff suppressed because it is too large Load diff

23
src/daemon/command_parser_executor.cpp
View file

@ -717,4 +717,27 @@ bool t_command_parser_executor::version(const std::vector<std::string>& args)
return true;
}
bool t_command_parser_executor::prune_blockchain(const std::vector<std::string>& args)
{
if (args.size() > 1) return false;
if (args.empty() || args[0] != "confirm")
{
std::cout << "Warning: pruning from within monerod will not shrink the database file size." << std::endl;
std::cout << "Instead, parts of the file will be marked as free, so the file will not grow" << std::endl;
std::cout << "until that newly free space is used up. If you want a smaller file size now," << std::endl;
std::cout << "exit monerod and run monero-blockchain-prune (you will temporarily need more" << std::endl;
std::cout << "disk space for the database conversion though). If you are OK with the database" << std::endl;
std::cout << "file keeping the same size, re-run this command with the \"confirm\" parameter." << std::endl;
return true;
}
return m_executor.prune_blockchain();
}
bool t_command_parser_executor::check_blockchain_pruning(const std::vector<std::string>& args)
{
return m_executor.check_blockchain_pruning();
}
} // namespace daemonize

4
src/daemon/command_parser_executor.h
View file

@ -142,6 +142,10 @@ public:
bool pop_blocks(const std::vector<std::string>& args);
bool version(const std::vector<std::string>& args);
bool prune_blockchain(const std::vector<std::string>& args);
bool check_blockchain_pruning(const std::vector<std::string>& args);
};
} // namespace daemonize

10
src/daemon/command_server.cpp
View file

@ -292,6 +292,16 @@ t_command_server::t_command_server(
, std::bind(&t_command_parser_executor::version, &m_parser, p::_1)
, "Print version information."
);
m_command_lookup.set_handler(
"prune_blockchain"
, std::bind(&t_command_parser_executor::prune_blockchain, &m_parser, p::_1)
, "Prune the blockchain."
);
m_command_lookup.set_handler(
"check_blockchain_pruning"
, std::bind(&t_command_parser_executor::check_blockchain_pruning, &m_parser, p::_1)
, "Check the blockchain pruning."
);
}
bool t_command_server::process_command_str(const std::string& cmd)

123
src/daemon/rpc_command_executor.cpp
View file

@ -31,6 +31,7 @@
#include "string_tools.h"
#include "common/password.h"
#include "common/scoped_message_writer.h"
#include "common/pruning.h"
#include "daemon/rpc_command_executor.h"
#include "rpc/core_rpc_server_commands_defs.h"
#include "cryptonote_core/cryptonote_core.h"
@ -60,7 +61,8 @@ namespace {
peer_id_str >> id_str;
epee::string_tools::xtype_to_string(peer.port, port_str);
std::string addr_str = ip_str + ":" + port_str;
tools::msg_writer() << boost::format("%-10s %-25s %-25s %s") % prefix % id_str % addr_str % elapsed;
std::string pruning_seed = epee::string_tools::to_string_hex(peer.pruning_seed);
tools::msg_writer() << boost::format("%-10s %-25s %-25s %-4s %s") % prefix % id_str % addr_str % pruning_seed % elapsed;
}
void print_block_header(cryptonote::block_header_response const & header)
@ -741,6 +743,7 @@ bool t_rpc_command_executor::print_transaction(crypto::hash transaction_hash,
req.txs_hashes.push_back(epee::string_tools::pod_to_hex(transaction_hash));
req.decode_as_json = false;
req.split = true;
req.prune = false;
if (m_is_rpc)
{
@ -766,13 +769,25 @@ bool t_rpc_command_executor::print_transaction(crypto::hash transaction_hash,
if (res.txs.front().in_pool)
tools::success_msg_writer() << "Found in pool";
else
tools::success_msg_writer() << "Found in blockchain at height " << res.txs.front().block_height;
tools::success_msg_writer() << "Found in blockchain at height " << res.txs.front().block_height << (res.txs.front().prunable_as_hex.empty() ? " (pruned)" : "");
}
const std::string &as_hex = (1 == res.txs.size()) ? res.txs.front().as_hex : res.txs_as_hex.front();
const std::string &pruned_as_hex = (1 == res.txs.size()) ? res.txs.front().pruned_as_hex : "";
const std::string &prunable_as_hex = (1 == res.txs.size()) ? res.txs.front().prunable_as_hex : "";
// Print raw hex if requested
if (include_hex)
tools::success_msg_writer() << as_hex << std::endl;
{
if (!as_hex.empty())
{
tools::success_msg_writer() << as_hex << std::endl;
}
else
{
std::string output = pruned_as_hex + prunable_as_hex;
tools::success_msg_writer() << output << std::endl;
}
}
// Print json if requested
if (include_json)
@ -780,17 +795,27 @@ bool t_rpc_command_executor::print_transaction(crypto::hash transaction_hash,
crypto::hash tx_hash, tx_prefix_hash;
cryptonote::transaction tx;
cryptonote::blobdata blob;
if (!string_tools::parse_hexstr_to_binbuff(as_hex, blob))
std::string source = as_hex.empty() ? pruned_as_hex + prunable_as_hex : as_hex;
bool pruned = !pruned_as_hex.empty() && prunable_as_hex.empty();
if (!string_tools::parse_hexstr_to_binbuff(source, blob))
{
tools::fail_msg_writer() << "Failed to parse tx to get json format";
}
else if (!cryptonote::parse_and_validate_tx_from_blob(blob, tx, tx_hash, tx_prefix_hash))
{
tools::fail_msg_writer() << "Failed to parse tx blob to get json format";
}
else
{
tools::success_msg_writer() << cryptonote::obj_to_json_str(tx) << std::endl;
bool ret;
if (pruned)
ret = cryptonote::parse_and_validate_tx_base_from_blob(blob, tx);
else
ret = cryptonote::parse_and_validate_tx_from_blob(blob, tx);
if (!ret)
{
tools::fail_msg_writer() << "Failed to parse tx blob to get json format";
}
else
{
tools::success_msg_writer() << cryptonote::obj_to_json_str(tx) << std::endl;
}
}
}
}
@ -1939,6 +1964,8 @@ bool t_rpc_command_executor::sync_info()
for (const auto &p: res.peers)
current_download += p.info.current_download;
tools::success_msg_writer() << "Downloading at " << current_download << " kB/s";
if (res.next_needed_pruning_seed)
tools::success_msg_writer() << "Next needed pruning seed: " << res.next_needed_pruning_seed;
tools::success_msg_writer() << std::to_string(res.peers.size()) << " peers";
for (const auto &p: res.peers)
@ -1946,25 +1973,30 @@ bool t_rpc_command_executor::sync_info()
std::string address = epee::string_tools::pad_string(p.info.address, 24);
uint64_t nblocks = 0, size = 0;
for (const auto &s: res.spans)
if (s.rate > 0.0f && s.connection_id == p.info.connection_id)
if (s.connection_id == p.info.connection_id)
nblocks += s.nblocks, size += s.size;
tools::success_msg_writer() << address << " " << epee::string_tools::pad_string(p.info.peer_id, 16, '0', true) << " " << epee::string_tools::pad_string(p.info.state, 16) << " " << p.info.height << " " << p.info.current_download << " kB/s, " << nblocks << " blocks / " << size/1e6 << " MB queued";
tools::success_msg_writer() << address << " " << epee::string_tools::pad_string(p.info.peer_id, 16, '0', true) << " " <<
epee::string_tools::pad_string(p.info.state, 16) << " " <<
epee::string_tools::pad_string(epee::string_tools::to_string_hex(p.info.pruning_seed), 8) << " " << p.info.height << " " <<
p.info.current_download << " kB/s, " << nblocks << " blocks / " << size/1e6 << " MB queued";
}
uint64_t total_size = 0;
for (const auto &s: res.spans)
total_size += s.size;
tools::success_msg_writer() << std::to_string(res.spans.size()) << " spans, " << total_size/1e6 << " MB";
tools::success_msg_writer() << res.overview;
for (const auto &s: res.spans)
{
std::string address = epee::string_tools::pad_string(s.remote_address, 24);
std::string pruning_seed = epee::string_tools::to_string_hex(tools::get_pruning_seed(s.start_block_height, std::numeric_limits<uint64_t>::max(), CRYPTONOTE_PRUNING_LOG_STRIPES));
if (s.size == 0)
{
tools::success_msg_writer() << address << " " << s.nblocks << " (" << s.start_block_height << " - " << (s.start_block_height + s.nblocks - 1) << ") -";
tools::success_msg_writer() << address << " " << s.nblocks << "/" << pruning_seed << " (" << s.start_block_height << " - " << (s.start_block_height + s.nblocks - 1) << ") -";
}
else
{
tools::success_msg_writer() << address << " " << s.nblocks << " (" << s.start_block_height << " - " << (s.start_block_height + s.nblocks - 1) << ", " << (uint64_t)(s.size/1e3) << " kB) " << (unsigned)(s.rate/1e3) << " kB/s (" << s.speed/100.0f << ")";
tools::success_msg_writer() << address << " " << s.nblocks << "/" << pruning_seed << " (" << s.start_block_height << " - " << (s.start_block_height + s.nblocks - 1) << ", " << (uint64_t)(s.size/1e3) << " kB) " << (unsigned)(s.rate/1e3) << " kB/s (" << s.speed/100.0f << ")";
}
}
@ -1998,4 +2030,69 @@ bool t_rpc_command_executor::pop_blocks(uint64_t num_blocks)
return true;
}
bool t_rpc_command_executor::prune_blockchain()
{
cryptonote::COMMAND_RPC_PRUNE_BLOCKCHAIN::request req;
cryptonote::COMMAND_RPC_PRUNE_BLOCKCHAIN::response res;
std::string fail_message = "Unsuccessful";
epee::json_rpc::error error_resp;
req.check = false;
if (m_is_rpc)
{
if (!m_rpc_client->json_rpc_request(req, res, "prune_blockchain", fail_message.c_str()))
{
return true;
}
}
else
{
if (!m_rpc_server->on_prune_blockchain(req, res, error_resp) || res.status != CORE_RPC_STATUS_OK)
{
tools::fail_msg_writer() << make_error(fail_message, res.status);
return true;
}
}
tools::success_msg_writer() << "Blockchain pruned: seed " << epee::string_tools::to_string_hex(res.pruning_seed);
return true;
}
bool t_rpc_command_executor::check_blockchain_pruning()
{
cryptonote::COMMAND_RPC_PRUNE_BLOCKCHAIN::request req;
cryptonote::COMMAND_RPC_PRUNE_BLOCKCHAIN::response res;
std::string fail_message = "Unsuccessful";
epee::json_rpc::error error_resp;
req.check = true;
if (m_is_rpc)
{
if (!m_rpc_client->json_rpc_request(req, res, "prune_blockchain", fail_message.c_str()))
{
return true;
}
}
else
{
if (!m_rpc_server->on_prune_blockchain(req, res, error_resp) || res.status != CORE_RPC_STATUS_OK)
{
tools::fail_msg_writer() << make_error(fail_message, res.status);
return true;
}
}
if (res.pruning_seed)
{
tools::success_msg_writer() << "Blockchain pruning checked";
}
else
{
tools::success_msg_writer() << "Blockchain is not pruned";
}
return true;
}
}// namespace daemonize

4
src/daemon/rpc_command_executor.h
View file

@ -154,6 +154,10 @@ public:
bool sync_info();
bool pop_blocks(uint64_t num_blocks);
bool prune_blockchain();
bool check_blockchain_pruning();
};
} // namespace daemonize

2
src/debug_utilities/cn_deserialize.cpp
View file

@ -180,11 +180,9 @@ int main(int argc, char* argv[])
}
else if (cryptonote::parse_and_validate_tx_from_blob(blob, tx) || cryptonote::parse_and_validate_tx_base_from_blob(blob, tx))
{
/*
if (tx.pruned)
std::cout << "Parsed pruned transaction:" << std::endl;
else
*/
std::cout << "Parsed transaction:" << std::endl;
std::cout << cryptonote::obj_to_json_str(tx) << std::endl;

15
src/p2p/net_node.h
View file

@ -29,6 +29,7 @@
// Parts of this file are originally copyright (c) 2012-2013 The Cryptonote developers
#pragma once
#include <array>
#include <boost/thread.hpp>
#include <boost/program_options/options_description.hpp>
#include <boost/program_options/variables_map.hpp>
@ -127,6 +128,11 @@ namespace nodetool
virtual bool block_host(const epee::net_utils::network_address &adress, time_t seconds = P2P_IP_BLOCKTIME);
virtual bool unblock_host(const epee::net_utils::network_address &address);
virtual std::map<std::string, time_t> get_blocked_hosts() { CRITICAL_REGION_LOCAL(m_blocked_hosts_lock); return m_blocked_hosts; }
virtual void add_used_stripe_peer(const typename t_payload_net_handler::connection_context &context);
virtual void remove_used_stripe_peer(const typename t_payload_net_handler::connection_context &context);
virtual void clear_used_stripe_peers();
private:
const std::vector<std::string> m_seed_nodes_list =
{ "seeds.moneroseeds.se"
@ -235,7 +241,13 @@ namespace nodetool
bool parse_peers_and_add_to_container(const boost::program_options::variables_map& vm, const command_line::arg_descriptor<std::vector<std::string> > & arg, Container& container);
bool set_max_out_peers(const boost::program_options::variables_map& vm, int64_t max);
bool get_max_out_peers() const { return m_config.m_net_config.max_out_connection_count; }
bool get_current_out_peers() const { return m_current_number_of_out_peers; }
bool set_max_in_peers(const boost::program_options::variables_map& vm, int64_t max);
bool get_max_in_peers() const { return m_config.m_net_config.max_in_connection_count; }
bool get_current_in_peers() const { return m_current_number_of_in_peers; }
bool set_tos_flag(const boost::program_options::variables_map& vm, int limit);
bool set_rate_up_limit(const boost::program_options::variables_map& vm, int64_t limit);
@ -336,6 +348,9 @@ namespace nodetool
epee::critical_section m_host_fails_score_lock;
std::map<std::string, uint64_t> m_host_fails_score;
boost::mutex m_used_stripe_peers_mutex;
std::array<std::list<epee::net_utils::network_address>, 1 << CRYPTONOTE_PRUNING_LOG_STRIPES> m_used_stripe_peers;
cryptonote::network_type m_nettype;
};

198
src/p2p/net_node.inl
View file

@ -41,6 +41,7 @@
#include "string_tools.h"
#include "common/util.h"
#include "common/dns_utils.h"
#include "common/pruning.h"
#include "net/net_helper.h"
#include "math_helper.h"
#include "p2p_protocol_defs.h"
@ -133,6 +134,28 @@ namespace nodetool
make_default_config();
}
#ifdef CRYPTONOTE_PRUNING_DEBUG_SPOOF_SEED
std::list<peerlist_entry> plw;
while (m_peerlist.get_white_peers_count())
{
plw.push_back(peerlist_entry());
m_peerlist.get_white_peer_by_index(plw.back(), 0);
m_peerlist.remove_from_peer_white(plw.back());
}
for (auto &e:plw)
m_peerlist.append_with_peer_white(e);
std::list<peerlist_entry> plg;
while (m_peerlist.get_gray_peers_count())
{
plg.push_back(peerlist_entry());
m_peerlist.get_gray_peer_by_index(plg.back(), 0);
m_peerlist.remove_from_peer_gray(plg.back());
}
for (auto &e:plg)
m_peerlist.append_with_peer_gray(e);
#endif
// always recreate a new peer id
make_default_peer_id();
@ -729,7 +752,7 @@ namespace nodetool
std::atomic<bool> hsh_result(false);
bool r = epee::net_utils::async_invoke_remote_command2<typename COMMAND_HANDSHAKE::response>(context_.m_connection_id, COMMAND_HANDSHAKE::ID, arg, m_net_server.get_config_object(),
[this, &pi, &ev, &hsh_result, &just_take_peerlist](int code, const typename COMMAND_HANDSHAKE::response& rsp, p2p_connection_context& context)
[this, &pi, &ev, &hsh_result, &just_take_peerlist, &context_](int code, const typename COMMAND_HANDSHAKE::response& rsp, p2p_connection_context& context)
{
epee::misc_utils::auto_scope_leave_caller scope_exit_handler = epee::misc_utils::create_scope_leave_handler([&](){ev.raise();});
@ -762,7 +785,7 @@ namespace nodetool
}
pi = context.peer_id = rsp.node_data.peer_id;
m_peerlist.set_peer_just_seen(rsp.node_data.peer_id, context.m_remote_address);
m_peerlist.set_peer_just_seen(rsp.node_data.peer_id, context.m_remote_address, context.m_pruning_seed);
if(rsp.node_data.peer_id == m_config.m_peer_id)
{
@ -770,11 +793,13 @@ namespace nodetool
hsh_result = false;
return;
}
LOG_INFO_CC(context, "New connection handshaked, pruning seed " << epee::string_tools::to_string_hex(context.m_pruning_seed));
LOG_DEBUG_CC(context, " COMMAND_HANDSHAKE INVOKED OK");
}else
{
LOG_DEBUG_CC(context, " COMMAND_HANDSHAKE(AND CLOSE) INVOKED OK");
}
context_ = context;
}, P2P_DEFAULT_HANDSHAKE_INVOKE_TIMEOUT);
if(r)
@ -821,7 +846,7 @@ namespace nodetool
add_host_fail(context.m_remote_address);
}
if(!context.m_is_income)
m_peerlist.set_peer_just_seen(context.peer_id, context.m_remote_address);
m_peerlist.set_peer_just_seen(context.peer_id, context.m_remote_address, context.m_pruning_seed);
m_payload_handler.process_payload_sync_data(rsp.payload_data, context, false);
});
@ -939,6 +964,7 @@ namespace nodetool
const epee::net_utils::ipv4_network_address &ipv4 = na.as<const epee::net_utils::ipv4_network_address>();
typename net_server::t_connection_context con = AUTO_VAL_INIT(con);
con.m_anchor = peer_type == anchor;
bool res = m_net_server.connect(epee::string_tools::get_ip_string_from_int32(ipv4.ip()),
epee::string_tools::num_to_string_fast(ipv4.port()),
m_config.m_net_config.connection_timeout,
@ -978,6 +1004,7 @@ namespace nodetool
time_t last_seen;
time(&last_seen);
pe_local.last_seen = static_cast<int64_t>(last_seen);
pe_local.pruning_seed = con.m_pruning_seed;
m_peerlist.append_with_peer_white(pe_local);
//update last seen and push it to peerlist manager
@ -1004,6 +1031,7 @@ namespace nodetool
const epee::net_utils::ipv4_network_address &ipv4 = na.as<epee::net_utils::ipv4_network_address>();
typename net_server::t_connection_context con = AUTO_VAL_INIT(con);
con.m_anchor = false;
bool res = m_net_server.connect(epee::string_tools::get_ip_string_from_int32(ipv4.ip()),
epee::string_tools::num_to_string_fast(ipv4.port()),
m_config.m_net_config.connection_timeout,
@ -1056,7 +1084,7 @@ namespace nodetool
bool node_server<t_payload_net_handler>::make_new_connection_from_anchor_peerlist(const std::vector<anchor_peerlist_entry>& anchor_peerlist)
{
for (const auto& pe: anchor_peerlist) {
_note("Considering connecting (out) to peer: " << peerid_type(pe.id) << " " << pe.adr.str());
_note("Considering connecting (out) to anchor peer: " << peerid_type(pe.id) << " " << pe.adr.str());
if(is_peer_used(pe)) {
_note("Peer is used");
@ -1089,11 +1117,7 @@ namespace nodetool
template<class t_payload_net_handler>
bool node_server<t_payload_net_handler>::make_new_connection_from_peerlist(bool use_white_list)
{
size_t local_peers_count = use_white_list ? m_peerlist.get_white_peers_count():m_peerlist.get_gray_peers_count();
if(!local_peers_count)
return false;//no peers
size_t max_random_index = std::min<uint64_t>(local_peers_count -1, 20);
size_t max_random_index = 0;
std::set<size_t> tried_peers;
@ -1103,21 +1127,54 @@ namespace nodetool
{
++rand_count;
size_t random_index;
const uint32_t next_needed_pruning_stripe = m_payload_handler.get_next_needed_pruning_stripe().second;
if (use_white_list) {
local_peers_count = m_peerlist.get_white_peers_count();
if (!local_peers_count)
std::deque<size_t> filtered;
const size_t limit = use_white_list ? 20 : std::numeric_limits<size_t>::max();
size_t idx = 0;
m_peerlist.foreach (use_white_list, [&filtered, &idx, limit, next_needed_pruning_stripe](const peerlist_entry &pe){
if (filtered.size() >= limit)
return false;
max_random_index = std::min<uint64_t>(local_peers_count -1, 20);
random_index = get_random_index_with_fixed_probability(max_random_index);
} else {
local_peers_count = m_peerlist.get_gray_peers_count();
if (!local_peers_count)
return false;
random_index = crypto::rand<size_t>() % local_peers_count;
if (next_needed_pruning_stripe == 0 || pe.pruning_seed == 0)
filtered.push_back(idx);
else if (next_needed_pruning_stripe == tools::get_pruning_stripe(pe.pruning_seed))
filtered.push_front(idx);
++idx;
return true;
});
if (filtered.empty())
{
MDEBUG("No available peer in " << (use_white_list ? "white" : "gray") << " list filtered by " << next_needed_pruning_stripe);
return false;
}
if (use_white_list)
{
// if using the white list, we first pick in the set of peers we've already been using earlier
random_index = get_random_index_with_fixed_probability(std::min<uint64_t>(filtered.size() - 1, 20));
CRITICAL_REGION_LOCAL(m_used_stripe_peers_mutex);
if (next_needed_pruning_stripe > 0 && next_needed_pruning_stripe <= (1ul << CRYPTONOTE_PRUNING_LOG_STRIPES) && !m_used_stripe_peers[next_needed_pruning_stripe-1].empty())
{
const epee::net_utils::network_address na = m_used_stripe_peers[next_needed_pruning_stripe-1].front();
m_used_stripe_peers[next_needed_pruning_stripe-1].pop_front();
for (size_t i = 0; i < filtered.size(); ++i)
{
peerlist_entry pe;
if (m_peerlist.get_white_peer_by_index(pe, filtered[i]) && pe.adr == na)
{
MDEBUG("Reusing stripe " << next_needed_pruning_stripe << " peer " << pe.adr.str());
random_index = i;
break;
}
}
}
}
else
random_index = crypto::rand<size_t>() % filtered.size();
CHECK_AND_ASSERT_MES(random_index < local_peers_count, false, "random_starter_index < peers_local.size() failed!!");
CHECK_AND_ASSERT_MES(random_index < filtered.size(), false, "random_index < filtered.size() failed!!");
random_index = filtered[random_index];
CHECK_AND_ASSERT_MES(random_index < (use_white_list ? m_peerlist.get_white_peers_count() : m_peerlist.get_gray_peers_count()),
false, "random_index < peers size failed!!");
if(tried_peers.count(random_index))
continue;
@ -1129,7 +1186,9 @@ namespace nodetool
++try_count;
_note("Considering connecting (out) to peer: " << peerid_to_string(pe.id) << " " << pe.adr.str());
_note("Considering connecting (out) to " << (use_white_list ? "white" : "gray") << " list peer: " <<
peerid_to_string(pe.id) << " " << pe.adr.str() << ", pruning seed " << epee::string_tools::to_string_hex(pe.pruning_seed) <<
" (stripe " << next_needed_pruning_stripe << " needed)");
if(is_peer_used(pe)) {
_note("Peer is used");
@ -1143,6 +1202,7 @@ namespace nodetool
continue;
MDEBUG("Selected peer: " << peerid_to_string(pe.id) << " " << pe.adr.str()
<< ", pruning seed " << epee::string_tools::to_string_hex(pe.pruning_seed) << " "
<< "[peer_list=" << (use_white_list ? white : gray)
<< "] last_seen: " << (pe.last_seen ? epee::misc_utils::get_time_interval_string(time(NULL) - pe.last_seen) : "never"));
@ -1219,31 +1279,35 @@ namespace nodetool
if (!connect_to_peerlist(m_priority_peers)) return false;
size_t expected_white_connections = (m_config.m_net_config.max_out_connection_count*P2P_DEFAULT_WHITELIST_CONNECTIONS_PERCENT)/100;
size_t base_expected_white_connections = (m_config.m_net_config.max_out_connection_count*P2P_DEFAULT_WHITELIST_CONNECTIONS_PERCENT)/100;
size_t conn_count = get_outgoing_connections_count();
if(conn_count < m_config.m_net_config.max_out_connection_count)
while(conn_count < m_config.m_net_config.max_out_connection_count)
{
const size_t expected_white_connections = m_payload_handler.get_next_needed_pruning_stripe().second ? m_config.m_net_config.max_out_connection_count : base_expected_white_connections;
if(conn_count < expected_white_connections)
{
//start from anchor list
if(!make_expected_connections_count(anchor, P2P_DEFAULT_ANCHOR_CONNECTIONS_COUNT))
return false;
while (get_outgoing_connections_count() < P2P_DEFAULT_ANCHOR_CONNECTIONS_COUNT
&& make_expected_connections_count(anchor, P2P_DEFAULT_ANCHOR_CONNECTIONS_COUNT));
//then do white list
if(!make_expected_connections_count(white, expected_white_connections))
return false;
while (get_outgoing_connections_count() < expected_white_connections
&& make_expected_connections_count(white, expected_white_connections));
//then do grey list
if(!make_expected_connections_count(gray, m_config.m_net_config.max_out_connection_count))
return false;
while (get_outgoing_connections_count() < m_config.m_net_config.max_out_connection_count
&& make_expected_connections_count(gray, m_config.m_net_config.max_out_connection_count));
}else
{
//start from grey list
if(!make_expected_connections_count(gray, m_config.m_net_config.max_out_connection_count))
return false;
while (get_outgoing_connections_count() < m_config.m_net_config.max_out_connection_count
&& make_expected_connections_count(gray, m_config.m_net_config.max_out_connection_count));
//and then do white list
if(!make_expected_connections_count(white, m_config.m_net_config.max_out_connection_count))
return false;
while (get_outgoing_connections_count() < m_config.m_net_config.max_out_connection_count
&& make_expected_connections_count(white, m_config.m_net_config.max_out_connection_count));
}
if(m_net_server.is_stop_signal_sent())
return false;
conn_count = get_outgoing_connections_count();
}
if (start_conn_count == get_outgoing_connections_count() && start_conn_count < m_config.m_net_config.max_out_connection_count)
@ -1260,7 +1324,7 @@ namespace nodetool
bool node_server<t_payload_net_handler>::make_expected_connections_count(PeerType peer_type, size_t expected_connections)
{
if (m_offline)
return true;
return false;
std::vector<anchor_peerlist_entry> apl;
@ -1270,24 +1334,24 @@ namespace nodetool
size_t conn_count = get_outgoing_connections_count();
//add new connections from white peers
while(conn_count < expected_connections)
if(conn_count < expected_connections)
{
if(m_net_server.is_stop_signal_sent())
return false;
MDEBUG("Making expected connection, type " << peer_type << ", " << conn_count << "/" << expected_connections << " connections");
if (peer_type == anchor && !make_new_connection_from_anchor_peerlist(apl)) {
break;
return false;
}
if (peer_type == white && !make_new_connection_from_peerlist(true)) {
break;
return false;
}
if (peer_type == gray && !make_new_connection_from_peerlist(false)) {
break;
return false;
}
conn_count = get_outgoing_connections_count();
}
return true;
}
@ -1390,6 +1454,9 @@ namespace nodetool
return false;
}
be.last_seen += delta;
#ifdef CRYPTONOTE_PRUNING_DEBUG_SPOOF_SEED
be.pruning_seed = tools::make_pruning_seed(1 + (be.adr.as<epee::net_utils::ipv4_network_address>().ip()) % (1ul << CRYPTONOTE_PRUNING_LOG_STRIPES), CRYPTONOTE_PRUNING_LOG_STRIPES);
#endif
}
return true;
}
@ -1750,6 +1817,7 @@ namespace nodetool
time(&last_seen);
pe.last_seen = static_cast<int64_t>(last_seen);
pe.id = peer_id_l;
pe.pruning_seed = context.m_pruning_seed;
this->m_peerlist.append_with_peer_white(pe);
LOG_DEBUG_CC(context, "PING SUCCESS " << context.m_remote_address.host_str() << ":" << port_l);
});
@ -1885,21 +1953,16 @@ namespace nodetool
template<class t_payload_net_handler>
bool node_server<t_payload_net_handler>::set_max_out_peers(const boost::program_options::variables_map& vm, int64_t max)
{
if(max == -1) {
m_config.m_net_config.max_out_connection_count = P2P_DEFAULT_CONNECTIONS_COUNT;
return true;
}
if(max == -1)
max = P2P_DEFAULT_CONNECTIONS_COUNT;
m_config.m_net_config.max_out_connection_count = max;
m_payload_handler.set_max_out_peers(max);
return true;
}
template<class t_payload_net_handler>
bool node_server<t_payload_net_handler>::set_max_in_peers(const boost::program_options::variables_map& vm, int64_t max)
{
if(max == -1) {
m_config.m_net_config.max_in_connection_count = -1;
return true;
}
m_config.m_net_config.max_in_connection_count = max;
return true;
}
@ -2010,6 +2073,7 @@ namespace nodetool
{
if (m_offline) return true;
if (!m_exclusive_peers.empty()) return true;
if (m_payload_handler.needs_new_sync_connections()) return true;
peerlist_entry pe = AUTO_VAL_INIT(pe);
@ -2030,13 +2094,49 @@ namespace nodetool
return true;
}
m_peerlist.set_peer_just_seen(pe.id, pe.adr);
m_peerlist.set_peer_just_seen(pe.id, pe.adr, pe.pruning_seed);
LOG_PRINT_L2("PEER PROMOTED TO WHITE PEER LIST IP address: " << pe.adr.host_str() << " Peer ID: " << peerid_type(pe.id));
return true;
}
template<class t_payload_net_handler>
void node_server<t_payload_net_handler>::add_used_stripe_peer(const typename t_payload_net_handler::connection_context &context)
{
const uint32_t stripe = tools::get_pruning_stripe(context.m_pruning_seed);
if (stripe == 0 || stripe > (1ul << CRYPTONOTE_PRUNING_LOG_STRIPES))
return;
const uint32_t index = stripe - 1;
CRITICAL_REGION_LOCAL(m_used_stripe_peers_mutex);
MINFO("adding stripe " << stripe << " peer: " << context.m_remote_address.str());
std::remove_if(m_used_stripe_peers[index].begin(), m_used_stripe_peers[index].end(),
[&context](const epee::net_utils::network_address &na){ return context.m_remote_address == na; });
m_used_stripe_peers[index].push_back(context.m_remote_address);
}
template<class t_payload_net_handler>
void node_server<t_payload_net_handler>::remove_used_stripe_peer(const typename t_payload_net_handler::connection_context &context)
{
const uint32_t stripe = tools::get_pruning_stripe(context.m_pruning_seed);
if (stripe == 0 || stripe > (1ul << CRYPTONOTE_PRUNING_LOG_STRIPES))
return;
const uint32_t index = stripe - 1;
CRITICAL_REGION_LOCAL(m_used_stripe_peers_mutex);
MINFO("removing stripe " << stripe << " peer: " << context.m_remote_address.str());
std::remove_if(m_used_stripe_peers[index].begin(), m_used_stripe_peers[index].end(),
[&context](const epee::net_utils::network_address &na){ return context.m_remote_address == na; });
}
template<class t_payload_net_handler>
void node_server<t_payload_net_handler>::clear_used_stripe_peers()
{
CRITICAL_REGION_LOCAL(m_used_stripe_peers_mutex);
MINFO("clearing used stripe peers");
for (auto &e: m_used_stripe_peers)
e.clear();
}
template<class t_payload_net_handler>
void node_server<t_payload_net_handler>::add_upnp_port_mapping(uint32_t port)
{

12
src/p2p/net_node_common.h
View file

@ -56,6 +56,9 @@ namespace nodetool
virtual bool unblock_host(const epee::net_utils::network_address &address)=0;
virtual std::map<std::string, time_t> get_blocked_hosts()=0;
virtual bool add_host_fail(const epee::net_utils::network_address &address)=0;
virtual void add_used_stripe_peer(const t_connection_context &context)=0;
virtual void remove_used_stripe_peer(const t_connection_context &context)=0;
virtual void clear_used_stripe_peers()=0;
};
template<class t_connection_context>
@ -114,5 +117,14 @@ namespace nodetool
{
return true;
}
virtual void add_used_stripe_peer(const t_connection_context &context)
{
}
virtual void remove_used_stripe_peer(const t_connection_context &context)
{
}
virtual void clear_used_stripe_peers()
{
}
};
}

36
src/p2p/net_peerlist.h
View file

@ -73,16 +73,18 @@ namespace nodetool
bool get_peerlist_full(std::vector<peerlist_entry>& pl_gray, std::vector<peerlist_entry>& pl_white);
bool get_white_peer_by_index(peerlist_entry& p, size_t i);
bool get_gray_peer_by_index(peerlist_entry& p, size_t i);
template<typename F> bool foreach(bool white, const F &f);
bool append_with_peer_white(const peerlist_entry& pr);
bool append_with_peer_gray(const peerlist_entry& pr);
bool append_with_peer_anchor(const anchor_peerlist_entry& ple);
bool set_peer_just_seen(peerid_type peer, const epee::net_utils::network_address& addr);
bool set_peer_just_seen(peerid_type peer, const epee::net_utils::network_address& addr, uint32_t pruning_seed);
bool set_peer_unreachable(const peerlist_entry& pr);
bool is_host_allowed(const epee::net_utils::network_address &address);
bool get_random_gray_peer(peerlist_entry& pe);
bool remove_from_peer_gray(const peerlist_entry& pe);
bool get_and_empty_anchor_peerlist(std::vector<anchor_peerlist_entry>& apl);
bool remove_from_peer_anchor(const epee::net_utils::network_address& addr);
bool remove_from_peer_white(const peerlist_entry& pe);
private:
struct by_time{};
@ -356,8 +358,19 @@ namespace nodetool
return true;
}
//--------------------------------------------------------------------------------------------------
template<typename F> inline
bool peerlist_manager::foreach(bool white, const F &f)
{
CRITICAL_REGION_LOCAL(m_peerlist_lock);
peers_indexed::index<by_time>::type& by_time_index = white ? m_peers_white.get<by_time>() : m_peers_gray.get<by_time>();
for(const peers_indexed::value_type& vl: boost::adaptors::reverse(by_time_index))
if (!f(vl))
return false;
return true;
}
//--------------------------------------------------------------------------------------------------
inline
bool peerlist_manager::set_peer_just_seen(peerid_type peer, const epee::net_utils::network_address& addr)
bool peerlist_manager::set_peer_just_seen(peerid_type peer, const epee::net_utils::network_address& addr, uint32_t pruning_seed)
{
TRY_ENTRY();
CRITICAL_REGION_LOCAL(m_peerlist_lock);
@ -366,6 +379,7 @@ namespace nodetool
ple.adr = addr;
ple.id = peer;
ple.last_seen = time(NULL);
ple.pruning_seed = pruning_seed;
return append_with_peer_white(ple);
CATCH_ENTRY_L0("peerlist_manager::set_peer_just_seen()", false);
}
@ -469,6 +483,24 @@ namespace nodetool
}
//--------------------------------------------------------------------------------------------------
inline
bool peerlist_manager::remove_from_peer_white(const peerlist_entry& pe)
{
TRY_ENTRY();
CRITICAL_REGION_LOCAL(m_peerlist_lock);
peers_indexed::index_iterator<by_addr>::type iterator = m_peers_white.get<by_addr>().find(pe.adr);
if (iterator != m_peers_white.get<by_addr>().end()) {
m_peers_white.erase(iterator);
}
return true;
CATCH_ENTRY_L0("peerlist_manager::remove_from_peer_white()", false);
}
//--------------------------------------------------------------------------------------------------
inline
bool peerlist_manager::remove_from_peer_gray(const peerlist_entry& pe)
{
TRY_ENTRY();

17
src/p2p/net_peerlist_boost_serialization.h
View file

@ -33,6 +33,10 @@
#include "net/net_utils_base.h"
#include "p2p/p2p_protocol_defs.h"
#ifdef CRYPTONOTE_PRUNING_DEBUG_SPOOF_SEED
#include "common/pruning.h"
#endif
namespace boost
{
namespace serialization
@ -77,6 +81,19 @@ namespace boost
a & pl.adr;
a & pl.id;
a & pl.last_seen;
if (ver < 1)
{
if (!typename Archive::is_saving())
pl.pruning_seed = 0;
return;
}
a & pl.pruning_seed;
#ifdef CRYPTONOTE_PRUNING_DEBUG_SPOOF_SEED
if (!typename Archive::is_saving())
{
pl.pruning_seed = tools::make_pruning_seed(1+pl.adr.as<epee::net_utils::ipv4_network_address>().ip() % (1<<CRYPTONOTE_PRUNING_LOG_STRIPES), CRYPTONOTE_PRUNING_LOG_STRIPES);
}
#endif
}
template <class Archive, class ver_type>

10
src/p2p/p2p_protocol_defs.h
View file

@ -31,6 +31,7 @@
#pragma once
#include <boost/uuid/uuid.hpp>
#include <boost/serialization/version.hpp>
#include "serialization/keyvalue_serialization.h"
#include "net/net_utils_base.h"
#include "misc_language.h"
@ -72,11 +73,13 @@ namespace nodetool
AddressType adr;
peerid_type id;
int64_t last_seen;
uint32_t pruning_seed;
BEGIN_KV_SERIALIZE_MAP()
KV_SERIALIZE(adr)
KV_SERIALIZE(id)
KV_SERIALIZE(last_seen)
KV_SERIALIZE_OPT(pruning_seed, (uint32_t)0)
END_KV_SERIALIZE_MAP()
};
typedef peerlist_entry_base<epee::net_utils::network_address> peerlist_entry;
@ -122,7 +125,7 @@ namespace nodetool
ss << std::setfill ('0') << std::setw (8) << std::hex << std::noshowbase;
for(const peerlist_entry& pe: pl)
{
ss << pe.id << "\t" << pe.adr.str() << " \tlast_seen: " << epee::misc_utils::get_time_interval_string(now_time - pe.last_seen) << std::endl;
ss << pe.id << "\t" << pe.adr.str() << " \tpruning seed " << pe.pruning_seed << " \tlast_seen: " << epee::misc_utils::get_time_interval_string(now_time - pe.last_seen) << std::endl;
}
return ss.str();
}
@ -205,7 +208,7 @@ namespace nodetool
{
const epee::net_utils::network_address &na = p.adr;
const epee::net_utils::ipv4_network_address &ipv4 = na.as<const epee::net_utils::ipv4_network_address>();
local_peerlist.push_back(peerlist_entry_base<network_address_old>({{ipv4.ip(), ipv4.port()}, p.id, p.last_seen}));
local_peerlist.push_back(peerlist_entry_base<network_address_old>({{ipv4.ip(), ipv4.port()}, p.id, p.last_seen, p.pruning_seed}));
}
else
MDEBUG("Not including in legacy peer list: " << p.adr.str());
@ -220,7 +223,7 @@ namespace nodetool
std::vector<peerlist_entry_base<network_address_old>> local_peerlist;
epee::serialization::selector<is_store>::serialize_stl_container_pod_val_as_blob(local_peerlist, stg, hparent_section, "local_peerlist");
for (const auto &p: local_peerlist)
((response&)this_ref).local_peerlist_new.push_back(peerlist_entry({epee::net_utils::ipv4_network_address(p.adr.ip, p.adr.port), p.id, p.last_seen}));
((response&)this_ref).local_peerlist_new.push_back(peerlist_entry({epee::net_utils::ipv4_network_address(p.adr.ip, p.adr.port), p.id, p.last_seen, p.pruning_seed}));
}
}
END_KV_SERIALIZE_MAP()
@ -463,5 +466,6 @@ namespace nodetool
}
BOOST_CLASS_VERSION(nodetool::peerlist_entry, 1)

87
src/rpc/core_rpc_server.cpp
View file

@ -485,8 +485,8 @@ namespace cryptonote
vh.push_back(*reinterpret_cast<const crypto::hash*>(b.data()));
}
std::vector<crypto::hash> missed_txs;
std::vector<transaction> txs;
bool r = m_core.get_transactions(vh, txs, missed_txs);
std::vector<std::tuple<crypto::hash, cryptonote::blobdata, crypto::hash, cryptonote::blobdata>> txs;
bool r = m_core.get_split_transactions_blobs(vh, txs, missed_txs);
if(!r)
{
res.status = "Failed";
@ -506,7 +506,7 @@ namespace cryptonote
if(r)
{
// sort to match original request
std::vector<transaction> sorted_txs;
std::vector<std::tuple<crypto::hash, cryptonote::blobdata, crypto::hash, cryptonote::blobdata>> sorted_txs;
std::vector<tx_info>::const_iterator i;
unsigned txs_processed = 0;
for (const crypto::hash &h: vh)
@ -519,7 +519,7 @@ namespace cryptonote
return true;
}
// core returns the ones it finds in the right order
if (get_transaction_hash(txs[txs_processed]) != h)
if (std::get<0>(txs[txs_processed]) != h)
{
res.status = "Failed: tx hash mismatch";
return true;
@ -535,7 +535,16 @@ namespace cryptonote
res.status = "Failed to parse and validate tx from blob";
return true;
}
sorted_txs.push_back(tx);
std::stringstream ss;
binary_archive<true> ba(ss);
bool r = const_cast<cryptonote::transaction&>(tx).serialize_base(ba);
if (!r)
{
res.status = "Failed to serialize transaction base";
return true;
}
const cryptonote::blobdata pruned = ss.str();
sorted_txs.push_back(std::make_tuple(h, pruned, get_transaction_prunable_hash(tx), std::string(i->tx_blob, pruned.size())));
missed_txs.erase(std::find(missed_txs.begin(), missed_txs.end(), h));
pool_tx_hashes.insert(h);
const std::string hash_string = epee::string_tools::pod_to_hex(h);
@ -564,11 +573,36 @@ namespace cryptonote
crypto::hash tx_hash = *vhi++;
e.tx_hash = *txhi++;
pruned_transaction pruned_tx{tx};
blobdata blob = req.prune ? t_serializable_object_to_blob(pruned_tx) : t_serializable_object_to_blob(tx);
e.as_hex = string_tools::buff_to_hex_nodelimer(blob);
if (req.decode_as_json)
e.as_json = req.prune ? obj_to_json_str(pruned_tx) : obj_to_json_str(tx);
e.prunable_hash = epee::string_tools::pod_to_hex(std::get<2>(tx));
if (req.split || req.prune || std::get<3>(tx).empty())
{
e.pruned_as_hex = string_tools::buff_to_hex_nodelimer(std::get<1>(tx));
if (!req.prune)
e.prunable_as_hex = string_tools::buff_to_hex_nodelimer(std::get<3>(tx));
}
else
{
cryptonote::blobdata tx_data;
if (req.prune)
tx_data = std::get<1>(tx);
else
tx_data = std::get<1>(tx) + std::get<3>(tx);
e.as_hex = string_tools::buff_to_hex_nodelimer(tx_data);
if (req.decode_as_json && !tx_data.empty())
{
cryptonote::transaction t;
if (cryptonote::parse_and_validate_tx_from_blob(tx_data, t))
{
if (req.prune)
{
pruned_transaction pruned_tx{t};
e.as_json = obj_to_json_str(pruned_tx);
}
else
e.as_json = obj_to_json_str(t);
}
}
}
e.in_pool = pool_tx_hashes.find(tx_hash) != pool_tx_hashes.end();
if (e.in_pool)
{
@ -869,9 +903,9 @@ namespace cryptonote
{
if (entry.adr.get_type_id() == epee::net_utils::ipv4_network_address::ID)
res.white_list.emplace_back(entry.id, entry.adr.as<epee::net_utils::ipv4_network_address>().ip(),
entry.adr.as<epee::net_utils::ipv4_network_address>().port(), entry.last_seen);
entry.adr.as<epee::net_utils::ipv4_network_address>().port(), entry.last_seen, entry.pruning_seed);
else
res.white_list.emplace_back(entry.id, entry.adr.str(), entry.last_seen);
res.white_list.emplace_back(entry.id, entry.adr.str(), entry.last_seen, entry.pruning_seed);
}
res.gray_list.reserve(gray_list.size());
@ -879,9 +913,9 @@ namespace cryptonote
{
if (entry.adr.get_type_id() == epee::net_utils::ipv4_network_address::ID)
res.gray_list.emplace_back(entry.id, entry.adr.as<epee::net_utils::ipv4_network_address>().ip(),
entry.adr.as<epee::net_utils::ipv4_network_address>().port(), entry.last_seen);
entry.adr.as<epee::net_utils::ipv4_network_address>().port(), entry.last_seen, entry.pruning_seed);
else
res.gray_list.emplace_back(entry.id, entry.adr.str(), entry.last_seen);
res.gray_list.emplace_back(entry.id, entry.adr.str(), entry.last_seen, entry.pruning_seed);
}
res.status = CORE_RPC_STATUS_OK;
@ -2102,6 +2136,7 @@ namespace cryptonote
m_core.get_blockchain_top(res.height, top_hash);
++res.height; // turn top block height into blockchain height
res.target_height = m_core.get_target_blockchain_height();
res.next_needed_pruning_seed = m_p2p.get_payload_object().get_next_needed_pruning_stripe().second;
for (const auto &c: m_p2p.get_payload_object().get_connections())
res.peers.push_back({c});
@ -2116,6 +2151,7 @@ namespace cryptonote
res.spans.push_back({span.start_block_height, span.nblocks, span_connection_id, (uint32_t)(span.rate + 0.5f), speed, span.size, address});
return true;
});
res.overview = block_queue.get_overview(res.height);
res.status = CORE_RPC_STATUS_OK;
return true;
@ -2215,6 +2251,29 @@ namespace cryptonote
return true;
}
//------------------------------------------------------------------------------------------------------------------------------
bool core_rpc_server::on_prune_blockchain(const COMMAND_RPC_PRUNE_BLOCKCHAIN::request& req, COMMAND_RPC_PRUNE_BLOCKCHAIN::response& res, epee::json_rpc::error& error_resp)
{
try
{
if (!(req.check ? m_core.check_blockchain_pruning() : m_core.prune_blockchain()))
{
error_resp.code = CORE_RPC_ERROR_CODE_INTERNAL_ERROR;
error_resp.message = req.check ? "Failed to check blockchain pruning" : "Failed to prune blockchain";
return false;
}
res.pruning_seed = m_core.get_blockchain_pruning_seed();
}
catch (const std::exception &e)
{
error_resp.code = CORE_RPC_ERROR_CODE_INTERNAL_ERROR;
error_resp.message = "Failed to prune blockchain";
return false;
}
res.status = CORE_RPC_STATUS_OK;
return true;
}
//------------------------------------------------------------------------------------------------------------------------------
const command_line::arg_descriptor<std::string, false, true, 2> core_rpc_server::arg_rpc_bind_port = {

2
src/rpc/core_rpc_server.h
View file

@ -153,6 +153,7 @@ namespace cryptonote
MAP_JON_RPC_WE_IF("sync_info", on_sync_info, COMMAND_RPC_SYNC_INFO, !m_restricted)
MAP_JON_RPC_WE("get_txpool_backlog", on_get_txpool_backlog, COMMAND_RPC_GET_TRANSACTION_POOL_BACKLOG)
MAP_JON_RPC_WE("get_output_distribution", on_get_output_distribution, COMMAND_RPC_GET_OUTPUT_DISTRIBUTION)
MAP_JON_RPC_WE_IF("prune_blockchain", on_prune_blockchain, COMMAND_RPC_PRUNE_BLOCKCHAIN, !m_restricted)
END_JSON_RPC_MAP()
END_URI_MAP2()
@ -217,6 +218,7 @@ namespace cryptonote
bool on_sync_info(const COMMAND_RPC_SYNC_INFO::request& req, COMMAND_RPC_SYNC_INFO::response& res, epee::json_rpc::error& error_resp);
bool on_get_txpool_backlog(const COMMAND_RPC_GET_TRANSACTION_POOL_BACKLOG::request& req, COMMAND_RPC_GET_TRANSACTION_POOL_BACKLOG::response& res, epee::json_rpc::error& error_resp);
bool on_get_output_distribution(const COMMAND_RPC_GET_OUTPUT_DISTRIBUTION::request& req, COMMAND_RPC_GET_OUTPUT_DISTRIBUTION::response& res, epee::json_rpc::error& error_resp);
bool on_prune_blockchain(const COMMAND_RPC_PRUNE_BLOCKCHAIN::request& req, COMMAND_RPC_PRUNE_BLOCKCHAIN::response& res, epee::json_rpc::error& error_resp);
//-----------------------
private:

47
src/rpc/core_rpc_server_commands_defs.h
View file

@ -84,7 +84,7 @@ namespace cryptonote
// advance which version they will stop working with
// Don't go over 32767 for any of these
#define CORE_RPC_VERSION_MAJOR 2
#define CORE_RPC_VERSION_MINOR 2
#define CORE_RPC_VERSION_MINOR 3
#define MAKE_CORE_RPC_VERSION(major,minor) (((major)<<16)|(minor))
#define CORE_RPC_VERSION MAKE_CORE_RPC_VERSION(CORE_RPC_VERSION_MAJOR, CORE_RPC_VERSION_MINOR)
@ -601,11 +601,13 @@ namespace cryptonote
std::vector<std::string> txs_hashes;
bool decode_as_json;
bool prune;
bool split;
BEGIN_KV_SERIALIZE_MAP()
KV_SERIALIZE(txs_hashes)
KV_SERIALIZE(decode_as_json)
KV_SERIALIZE_OPT(prune, false)
KV_SERIALIZE_OPT(split, false)
END_KV_SERIALIZE_MAP()
};
@ -613,6 +615,9 @@ namespace cryptonote
{
std::string tx_hash;
std::string as_hex;
std::string pruned_as_hex;
std::string prunable_as_hex;
std::string prunable_hash;
std::string as_json;
bool in_pool;
bool double_spend_seen;
@ -623,6 +628,9 @@ namespace cryptonote
BEGIN_KV_SERIALIZE_MAP()
KV_SERIALIZE(tx_hash)
KV_SERIALIZE(as_hex)
KV_SERIALIZE(pruned_as_hex)
KV_SERIALIZE(prunable_as_hex)
KV_SERIALIZE(prunable_hash)
KV_SERIALIZE(as_json)
KV_SERIALIZE(in_pool)
KV_SERIALIZE(double_spend_seen)
@ -1311,14 +1319,15 @@ namespace cryptonote
uint32_t ip;
uint16_t port;
uint64_t last_seen;
uint32_t pruning_seed;
peer() = default;
peer(uint64_t id, const std::string &host, uint64_t last_seen)
: id(id), host(host), ip(0), port(0), last_seen(last_seen)
peer(uint64_t id, const std::string &host, uint64_t last_seen, uint32_t pruning_seed)
: id(id), host(host), ip(0), port(0), last_seen(last_seen), pruning_seed(pruning_seed)
{}
peer(uint64_t id, uint32_t ip, uint16_t port, uint64_t last_seen)
: id(id), host(std::to_string(ip)), ip(ip), port(port), last_seen(last_seen)
peer(uint64_t id, uint32_t ip, uint16_t port, uint64_t last_seen, uint32_t pruning_seed)
: id(id), host(std::to_string(ip)), ip(ip), port(port), last_seen(last_seen), pruning_seed(pruning_seed)
{}
BEGIN_KV_SERIALIZE_MAP()
@ -1327,6 +1336,7 @@ namespace cryptonote
KV_SERIALIZE(ip)
KV_SERIALIZE(port)
KV_SERIALIZE(last_seen)
KV_SERIALIZE_OPT(pruning_seed, (uint32_t)0)
END_KV_SERIALIZE_MAP()
};
@ -2238,15 +2248,19 @@ namespace cryptonote
std::string status;
uint64_t height;
uint64_t target_height;
uint32_t next_needed_pruning_seed;
std::list<peer> peers;
std::list<span> spans;
std::string overview;
BEGIN_KV_SERIALIZE_MAP()
KV_SERIALIZE(status)
KV_SERIALIZE(height)
KV_SERIALIZE(target_height)
KV_SERIALIZE(next_needed_pruning_seed)
KV_SERIALIZE(peers)
KV_SERIALIZE(spans)
KV_SERIALIZE(overview)
END_KV_SERIALIZE_MAP()
};
};
@ -2351,4 +2365,27 @@ namespace cryptonote
};
};
struct COMMAND_RPC_PRUNE_BLOCKCHAIN
{
struct request
{
bool check;
BEGIN_KV_SERIALIZE_MAP()
KV_SERIALIZE_OPT(check, false)
END_KV_SERIALIZE_MAP()
};
struct response
{
uint32_t pruning_seed;
std::string status;
BEGIN_KV_SERIALIZE_MAP()
KV_SERIALIZE(status)
KV_SERIALIZE(pruning_seed)
END_KV_SERIALIZE_MAP()
};
};
}

1
src/rpc/message_data_structs.h
View file

@ -79,6 +79,7 @@ namespace rpc
uint32_t ip;
uint16_t port;
uint64_t last_seen;
uint32_t pruning_seed;
};
struct tx_in_pool

2
src/serialization/json_object.cpp
View file

@ -734,6 +734,7 @@ void toJsonValue(rapidjson::Document& doc, const cryptonote::rpc::peer& peer, ra
INSERT_INTO_JSON_OBJECT(val, doc, ip, peer.ip);
INSERT_INTO_JSON_OBJECT(val, doc, port, peer.port);
INSERT_INTO_JSON_OBJECT(val, doc, last_seen, peer.last_seen);
INSERT_INTO_JSON_OBJECT(val, doc, pruning_seed, peer.pruning_seed);
}
@ -748,6 +749,7 @@ void fromJsonValue(const rapidjson::Value& val, cryptonote::rpc::peer& peer)
GET_FROM_JSON_OBJECT(val, peer.ip, ip);
GET_FROM_JSON_OBJECT(val, peer.port, port);
GET_FROM_JSON_OBJECT(val, peer.last_seen, last_seen);
GET_FROM_JSON_OBJECT(val, peer.pruning_seed, pruning_seed);
}
void toJsonValue(rapidjson::Document& doc, const cryptonote::rpc::tx_in_pool& tx, rapidjson::Value& val)

207
src/wallet/wallet2.cpp
View file

@ -814,6 +814,43 @@ static void setup_shim(hw::wallet_shim * shim, tools::wallet2 * wallet)
shim->get_tx_pub_key_from_received_outs = boost::bind(&tools::wallet2::get_tx_pub_key_from_received_outs, wallet, _1);
}
bool get_pruned_tx(const cryptonote::COMMAND_RPC_GET_TRANSACTIONS::entry &entry, cryptonote::transaction &tx, crypto::hash &tx_hash)
{
cryptonote::blobdata bd;
// easy case if we have the whole tx
if (!entry.as_hex.empty() || (!entry.prunable_as_hex.empty() && !entry.pruned_as_hex.empty()))
{
CHECK_AND_ASSERT_MES(epee::string_tools::parse_hexstr_to_binbuff(entry.as_hex.empty() ? entry.pruned_as_hex + entry.prunable_as_hex : entry.as_hex, bd), false, "Failed to parse tx data");
CHECK_AND_ASSERT_MES(cryptonote::parse_and_validate_tx_from_blob(bd, tx), false, "Invalid tx data");
tx_hash = cryptonote::get_transaction_hash(tx);
// if the hash was given, check it matches
CHECK_AND_ASSERT_MES(entry.tx_hash.empty() || epee::string_tools::pod_to_hex(tx_hash) == entry.tx_hash, false,
"Response claims a different hash than the data yields");
return true;
}
// case of a pruned tx with its prunable data hash
if (!entry.pruned_as_hex.empty() && !entry.prunable_hash.empty())
{
crypto::hash ph;
CHECK_AND_ASSERT_MES(epee::string_tools::hex_to_pod(entry.prunable_hash, ph), false, "Failed to parse prunable hash");
CHECK_AND_ASSERT_MES(epee::string_tools::parse_hexstr_to_binbuff(entry.pruned_as_hex, bd), false, "Failed to parse pruned data");
CHECK_AND_ASSERT_MES(parse_and_validate_tx_base_from_blob(bd, tx), false, "Invalid base tx data");
// only v2 txes can calculate their txid after pruned
if (bd[0] > 1)
{
tx_hash = cryptonote::get_pruned_transaction_hash(tx, ph);
}
else
{
// for v1, we trust the dameon
CHECK_AND_ASSERT_MES(epee::string_tools::hex_to_pod(entry.tx_hash, tx_hash), false, "Failed to parse tx hash");
}
return true;
}
return false;
}
//-----------------------------------------------------------------
} //namespace
@ -2588,7 +2625,7 @@ void wallet2::update_pool_state(bool refreshed)
req.txs_hashes.push_back(epee::string_tools::pod_to_hex(p.first));
MDEBUG("asking for " << txids.size() << " transactions");
req.decode_as_json = false;
req.prune = false;
req.prune = true;
m_daemon_rpc_mutex.lock();
bool r = epee::net_utils::invoke_http_json("/gettransactions", req, res, m_http_client, rpc_timeout);
m_daemon_rpc_mutex.unlock();
@ -2603,11 +2640,10 @@ void wallet2::update_pool_state(bool refreshed)
{
cryptonote::transaction tx;
cryptonote::blobdata bd;
crypto::hash tx_hash, tx_prefix_hash;
if (epee::string_tools::parse_hexstr_to_binbuff(tx_entry.as_hex, bd))
crypto::hash tx_hash;
if (get_pruned_tx(tx_entry, tx, tx_hash))
{
if (cryptonote::parse_and_validate_tx_from_blob(bd, tx, tx_hash, tx_prefix_hash))
{
const std::vector<std::pair<crypto::hash, bool>>::const_iterator i = std::find_if(txids.begin(), txids.end(),
[tx_hash](const std::pair<crypto::hash, bool> &e) { return e.first == tx_hash; });
if (i != txids.end())
@ -2624,11 +2660,6 @@ void wallet2::update_pool_state(bool refreshed)
{
MERROR("Got txid " << tx_hash << " which we did not ask for");
}
}
else
{
LOG_PRINT_L0("failed to validate transaction from daemon");
}
}
else
{
@ -6814,11 +6845,12 @@ bool wallet2::find_and_save_rings(bool force)
MDEBUG("Found " << std::to_string(txs_hashes.size()) << " transactions");
// get those transactions from the daemon
auto it = txs_hashes.begin();
static const size_t SLICE_SIZE = 200;
for (size_t slice = 0; slice < txs_hashes.size(); slice += SLICE_SIZE)
{
req.decode_as_json = false;
req.prune = false;
req.prune = true;
req.txs_hashes.clear();
size_t ntxes = slice + SLICE_SIZE > txs_hashes.size() ? txs_hashes.size() - slice : SLICE_SIZE;
for (size_t s = slice; s < slice + ntxes; ++s)
@ -6837,19 +6869,15 @@ bool wallet2::find_and_save_rings(bool force)
MDEBUG("Scanning " << res.txs.size() << " transactions");
THROW_WALLET_EXCEPTION_IF(slice + res.txs.size() > txs_hashes.size(), error::wallet_internal_error, "Unexpected tx array size");
auto it = req.txs_hashes.begin();
for (size_t i = 0; i < res.txs.size(); ++i, ++it)
{
const auto &tx_info = res.txs[i];
THROW_WALLET_EXCEPTION_IF(tx_info.tx_hash != epee::string_tools::pod_to_hex(txs_hashes[slice + i]), error::wallet_internal_error, "Wrong txid received");
THROW_WALLET_EXCEPTION_IF(tx_info.tx_hash != *it, error::wallet_internal_error, "Wrong txid received");
cryptonote::blobdata bd;
THROW_WALLET_EXCEPTION_IF(!epee::string_tools::parse_hexstr_to_binbuff(tx_info.as_hex, bd), error::wallet_internal_error, "failed to parse tx from hexstr");
cryptonote::transaction tx;
crypto::hash tx_hash, tx_prefix_hash;
THROW_WALLET_EXCEPTION_IF(!cryptonote::parse_and_validate_tx_from_blob(bd, tx, tx_hash, tx_prefix_hash), error::wallet_internal_error, "failed to parse tx from blob");
THROW_WALLET_EXCEPTION_IF(epee::string_tools::pod_to_hex(tx_hash) != tx_info.tx_hash, error::wallet_internal_error, "txid mismatch");
THROW_WALLET_EXCEPTION_IF(!add_rings(get_ringdb_key(), tx), error::wallet_internal_error, "Failed to save ring");
cryptonote::transaction tx;
crypto::hash tx_hash;
THROW_WALLET_EXCEPTION_IF(!get_pruned_tx(tx_info, tx, tx_hash), error::wallet_internal_error,
"Failed to get transaction from daemon");
THROW_WALLET_EXCEPTION_IF(!(tx_hash == *it), error::wallet_internal_error, "Wrong txid received");
THROW_WALLET_EXCEPTION_IF(!add_rings(get_ringdb_key(), tx), error::wallet_internal_error, "Failed to save ring");
}
}
@ -9782,7 +9810,7 @@ void wallet2::set_tx_key(const crypto::hash &txid, const crypto::secret_key &tx_
COMMAND_RPC_GET_TRANSACTIONS::request req = AUTO_VAL_INIT(req);
req.txs_hashes.push_back(epee::string_tools::pod_to_hex(txid));
req.decode_as_json = false;
req.prune = false;
req.prune = true;
COMMAND_RPC_GET_TRANSACTIONS::response res = AUTO_VAL_INIT(res);
bool r;
{
@ -9795,11 +9823,10 @@ void wallet2::set_tx_key(const crypto::hash &txid, const crypto::secret_key &tx_
THROW_WALLET_EXCEPTION_IF(res.txs.size() != 1, error::wallet_internal_error,
"daemon returned wrong response for gettransactions, wrong txs count = " +
std::to_string(res.txs.size()) + ", expected 1");
cryptonote::blobdata bd;
THROW_WALLET_EXCEPTION_IF(!epee::string_tools::parse_hexstr_to_binbuff(res.txs[0].as_hex, bd), error::wallet_internal_error, "failed to parse tx from hexstr");
cryptonote::transaction tx;
crypto::hash tx_hash, tx_prefix_hash;
THROW_WALLET_EXCEPTION_IF(!cryptonote::parse_and_validate_tx_from_blob(bd, tx, tx_hash, tx_prefix_hash), error::wallet_internal_error, "failed to parse tx from blob");
crypto::hash tx_hash;
THROW_WALLET_EXCEPTION_IF(!get_pruned_tx(res.txs[0], tx, tx_hash), error::wallet_internal_error,
"Failed to get transaction from daemon");
THROW_WALLET_EXCEPTION_IF(tx_hash != txid, error::wallet_internal_error, "txid mismatch");
std::vector<tx_extra_field> tx_extra_fields;
THROW_WALLET_EXCEPTION_IF(!parse_tx_extra(tx.extra, tx_extra_fields), error::wallet_internal_error, "Transaction extra has unsupported format");
@ -9833,7 +9860,7 @@ std::string wallet2::get_spend_proof(const crypto::hash &txid, const std::string
COMMAND_RPC_GET_TRANSACTIONS::request req = AUTO_VAL_INIT(req);
req.txs_hashes.push_back(epee::string_tools::pod_to_hex(txid));
req.decode_as_json = false;
req.prune = false;
req.prune = true;
COMMAND_RPC_GET_TRANSACTIONS::response res = AUTO_VAL_INIT(res);
bool r;
{
@ -9846,12 +9873,10 @@ std::string wallet2::get_spend_proof(const crypto::hash &txid, const std::string
THROW_WALLET_EXCEPTION_IF(res.txs.size() != 1, error::wallet_internal_error,
"daemon returned wrong response for gettransactions, wrong txs count = " +
std::to_string(res.txs.size()) + ", expected 1");
cryptonote::blobdata bd;
THROW_WALLET_EXCEPTION_IF(!epee::string_tools::parse_hexstr_to_binbuff(res.txs[0].as_hex, bd), error::wallet_internal_error, "failed to parse tx from hexstr");
cryptonote::transaction tx;
crypto::hash tx_hash, tx_prefix_hash;
THROW_WALLET_EXCEPTION_IF(!cryptonote::parse_and_validate_tx_from_blob(bd, tx, tx_hash, tx_prefix_hash), error::wallet_internal_error, "failed to parse tx from blob");
THROW_WALLET_EXCEPTION_IF(tx_hash != txid, error::wallet_internal_error, "txid mismatch");
crypto::hash tx_hash;
THROW_WALLET_EXCEPTION_IF(!get_pruned_tx(res.txs[0], tx, tx_hash), error::wallet_internal_error, "Failed to get tx from daemon");
std::vector<std::vector<crypto::signature>> signatures;
@ -9953,7 +9978,7 @@ bool wallet2::check_spend_proof(const crypto::hash &txid, const std::string &mes
COMMAND_RPC_GET_TRANSACTIONS::request req = AUTO_VAL_INIT(req);
req.txs_hashes.push_back(epee::string_tools::pod_to_hex(txid));
req.decode_as_json = false;
req.prune = false;
req.prune = true;
COMMAND_RPC_GET_TRANSACTIONS::response res = AUTO_VAL_INIT(res);
bool r;
{
@ -9966,12 +9991,10 @@ bool wallet2::check_spend_proof(const crypto::hash &txid, const std::string &mes
THROW_WALLET_EXCEPTION_IF(res.txs.size() != 1, error::wallet_internal_error,
"daemon returned wrong response for gettransactions, wrong txs count = " +
std::to_string(res.txs.size()) + ", expected 1");
cryptonote::blobdata bd;
THROW_WALLET_EXCEPTION_IF(!epee::string_tools::parse_hexstr_to_binbuff(res.txs[0].as_hex, bd), error::wallet_internal_error, "failed to parse tx from hexstr");
cryptonote::transaction tx;
crypto::hash tx_hash, tx_prefix_hash;
THROW_WALLET_EXCEPTION_IF(!cryptonote::parse_and_validate_tx_from_blob(bd, tx, tx_hash, tx_prefix_hash), error::wallet_internal_error, "failed to parse tx from blob");
THROW_WALLET_EXCEPTION_IF(tx_hash != txid, error::wallet_internal_error, "txid mismatch");
crypto::hash tx_hash;
THROW_WALLET_EXCEPTION_IF(!get_pruned_tx(res.txs[0], tx, tx_hash), error::wallet_internal_error, "failed to get tx from daemon");
// check signature size
size_t num_sigs = 0;
@ -10078,24 +10101,30 @@ void wallet2::check_tx_key_helper(const crypto::hash &txid, const crypto::key_de
COMMAND_RPC_GET_TRANSACTIONS::response res;
req.txs_hashes.push_back(epee::string_tools::pod_to_hex(txid));
req.decode_as_json = false;
req.prune = false;
req.prune = true;
m_daemon_rpc_mutex.lock();
bool ok = epee::net_utils::invoke_http_json("/gettransactions", req, res, m_http_client);
m_daemon_rpc_mutex.unlock();
THROW_WALLET_EXCEPTION_IF(!ok || (res.txs.size() != 1 && res.txs_as_hex.size() != 1),
error::wallet_internal_error, "Failed to get transaction from daemon");
cryptonote::blobdata tx_data;
if (res.txs.size() == 1)
ok = string_tools::parse_hexstr_to_binbuff(res.txs.front().as_hex, tx_data);
else
ok = string_tools::parse_hexstr_to_binbuff(res.txs_as_hex.front(), tx_data);
THROW_WALLET_EXCEPTION_IF(!ok, error::wallet_internal_error, "Failed to parse transaction from daemon");
crypto::hash tx_hash, tx_prefix_hash;
cryptonote::transaction tx;
THROW_WALLET_EXCEPTION_IF(!cryptonote::parse_and_validate_tx_from_blob(tx_data, tx, tx_hash, tx_prefix_hash), error::wallet_internal_error,
"Failed to validate transaction from daemon");
crypto::hash tx_hash;
if (res.txs.size() == 1)
{
ok = get_pruned_tx(res.txs.front(), tx, tx_hash);
THROW_WALLET_EXCEPTION_IF(!ok, error::wallet_internal_error, "Failed to parse transaction from daemon");
}
else
{
cryptonote::blobdata tx_data;
crypto::hash tx_prefix_hash;
ok = string_tools::parse_hexstr_to_binbuff(res.txs_as_hex.front(), tx_data);
THROW_WALLET_EXCEPTION_IF(!ok, error::wallet_internal_error, "Failed to parse transaction from daemon");
THROW_WALLET_EXCEPTION_IF(!cryptonote::parse_and_validate_tx_from_blob(tx_data, tx, tx_hash, tx_prefix_hash),
error::wallet_internal_error, "Failed to validate transaction from daemon");
}
THROW_WALLET_EXCEPTION_IF(tx_hash != txid, error::wallet_internal_error,
"Failed to get the right transaction from daemon");
THROW_WALLET_EXCEPTION_IF(!additional_derivations.empty() && additional_derivations.size() != tx.vout.size(), error::wallet_internal_error,
@ -10218,24 +10247,30 @@ std::string wallet2::get_tx_proof(const crypto::hash &txid, const cryptonote::ac
COMMAND_RPC_GET_TRANSACTIONS::response res;
req.txs_hashes.push_back(epee::string_tools::pod_to_hex(txid));
req.decode_as_json = false;
req.prune = false;
req.prune = true;
m_daemon_rpc_mutex.lock();
bool ok = net_utils::invoke_http_json("/gettransactions", req, res, m_http_client);
m_daemon_rpc_mutex.unlock();
THROW_WALLET_EXCEPTION_IF(!ok || (res.txs.size() != 1 && res.txs_as_hex.size() != 1),
error::wallet_internal_error, "Failed to get transaction from daemon");
cryptonote::blobdata tx_data;
if (res.txs.size() == 1)
ok = string_tools::parse_hexstr_to_binbuff(res.txs.front().as_hex, tx_data);
else
ok = string_tools::parse_hexstr_to_binbuff(res.txs_as_hex.front(), tx_data);
THROW_WALLET_EXCEPTION_IF(!ok, error::wallet_internal_error, "Failed to parse transaction from daemon");
crypto::hash tx_hash, tx_prefix_hash;
cryptonote::transaction tx;
THROW_WALLET_EXCEPTION_IF(!cryptonote::parse_and_validate_tx_from_blob(tx_data, tx, tx_hash, tx_prefix_hash), error::wallet_internal_error,
"Failed to validate transaction from daemon");
crypto::hash tx_hash;
if (res.txs.size() == 1)
{
ok = get_pruned_tx(res.txs.front(), tx, tx_hash);
THROW_WALLET_EXCEPTION_IF(!ok, error::wallet_internal_error, "Failed to parse transaction from daemon");
}
else
{
cryptonote::blobdata tx_data;
crypto::hash tx_prefix_hash;
ok = string_tools::parse_hexstr_to_binbuff(res.txs_as_hex.front(), tx_data);
THROW_WALLET_EXCEPTION_IF(!ok, error::wallet_internal_error, "Failed to parse transaction from daemon");
THROW_WALLET_EXCEPTION_IF(!cryptonote::parse_and_validate_tx_from_blob(tx_data, tx, tx_hash, tx_prefix_hash),
error::wallet_internal_error, "Failed to validate transaction from daemon");
}
THROW_WALLET_EXCEPTION_IF(tx_hash != txid, error::wallet_internal_error, "Failed to get the right transaction from daemon");
crypto::public_key tx_pub_key = get_tx_pub_key_from_extra(tx);
@ -10330,24 +10365,30 @@ bool wallet2::check_tx_proof(const crypto::hash &txid, const cryptonote::account
COMMAND_RPC_GET_TRANSACTIONS::response res;
req.txs_hashes.push_back(epee::string_tools::pod_to_hex(txid));
req.decode_as_json = false;
req.prune = false;
req.prune = true;
m_daemon_rpc_mutex.lock();
bool ok = net_utils::invoke_http_json("/gettransactions", req, res, m_http_client);
m_daemon_rpc_mutex.unlock();
THROW_WALLET_EXCEPTION_IF(!ok || (res.txs.size() != 1 && res.txs_as_hex.size() != 1),
error::wallet_internal_error, "Failed to get transaction from daemon");
cryptonote::blobdata tx_data;
if (res.txs.size() == 1)
ok = string_tools::parse_hexstr_to_binbuff(res.txs.front().as_hex, tx_data);
else
ok = string_tools::parse_hexstr_to_binbuff(res.txs_as_hex.front(), tx_data);
THROW_WALLET_EXCEPTION_IF(!ok, error::wallet_internal_error, "Failed to parse transaction from daemon");
crypto::hash tx_hash, tx_prefix_hash;
cryptonote::transaction tx;
THROW_WALLET_EXCEPTION_IF(!cryptonote::parse_and_validate_tx_from_blob(tx_data, tx, tx_hash, tx_prefix_hash), error::wallet_internal_error,
"Failed to validate transaction from daemon");
crypto::hash tx_hash;
if (res.txs.size() == 1)
{
ok = get_pruned_tx(res.txs.front(), tx, tx_hash);
THROW_WALLET_EXCEPTION_IF(!ok, error::wallet_internal_error, "Failed to parse transaction from daemon");
}
else
{
cryptonote::blobdata tx_data;
crypto::hash tx_prefix_hash;
ok = string_tools::parse_hexstr_to_binbuff(res.txs_as_hex.front(), tx_data);
THROW_WALLET_EXCEPTION_IF(!ok, error::wallet_internal_error, "Failed to parse transaction from daemon");
THROW_WALLET_EXCEPTION_IF(!cryptonote::parse_and_validate_tx_from_blob(tx_data, tx, tx_hash, tx_prefix_hash),
error::wallet_internal_error, "Failed to validate transaction from daemon");
}
THROW_WALLET_EXCEPTION_IF(tx_hash != txid, error::wallet_internal_error, "Failed to get the right transaction from daemon");
crypto::public_key tx_pub_key = get_tx_pub_key_from_extra(tx);
@ -10566,7 +10607,7 @@ bool wallet2::check_reserve_proof(const cryptonote::account_public_address &addr
for (size_t i = 0; i < proofs.size(); ++i)
gettx_req.txs_hashes.push_back(epee::string_tools::pod_to_hex(proofs[i].txid));
gettx_req.decode_as_json = false;
gettx_req.prune = false;
gettx_req.prune = true;
m_daemon_rpc_mutex.lock();
bool ok = net_utils::invoke_http_json("/gettransactions", gettx_req, gettx_res, m_http_client);
m_daemon_rpc_mutex.unlock();
@ -10590,14 +10631,11 @@ bool wallet2::check_reserve_proof(const cryptonote::account_public_address &addr
const reserve_proof_entry& proof = proofs[i];
THROW_WALLET_EXCEPTION_IF(gettx_res.txs[i].in_pool, error::wallet_internal_error, "Tx is unconfirmed");
cryptonote::blobdata tx_data;
ok = string_tools::parse_hexstr_to_binbuff(gettx_res.txs[i].as_hex, tx_data);
cryptonote::transaction tx;
crypto::hash tx_hash;
ok = get_pruned_tx(gettx_res.txs[i], tx, tx_hash);
THROW_WALLET_EXCEPTION_IF(!ok, error::wallet_internal_error, "Failed to parse transaction from daemon");
crypto::hash tx_hash, tx_prefix_hash;
cryptonote::transaction tx;
THROW_WALLET_EXCEPTION_IF(!cryptonote::parse_and_validate_tx_from_blob(tx_data, tx, tx_hash, tx_prefix_hash), error::wallet_internal_error,
"Failed to validate transaction from daemon");
THROW_WALLET_EXCEPTION_IF(tx_hash != proof.txid, error::wallet_internal_error, "Failed to get the right transaction from daemon");
THROW_WALLET_EXCEPTION_IF(proof.index_in_tx >= tx.vout.size(), error::wallet_internal_error, "index_in_tx is out of bound");
@ -11207,7 +11245,7 @@ uint64_t wallet2::import_key_images(const std::vector<std::pair<crypto::key_imag
COMMAND_RPC_GET_TRANSACTIONS::request gettxs_req;
COMMAND_RPC_GET_TRANSACTIONS::response gettxs_res;
gettxs_req.decode_as_json = false;
gettxs_req.prune = false;
gettxs_req.prune = true;
gettxs_req.txs_hashes.reserve(spent_txids.size());
for (const crypto::hash& spent_txid : spent_txids)
gettxs_req.txs_hashes.push_back(epee::string_tools::pod_to_hex(spent_txid));
@ -11227,17 +11265,16 @@ uint64_t wallet2::import_key_images(const std::vector<std::pair<crypto::key_imag
PERF_TIMER_START(import_key_images_F);
auto spent_txid = spent_txids.begin();
hw::device &hwdev = m_account.get_device();
auto it = spent_txids.begin();
for (const COMMAND_RPC_GET_TRANSACTIONS::entry& e : gettxs_res.txs)
{
THROW_WALLET_EXCEPTION_IF(e.in_pool, error::wallet_internal_error, "spent tx isn't supposed to be in txpool");
// parse tx
cryptonote::blobdata bd;
THROW_WALLET_EXCEPTION_IF(!epee::string_tools::parse_hexstr_to_binbuff(e.as_hex, bd), error::wallet_internal_error, "parse_hexstr_to_binbuff failed");
cryptonote::transaction spent_tx;
crypto::hash spnet_txid_parsed, spent_txid_prefix;
THROW_WALLET_EXCEPTION_IF(!cryptonote::parse_and_validate_tx_from_blob(bd, spent_tx, spnet_txid_parsed, spent_txid_prefix), error::wallet_internal_error, "parse_and_validate_tx_from_blob failed");
THROW_WALLET_EXCEPTION_IF(*spent_txid != spnet_txid_parsed, error::wallet_internal_error, "parsed txid mismatch");
crypto::hash spnet_txid_parsed;
THROW_WALLET_EXCEPTION_IF(!get_pruned_tx(e, spent_tx, spnet_txid_parsed), error::wallet_internal_error, "Failed to get tx from daemon");
THROW_WALLET_EXCEPTION_IF(!(spnet_txid_parsed == *it), error::wallet_internal_error, "parsed txid mismatch");
++it;
// get received (change) amount
uint64_t tx_money_got_in_outs = 0;

2
tests/core_proxy/core_proxy.h
View file

@ -105,5 +105,7 @@ namespace tests
bool fluffy_blocks_enabled() const { return false; }
uint64_t prevalidate_block_hashes(uint64_t height, const std::vector<crypto::hash> &hashes) { return 0; }
bool pad_transactions() const { return false; }
uint32_t get_blockchain_pruning_seed() const { return 0; }
bool prune_blockchain(uint32_t pruning_seed) const { return true; }
};
}

1
tests/unit_tests/CMakeLists.txt
View file

@ -65,6 +65,7 @@ set(unit_tests_sources
notify.cpp
output_distribution.cpp
parse_amount.cpp
pruning.cpp
random.cpp
serialization.cpp
sha256.cpp

2
tests/unit_tests/ban.cpp
View file

@ -84,6 +84,8 @@ public:
bool fluffy_blocks_enabled() const { return false; }
uint64_t prevalidate_block_hashes(uint64_t height, const std::vector<crypto::hash> &hashes) { return 0; }
bool pad_transactions() { return false; }
uint32_t get_blockchain_pruning_seed() const { return 0; }
bool prune_blockchain(uint32_t pruning_seed = 0) { return true; }
void stop() {}
};

240
tests/unit_tests/pruning.cpp Normal file
View file

@ -0,0 +1,240 @@
// Copyright (c) 2018, The Monero Project
//
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without modification, are
// permitted provided that the following conditions are met:
//
// 1. Redistributions of source code must retain the above copyright notice, this list of
// conditions and the following disclaimer.
//
// 2. Redistributions in binary form must reproduce the above copyright notice, this list
// of conditions and the following disclaimer in the documentation and/or other
// materials provided with the distribution.
//
// 3. Neither the name of the copyright holder nor the names of its contributors may be
// used to endorse or promote products derived from this software without specific
// prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include "gtest/gtest.h"
#include "misc_log_ex.h"
#include "cryptonote_config.h"
#include "common/pruning.h"
#define ASSERT_EX(x) do { bool ex = false; try { x; } catch(...) { ex = true; } ASSERT_TRUE(ex); } while(0)
TEST(pruning, parts)
{
ASSERT_EQ(tools::get_pruning_stripe(tools::make_pruning_seed(3, 2)), 3);
ASSERT_EQ(tools::get_pruning_stripe(tools::make_pruning_seed(1, 2)), 1);
ASSERT_EQ(tools::get_pruning_stripe(tools::make_pruning_seed(7, 7)), 7);
ASSERT_EQ(tools::get_pruning_log_stripes(tools::make_pruning_seed(1, 2)), 2);
ASSERT_EQ(tools::get_pruning_log_stripes(tools::make_pruning_seed(1, 0)), 0);
ASSERT_EQ(tools::get_pruning_log_stripes(tools::make_pruning_seed(1, 7)), 7);
ASSERT_EQ(tools::get_pruning_log_stripes(tools::make_pruning_seed(7, 7)), 7);
for (uint32_t log_stripes = 1; log_stripes <= tools::PRUNING_SEED_LOG_STRIPES_MASK; ++log_stripes)
{
for (uint32_t stripe = 1; stripe <= (1 << log_stripes); ++stripe)
{
ASSERT_EQ(tools::get_pruning_log_stripes(tools::make_pruning_seed(stripe, log_stripes)), log_stripes);
ASSERT_EQ(tools::get_pruning_stripe(tools::make_pruning_seed(stripe, log_stripes)), stripe);
}
}
}
TEST(pruning, out_of_range)
{
ASSERT_EX(tools::make_pruning_seed(5, 2));
ASSERT_EX(tools::make_pruning_seed(0, 2));
}
TEST(pruning, fits)
{
const uint32_t log_stripes = 3;
const uint32_t num_stripes = 1 << log_stripes;
for (uint32_t stripe = 1; stripe <= num_stripes; ++stripe)
{
const uint32_t seed = tools::make_pruning_seed(stripe, log_stripes);
ASSERT_NE(seed, 0);
ASSERT_EQ(tools::get_pruning_log_stripes(seed), log_stripes);
ASSERT_EQ(tools::get_pruning_stripe(seed), stripe);
}
}
TEST(pruning, tip)
{
static constexpr uint64_t H = CRYPTONOTE_PRUNING_TIP_BLOCKS + 1000;
static_assert(H >= CRYPTONOTE_PRUNING_TIP_BLOCKS, "H must be >= CRYPTONOTE_PRUNING_TIP_BLOCKS");
for (uint64_t h = H - CRYPTONOTE_PRUNING_TIP_BLOCKS; h < H; ++h)
{
uint32_t pruning_seed = tools::get_pruning_seed(h, H, CRYPTONOTE_PRUNING_LOG_STRIPES);
ASSERT_EQ(pruning_seed, 0);
for (pruning_seed = 0; pruning_seed <= (1 << CRYPTONOTE_PRUNING_LOG_STRIPES); ++pruning_seed)
ASSERT_TRUE(tools::has_unpruned_block(h, H, pruning_seed));
}
}
TEST(pruning, seed)
{
const uint64_t SS = CRYPTONOTE_PRUNING_STRIPE_SIZE;
const uint64_t NS = 1 << CRYPTONOTE_PRUNING_LOG_STRIPES;
const uint64_t TB = NS * SS;
for (uint64_t cycle = 0; cycle < 10; ++cycle)
{
const uint64_t O = TB * cycle;
ASSERT_EQ(tools::get_pruning_stripe(O + 0, 10000000, CRYPTONOTE_PRUNING_LOG_STRIPES), 1);
ASSERT_EQ(tools::get_pruning_stripe(O + 1, 10000000, CRYPTONOTE_PRUNING_LOG_STRIPES), 1);
ASSERT_EQ(tools::get_pruning_stripe(O + SS-1, 10000000, CRYPTONOTE_PRUNING_LOG_STRIPES), 1);
ASSERT_EQ(tools::get_pruning_stripe(O + SS, 10000000, CRYPTONOTE_PRUNING_LOG_STRIPES), 2);
ASSERT_EQ(tools::get_pruning_stripe(O + SS*2-1, 10000000, CRYPTONOTE_PRUNING_LOG_STRIPES), 2);
ASSERT_EQ(tools::get_pruning_stripe(O + SS*2, 10000000, CRYPTONOTE_PRUNING_LOG_STRIPES), 3);
ASSERT_EQ(tools::get_pruning_stripe(O + SS*NS-1, 10000000, CRYPTONOTE_PRUNING_LOG_STRIPES), NS);
ASSERT_EQ(tools::get_pruning_stripe(O + SS*NS, 10000000, CRYPTONOTE_PRUNING_LOG_STRIPES), 1);
}
}
TEST(pruning, match)
{
static constexpr uint64_t H = CRYPTONOTE_PRUNING_TIP_BLOCKS + 1000;
static_assert(H >= CRYPTONOTE_PRUNING_TIP_BLOCKS, "H must be >= CRYPTONOTE_PRUNING_TIP_BLOCKS");
for (uint64_t h = 0; h < H - CRYPTONOTE_PRUNING_TIP_BLOCKS; ++h)
{
uint32_t pruning_seed = tools::get_pruning_seed(h, H, CRYPTONOTE_PRUNING_LOG_STRIPES);
uint32_t pruning_stripe = tools::get_pruning_stripe(pruning_seed);
ASSERT_TRUE(pruning_stripe > 0 && pruning_stripe <= (1 << CRYPTONOTE_PRUNING_LOG_STRIPES));
for (uint32_t other_pruning_stripe = 1; other_pruning_stripe <= (1 << CRYPTONOTE_PRUNING_LOG_STRIPES); ++other_pruning_stripe)
{
uint32_t other_pruning_seed = tools::make_pruning_seed(other_pruning_stripe, CRYPTONOTE_PRUNING_LOG_STRIPES);
ASSERT_TRUE(tools::has_unpruned_block(h, H, other_pruning_seed) == (other_pruning_seed == pruning_seed));
}
}
}
TEST(pruning, stripe_size)
{
static constexpr uint64_t H = CRYPTONOTE_PRUNING_TIP_BLOCKS + CRYPTONOTE_PRUNING_STRIPE_SIZE * (1 << CRYPTONOTE_PRUNING_LOG_STRIPES) + 1000;
static_assert(H >= CRYPTONOTE_PRUNING_TIP_BLOCKS + CRYPTONOTE_PRUNING_STRIPE_SIZE * (1 << CRYPTONOTE_PRUNING_LOG_STRIPES), "H must be >= that stuff in front");
for (uint32_t pruning_stripe = 1; pruning_stripe <= (1 << CRYPTONOTE_PRUNING_LOG_STRIPES); ++pruning_stripe)
{
uint32_t pruning_seed = tools::make_pruning_seed(pruning_stripe, CRYPTONOTE_PRUNING_LOG_STRIPES);
unsigned int current_run = 0, best_run = 0;
for (uint64_t h = 0; h < H - CRYPTONOTE_PRUNING_TIP_BLOCKS; ++h)
{
if (tools::has_unpruned_block(h, H, pruning_seed))
{
++current_run;
}
else if (current_run)
{
ASSERT_EQ(current_run, CRYPTONOTE_PRUNING_STRIPE_SIZE);
best_run = std::max(best_run, current_run);
current_run = 0;
}
}
ASSERT_EQ(best_run, CRYPTONOTE_PRUNING_STRIPE_SIZE);
}
}
TEST(pruning, next_unpruned)
{
static_assert((1 << CRYPTONOTE_PRUNING_LOG_STRIPES) >= 4, "CRYPTONOTE_PRUNING_LOG_STRIPES too low");
const uint64_t SS = CRYPTONOTE_PRUNING_STRIPE_SIZE;
const uint64_t NS = 1 << CRYPTONOTE_PRUNING_LOG_STRIPES;
const uint64_t TB = NS * SS;
for (uint64_t h = 0; h < 100; ++h)
ASSERT_EQ(tools::get_next_unpruned_block_height(h, 10000000, 0), h);
const uint32_t seed1 = tools::make_pruning_seed(1, CRYPTONOTE_PRUNING_LOG_STRIPES);
const uint32_t seed2 = tools::make_pruning_seed(2, CRYPTONOTE_PRUNING_LOG_STRIPES);
const uint32_t seed3 = tools::make_pruning_seed(3, CRYPTONOTE_PRUNING_LOG_STRIPES);
const uint32_t seed4 = tools::make_pruning_seed(4, CRYPTONOTE_PRUNING_LOG_STRIPES);
const uint32_t seedNS = tools::make_pruning_seed(NS, CRYPTONOTE_PRUNING_LOG_STRIPES);
ASSERT_EQ(tools::get_next_unpruned_block_height(0, 10000000, seed1), 0);
ASSERT_EQ(tools::get_next_unpruned_block_height(1, 10000000, seed1), 1);
ASSERT_EQ(tools::get_next_unpruned_block_height(SS-1, 10000000, seed1), SS-1);
ASSERT_EQ(tools::get_next_unpruned_block_height(SS, 10000000, seed1), TB);
ASSERT_EQ(tools::get_next_unpruned_block_height(TB, 10000000, seed1), TB);
ASSERT_EQ(tools::get_next_unpruned_block_height(TB-1, 10000000, seed1), TB);
ASSERT_EQ(tools::get_next_unpruned_block_height(0, 10000000, seed2), SS);
ASSERT_EQ(tools::get_next_unpruned_block_height(1, 10000000, seed2), SS);
ASSERT_EQ(tools::get_next_unpruned_block_height(SS-1, 10000000, seed2), SS);
ASSERT_EQ(tools::get_next_unpruned_block_height(SS, 10000000, seed2), SS);
ASSERT_EQ(tools::get_next_unpruned_block_height(2*SS-1, 10000000, seed2), 2*SS-1);
ASSERT_EQ(tools::get_next_unpruned_block_height(2*SS, 10000000, seed2), TB+SS);
ASSERT_EQ(tools::get_next_unpruned_block_height(TB+2*SS,10000000, seed2), TB*2+SS);
ASSERT_EQ(tools::get_next_unpruned_block_height(0, 10000000, seed3), SS*2);
ASSERT_EQ(tools::get_next_unpruned_block_height(SS, 10000000, seed3), SS*2);
ASSERT_EQ(tools::get_next_unpruned_block_height(2*SS, 10000000, seed3), SS*2);
ASSERT_EQ(tools::get_next_unpruned_block_height(3*SS-1, 10000000, seed3), SS*3-1);
ASSERT_EQ(tools::get_next_unpruned_block_height(3*SS, 10000000, seed3), TB+SS*2);
ASSERT_EQ(tools::get_next_unpruned_block_height(TB+3*SS,10000000, seed3), TB*2+SS*2);
ASSERT_EQ(tools::get_next_unpruned_block_height(SS, 10000000, seed4), 3*SS);
ASSERT_EQ(tools::get_next_unpruned_block_height(4*SS-1, 10000000, seed4), 4*SS-1);
ASSERT_EQ(tools::get_next_unpruned_block_height(4*SS, 10000000, seed4), TB+3*SS);
ASSERT_EQ(tools::get_next_unpruned_block_height(TB+4*SS,10000000, seed4), TB*2+3*SS);
ASSERT_EQ(tools::get_next_unpruned_block_height(SS, 10000000, seedNS), (NS-1)*SS);
ASSERT_EQ(tools::get_next_unpruned_block_height(NS*SS-1,10000000, seedNS), NS*SS-1);
ASSERT_EQ(tools::get_next_unpruned_block_height(NS*SS, 10000000, seedNS), TB+(NS-1)*SS);
ASSERT_EQ(tools::get_next_unpruned_block_height(TB+NS*SS, 10000000, seedNS), TB*2+(NS-1)*SS);
}
TEST(pruning, next_pruned)
{
static_assert((1 << CRYPTONOTE_PRUNING_LOG_STRIPES) >= 4, "CRYPTONOTE_PRUNING_LOG_STRIPES too low");
const uint64_t SS = CRYPTONOTE_PRUNING_STRIPE_SIZE;
const uint64_t NS = 1 << CRYPTONOTE_PRUNING_LOG_STRIPES;
const uint64_t TB = NS * SS;
const uint32_t seed1 = tools::make_pruning_seed(1, CRYPTONOTE_PRUNING_LOG_STRIPES);
const uint32_t seed2 = tools::make_pruning_seed(2, CRYPTONOTE_PRUNING_LOG_STRIPES);
const uint32_t seedNS_1 = tools::make_pruning_seed(NS-1, CRYPTONOTE_PRUNING_LOG_STRIPES);
const uint32_t seedNS = tools::make_pruning_seed(NS, CRYPTONOTE_PRUNING_LOG_STRIPES);
for (uint64_t h = 0; h < 100; ++h)
ASSERT_EQ(tools::get_next_pruned_block_height(h, 10000000, 0), 10000000);
for (uint64_t h = 10000000 - 1 - CRYPTONOTE_PRUNING_TIP_BLOCKS; h < 10000000; ++h)
ASSERT_EQ(tools::get_next_pruned_block_height(h, 10000000, 0), 10000000);
ASSERT_EQ(tools::get_next_pruned_block_height(1, 10000000, seed1), SS);
ASSERT_EQ(tools::get_next_pruned_block_height(SS-1, 10000000, seed1), SS);
ASSERT_EQ(tools::get_next_pruned_block_height(SS, 10000000, seed1), SS);
ASSERT_EQ(tools::get_next_pruned_block_height(TB-1, 10000000, seed1), TB-1);
ASSERT_EQ(tools::get_next_pruned_block_height(TB, 10000000, seed1), TB+SS);
ASSERT_EQ(tools::get_next_pruned_block_height(1, 10000000, seed2), 1);
ASSERT_EQ(tools::get_next_pruned_block_height(SS-1, 10000000, seed2), SS-1);
ASSERT_EQ(tools::get_next_pruned_block_height(SS, 10000000, seed2), 2*SS);
ASSERT_EQ(tools::get_next_pruned_block_height(TB-1, 10000000, seed2), TB-1);
ASSERT_EQ(tools::get_next_pruned_block_height(1, 10000000, seedNS_1), 1);
ASSERT_EQ(tools::get_next_pruned_block_height(SS-1, 10000000, seedNS_1), SS-1);
ASSERT_EQ(tools::get_next_pruned_block_height(SS, 10000000, seedNS_1), SS);
ASSERT_EQ(tools::get_next_pruned_block_height(TB-1, 10000000, seedNS_1), TB-1);
ASSERT_EQ(tools::get_next_pruned_block_height(1, 10000000, seedNS), 1);
ASSERT_EQ(tools::get_next_pruned_block_height(SS-1, 10000000, seedNS), SS-1);
ASSERT_EQ(tools::get_next_pruned_block_height(SS, 10000000, seedNS), SS);
ASSERT_EQ(tools::get_next_pruned_block_height(TB-1, 10000000, seedNS), TB);
}