mirror of
https://git.wownero.com/wownero/wownero.git
synced 2024-08-15 01:03:23 +00:00
Merge pull request #2469
7adceee6
precomputed block hashes are now in blocks of N (currently 256) (moneromooo-monero)
This commit is contained in:
commit
5ea20d6944
12 changed files with 182 additions and 12 deletions
|
@ -169,6 +169,26 @@ int check_flush(cryptonote::core &core, std::list<block_complete_entry> &blocks,
|
||||||
if (!force && blocks.size() < db_batch_size)
|
if (!force && blocks.size() < db_batch_size)
|
||||||
return 0;
|
return 0;
|
||||||
|
|
||||||
|
// wait till we can verify a full HOH without extra, for speed
|
||||||
|
uint64_t new_height = core.get_blockchain_storage().get_db().height() + blocks.size();
|
||||||
|
if (!force && new_height % HASH_OF_HASHES_STEP)
|
||||||
|
return 0;
|
||||||
|
|
||||||
|
std::list<crypto::hash> hashes;
|
||||||
|
for (const auto &b: blocks)
|
||||||
|
{
|
||||||
|
cryptonote::block block;
|
||||||
|
if (!parse_and_validate_block_from_blob(b.block, block))
|
||||||
|
{
|
||||||
|
MERROR("Failed to parse block: "
|
||||||
|
<< epee::string_tools::pod_to_hex(get_blob_hash(b.block)));
|
||||||
|
core.cleanup_handle_incoming_blocks();
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
hashes.push_back(cryptonote::get_block_hash(block));
|
||||||
|
}
|
||||||
|
core.prevalidate_block_hashes(core.get_blockchain_storage().get_db().height(), hashes);
|
||||||
|
|
||||||
core.prepare_handle_incoming_blocks(blocks);
|
core.prepare_handle_incoming_blocks(blocks);
|
||||||
|
|
||||||
for(const block_complete_entry& block_entry: blocks)
|
for(const block_complete_entry& block_entry: blocks)
|
||||||
|
|
|
@ -82,7 +82,7 @@ bool BlocksdatFile::open_writer(const boost::filesystem::path& file_path, uint64
|
||||||
|
|
||||||
bool BlocksdatFile::initialize_file(uint64_t block_stop)
|
bool BlocksdatFile::initialize_file(uint64_t block_stop)
|
||||||
{
|
{
|
||||||
const uint32_t nblocks = block_stop + 1;
|
const uint32_t nblocks = (block_stop + 1) / HASH_OF_HASHES_STEP;
|
||||||
unsigned char nblocksc[4];
|
unsigned char nblocksc[4];
|
||||||
|
|
||||||
nblocksc[0] = nblocks & 0xff;
|
nblocksc[0] = nblocks & 0xff;
|
||||||
|
@ -101,8 +101,16 @@ bool BlocksdatFile::initialize_file(uint64_t block_stop)
|
||||||
|
|
||||||
void BlocksdatFile::write_block(const crypto::hash& block_hash)
|
void BlocksdatFile::write_block(const crypto::hash& block_hash)
|
||||||
{
|
{
|
||||||
const std::string data(block_hash.data, sizeof(block_hash));
|
m_hashes.push_back(block_hash);
|
||||||
*m_raw_data_file << data;
|
while (m_hashes.size() >= HASH_OF_HASHES_STEP)
|
||||||
|
{
|
||||||
|
crypto::hash hash;
|
||||||
|
crypto::cn_fast_hash(m_hashes.data(), HASH_OF_HASHES_STEP * sizeof(crypto::hash), hash);
|
||||||
|
memmove(m_hashes.data(), m_hashes.data() + HASH_OF_HASHES_STEP * sizeof(crypto::hash), (m_hashes.size() - HASH_OF_HASHES_STEP) * sizeof(crypto::hash));
|
||||||
|
m_hashes.resize(m_hashes.size() - HASH_OF_HASHES_STEP);
|
||||||
|
const std::string data(hash.data, sizeof(hash));
|
||||||
|
*m_raw_data_file << data;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
bool BlocksdatFile::close()
|
bool BlocksdatFile::close()
|
||||||
|
|
|
@ -76,4 +76,5 @@ protected:
|
||||||
private:
|
private:
|
||||||
|
|
||||||
uint64_t m_cur_height; // tracks current height during export
|
uint64_t m_cur_height; // tracks current height during export
|
||||||
|
std::vector<crypto::hash> m_hashes;
|
||||||
};
|
};
|
||||||
|
|
Binary file not shown.
|
@ -136,6 +136,9 @@
|
||||||
|
|
||||||
#define PER_KB_FEE_QUANTIZATION_DECIMALS 8
|
#define PER_KB_FEE_QUANTIZATION_DECIMALS 8
|
||||||
|
|
||||||
|
#define HASH_OF_HASHES_STEP 256
|
||||||
|
|
||||||
|
|
||||||
// New constants are intended to go here
|
// New constants are intended to go here
|
||||||
namespace config
|
namespace config
|
||||||
{
|
{
|
||||||
|
|
|
@ -3224,13 +3224,21 @@ leave:
|
||||||
if (m_db->height() < m_blocks_hash_check.size())
|
if (m_db->height() < m_blocks_hash_check.size())
|
||||||
{
|
{
|
||||||
auto hash = get_block_hash(bl);
|
auto hash = get_block_hash(bl);
|
||||||
if (memcmp(&hash, &m_blocks_hash_check[m_db->height()], sizeof(hash)) != 0)
|
const auto &expected_hash = m_blocks_hash_check[m_db->height()];
|
||||||
|
if (expected_hash != cryptonote::null_hash)
|
||||||
{
|
{
|
||||||
MERROR_VER("Block with id is INVALID: " << id);
|
if (memcmp(&hash, &expected_hash, sizeof(hash)) != 0)
|
||||||
bvc.m_verifivation_failed = true;
|
{
|
||||||
goto leave;
|
MERROR_VER("Block with id is INVALID: " << id);
|
||||||
|
bvc.m_verifivation_failed = true;
|
||||||
|
goto leave;
|
||||||
|
}
|
||||||
|
fast_check = true;
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
MCINFO("verify", "No pre-validated hash at height " << m_db->height() << ", verifying fully");
|
||||||
}
|
}
|
||||||
fast_check = true;
|
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
#endif
|
#endif
|
||||||
|
@ -3675,6 +3683,14 @@ bool Blockchain::cleanup_handle_incoming_blocks(bool force_sync)
|
||||||
m_blocks_txs_check.clear();
|
m_blocks_txs_check.clear();
|
||||||
m_check_txin_table.clear();
|
m_check_txin_table.clear();
|
||||||
|
|
||||||
|
// when we're well clear of the precomputed hashes, free the memory
|
||||||
|
if (!m_blocks_hash_check.empty() && m_db->height() > m_blocks_hash_check.size() + 4096)
|
||||||
|
{
|
||||||
|
MINFO("Dumping block hashes, we're now 4k past " << m_blocks_hash_check.size());
|
||||||
|
m_blocks_hash_check.clear();
|
||||||
|
m_blocks_hash_check.shrink_to_fit();
|
||||||
|
}
|
||||||
|
|
||||||
CRITICAL_REGION_END();
|
CRITICAL_REGION_END();
|
||||||
m_tx_pool.unlock();
|
m_tx_pool.unlock();
|
||||||
|
|
||||||
|
@ -3699,6 +3715,98 @@ void Blockchain::output_scan_worker(const uint64_t amount, const std::vector<uin
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
uint64_t Blockchain::prevalidate_block_hashes(uint64_t height, const std::list<crypto::hash> &hashes)
|
||||||
|
{
|
||||||
|
// new: . . . . . X X X X X . . . . . .
|
||||||
|
// pre: A A A A B B B B C C C C D D D D
|
||||||
|
|
||||||
|
// easy case: height >= hashes
|
||||||
|
if (height >= m_blocks_hash_of_hashes.size() * HASH_OF_HASHES_STEP)
|
||||||
|
return hashes.size();
|
||||||
|
|
||||||
|
// find hashes encompassing those block
|
||||||
|
size_t first_index = height / HASH_OF_HASHES_STEP;
|
||||||
|
size_t last_index = (height + hashes.size() - 1) / HASH_OF_HASHES_STEP;
|
||||||
|
MDEBUG("Blocks " << height << " - " << (height + hashes.size() - 1) << " start at " << first_index << " and end at " << last_index);
|
||||||
|
|
||||||
|
// case of not enough to calculate even a single hash
|
||||||
|
if (first_index == last_index && hashes.size() < HASH_OF_HASHES_STEP && (height + hashes.size()) % HASH_OF_HASHES_STEP)
|
||||||
|
return hashes.size();
|
||||||
|
|
||||||
|
// build hashes vector to hash hashes together
|
||||||
|
std::vector<crypto::hash> data;
|
||||||
|
data.reserve(hashes.size() + HASH_OF_HASHES_STEP - 1); // may be a bit too much
|
||||||
|
|
||||||
|
// we expect height to be either equal or a bit below db height
|
||||||
|
bool disconnected = (height > m_db->height());
|
||||||
|
size_t pop;
|
||||||
|
if (disconnected && height % HASH_OF_HASHES_STEP)
|
||||||
|
{
|
||||||
|
++first_index;
|
||||||
|
pop = HASH_OF_HASHES_STEP - height % HASH_OF_HASHES_STEP;
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
// we might need some already in the chain for the first part of the first hash
|
||||||
|
for (uint64_t h = first_index * HASH_OF_HASHES_STEP; h < height; ++h)
|
||||||
|
{
|
||||||
|
data.push_back(m_db->get_block_hash_from_height(h));
|
||||||
|
}
|
||||||
|
pop = 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
// push the data to check
|
||||||
|
for (const auto &h: hashes)
|
||||||
|
{
|
||||||
|
if (pop)
|
||||||
|
--pop;
|
||||||
|
else
|
||||||
|
data.push_back(h);
|
||||||
|
}
|
||||||
|
|
||||||
|
// hash and check
|
||||||
|
uint64_t usable = first_index * HASH_OF_HASHES_STEP - height; // may start negative, but unsigned under/overflow is not UB
|
||||||
|
for (size_t n = first_index; n <= last_index; ++n)
|
||||||
|
{
|
||||||
|
if (n < m_blocks_hash_of_hashes.size())
|
||||||
|
{
|
||||||
|
// if the last index isn't fully filled, we can't tell if valid
|
||||||
|
if (data.size() < (n - first_index) * HASH_OF_HASHES_STEP + HASH_OF_HASHES_STEP)
|
||||||
|
break;
|
||||||
|
|
||||||
|
crypto::hash hash;
|
||||||
|
cn_fast_hash(data.data() + (n - first_index) * HASH_OF_HASHES_STEP, HASH_OF_HASHES_STEP * sizeof(crypto::hash), hash);
|
||||||
|
bool valid = hash == m_blocks_hash_of_hashes[n];
|
||||||
|
|
||||||
|
// add to the known hashes array
|
||||||
|
if (!valid)
|
||||||
|
{
|
||||||
|
MWARNING("invalid hash for blocks " << n * HASH_OF_HASHES_STEP << " - " << (n * HASH_OF_HASHES_STEP + HASH_OF_HASHES_STEP - 1));
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
size_t end = n * HASH_OF_HASHES_STEP + HASH_OF_HASHES_STEP;
|
||||||
|
for (size_t i = n * HASH_OF_HASHES_STEP; i < end; ++i)
|
||||||
|
{
|
||||||
|
CHECK_AND_ASSERT_MES(m_blocks_hash_check[i] == cryptonote::null_hash || m_blocks_hash_check[i] == data[i - first_index * HASH_OF_HASHES_STEP],
|
||||||
|
0, "Consistency failure in m_blocks_hash_check construction");
|
||||||
|
m_blocks_hash_check[i] = data[i - first_index * HASH_OF_HASHES_STEP];
|
||||||
|
}
|
||||||
|
usable += HASH_OF_HASHES_STEP;
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
// if after the end of the precomputed blocks, accept anything
|
||||||
|
usable += HASH_OF_HASHES_STEP;
|
||||||
|
if (usable > hashes.size())
|
||||||
|
usable = hashes.size();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
MDEBUG("usable: " << usable << " / " << hashes.size());
|
||||||
|
CHECK_AND_ASSERT_MES(usable < std::numeric_limits<uint64_t>::max() / 2, 0, "usable is negative");
|
||||||
|
return usable;
|
||||||
|
}
|
||||||
|
|
||||||
//------------------------------------------------------------------
|
//------------------------------------------------------------------
|
||||||
// ND: Speedups:
|
// ND: Speedups:
|
||||||
// 1. Thread long_hash computations if possible (m_max_prepare_blocks_threads = nthreads, default = 4)
|
// 1. Thread long_hash computations if possible (m_max_prepare_blocks_threads = nthreads, default = 4)
|
||||||
|
@ -4171,7 +4279,7 @@ void Blockchain::cancel()
|
||||||
}
|
}
|
||||||
|
|
||||||
#if defined(PER_BLOCK_CHECKPOINT)
|
#if defined(PER_BLOCK_CHECKPOINT)
|
||||||
static const char expected_block_hashes_hash[] = "d3ca80d50661684cde0e715d46d7c19704d2e216b21ed088af9fd4ef37ed4d65";
|
static const char expected_block_hashes_hash[] = "4b553162ee4e7af3c53666506591489c68560b9175e6e941dc96c89f96f0e35c";
|
||||||
void Blockchain::load_compiled_in_block_hashes()
|
void Blockchain::load_compiled_in_block_hashes()
|
||||||
{
|
{
|
||||||
if (m_fast_sync && get_blocks_dat_start(m_testnet) != nullptr && get_blocks_dat_size(m_testnet) > 0)
|
if (m_fast_sync && get_blocks_dat_start(m_testnet) != nullptr && get_blocks_dat_size(m_testnet) > 0)
|
||||||
|
@ -4207,16 +4315,18 @@ void Blockchain::load_compiled_in_block_hashes()
|
||||||
const unsigned char *p = get_blocks_dat_start(m_testnet);
|
const unsigned char *p = get_blocks_dat_start(m_testnet);
|
||||||
const uint32_t nblocks = *p | ((*(p+1))<<8) | ((*(p+2))<<16) | ((*(p+3))<<24);
|
const uint32_t nblocks = *p | ((*(p+1))<<8) | ((*(p+2))<<16) | ((*(p+3))<<24);
|
||||||
const size_t size_needed = 4 + nblocks * sizeof(crypto::hash);
|
const size_t size_needed = 4 + nblocks * sizeof(crypto::hash);
|
||||||
if(nblocks > 0 && nblocks > m_db->height() && get_blocks_dat_size(m_testnet) >= size_needed)
|
if(nblocks > 0 && nblocks * HASH_OF_HASHES_STEP > m_db->height() && get_blocks_dat_size(m_testnet) >= size_needed)
|
||||||
{
|
{
|
||||||
p += sizeof(uint32_t);
|
p += sizeof(uint32_t);
|
||||||
|
m_blocks_hash_of_hashes.reserve(nblocks);
|
||||||
for (uint32_t i = 0; i < nblocks; i++)
|
for (uint32_t i = 0; i < nblocks; i++)
|
||||||
{
|
{
|
||||||
crypto::hash hash;
|
crypto::hash hash;
|
||||||
memcpy(hash.data, p, sizeof(hash.data));
|
memcpy(hash.data, p, sizeof(hash.data));
|
||||||
p += sizeof(hash.data);
|
p += sizeof(hash.data);
|
||||||
m_blocks_hash_check.push_back(hash);
|
m_blocks_hash_of_hashes.push_back(hash);
|
||||||
}
|
}
|
||||||
|
m_blocks_hash_check.resize(m_blocks_hash_of_hashes.size() * HASH_OF_HASHES_STEP, cryptonote::null_hash);
|
||||||
MINFO(nblocks << " block hashes loaded");
|
MINFO(nblocks << " block hashes loaded");
|
||||||
|
|
||||||
// FIXME: clear tx_pool because the process might have been
|
// FIXME: clear tx_pool because the process might have been
|
||||||
|
@ -4247,7 +4357,7 @@ void Blockchain::load_compiled_in_block_hashes()
|
||||||
bool Blockchain::is_within_compiled_block_hash_area(uint64_t height) const
|
bool Blockchain::is_within_compiled_block_hash_area(uint64_t height) const
|
||||||
{
|
{
|
||||||
#if defined(PER_BLOCK_CHECKPOINT)
|
#if defined(PER_BLOCK_CHECKPOINT)
|
||||||
return height < m_blocks_hash_check.size();
|
return height < m_blocks_hash_of_hashes.size() * HASH_OF_HASHES_STEP;
|
||||||
#else
|
#else
|
||||||
return false;
|
return false;
|
||||||
#endif
|
#endif
|
||||||
|
|
|
@ -955,6 +955,7 @@ namespace cryptonote
|
||||||
|
|
||||||
bool is_within_compiled_block_hash_area(uint64_t height) const;
|
bool is_within_compiled_block_hash_area(uint64_t height) const;
|
||||||
bool is_within_compiled_block_hash_area() const { return is_within_compiled_block_hash_area(m_db->height()); }
|
bool is_within_compiled_block_hash_area() const { return is_within_compiled_block_hash_area(m_db->height()); }
|
||||||
|
uint64_t prevalidate_block_hashes(uint64_t height, const std::list<crypto::hash> &hashes);
|
||||||
|
|
||||||
void lock();
|
void lock();
|
||||||
void unlock();
|
void unlock();
|
||||||
|
@ -995,6 +996,7 @@ namespace cryptonote
|
||||||
std::unordered_map<crypto::hash, std::unordered_map<crypto::key_image, bool>> m_check_txin_table;
|
std::unordered_map<crypto::hash, std::unordered_map<crypto::key_image, bool>> m_check_txin_table;
|
||||||
|
|
||||||
// SHA-3 hashes for each block and for fast pow checking
|
// SHA-3 hashes for each block and for fast pow checking
|
||||||
|
std::vector<crypto::hash> m_blocks_hash_of_hashes;
|
||||||
std::vector<crypto::hash> m_blocks_hash_check;
|
std::vector<crypto::hash> m_blocks_hash_check;
|
||||||
std::vector<crypto::hash> m_blocks_txs_check;
|
std::vector<crypto::hash> m_blocks_txs_check;
|
||||||
|
|
||||||
|
|
|
@ -1399,6 +1399,11 @@ namespace cryptonote
|
||||||
return m_target_blockchain_height;
|
return m_target_blockchain_height;
|
||||||
}
|
}
|
||||||
//-----------------------------------------------------------------------------------------------
|
//-----------------------------------------------------------------------------------------------
|
||||||
|
uint64_t core::prevalidate_block_hashes(uint64_t height, const std::list<crypto::hash> &hashes)
|
||||||
|
{
|
||||||
|
return get_blockchain_storage().prevalidate_block_hashes(height, hashes);
|
||||||
|
}
|
||||||
|
//-----------------------------------------------------------------------------------------------
|
||||||
std::time_t core::get_start_time() const
|
std::time_t core::get_start_time() const
|
||||||
{
|
{
|
||||||
return start_time;
|
return start_time;
|
||||||
|
|
|
@ -750,6 +750,13 @@ namespace cryptonote
|
||||||
*/
|
*/
|
||||||
bool fluffy_blocks_enabled() const { return m_fluffy_blocks_enabled; }
|
bool fluffy_blocks_enabled() const { return m_fluffy_blocks_enabled; }
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @brief check a set of hashes against the precompiled hash set
|
||||||
|
*
|
||||||
|
* @return number of usable blocks
|
||||||
|
*/
|
||||||
|
uint64_t prevalidate_block_hashes(uint64_t height, const std::list<crypto::hash> &hashes);
|
||||||
|
|
||||||
private:
|
private:
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
|
@ -1579,10 +1579,22 @@ skip:
|
||||||
return 1;
|
return 1;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
uint64_t n_use_blocks = m_core.prevalidate_block_hashes(arg.start_height, arg.m_block_ids);
|
||||||
|
if (n_use_blocks == 0)
|
||||||
|
{
|
||||||
|
LOG_ERROR_CCONTEXT("Peer yielded no usable blocks, dropping connection");
|
||||||
|
drop_connection(context, false, false);
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
uint64_t added = 0;
|
||||||
for(auto& bl_id: arg.m_block_ids)
|
for(auto& bl_id: arg.m_block_ids)
|
||||||
{
|
{
|
||||||
context.m_needed_objects.push_back(bl_id);
|
context.m_needed_objects.push_back(bl_id);
|
||||||
|
if (++added == n_use_blocks)
|
||||||
|
break;
|
||||||
}
|
}
|
||||||
|
context.m_last_response_height -= arg.m_block_ids.size() - n_use_blocks;
|
||||||
|
|
||||||
if (!request_missing_objects(context, false))
|
if (!request_missing_objects(context, false))
|
||||||
{
|
{
|
||||||
|
|
|
@ -101,5 +101,6 @@ namespace tests
|
||||||
uint8_t get_hard_fork_version(uint64_t height) const { return 0; }
|
uint8_t get_hard_fork_version(uint64_t height) const { return 0; }
|
||||||
cryptonote::difficulty_type get_block_cumulative_difficulty(uint64_t height) const { return 0; }
|
cryptonote::difficulty_type get_block_cumulative_difficulty(uint64_t height) const { return 0; }
|
||||||
bool fluffy_blocks_enabled() const { return false; }
|
bool fluffy_blocks_enabled() const { return false; }
|
||||||
|
uint64_t prevalidate_block_hashes(uint64_t height, const std::list<crypto::hash> &hashes) { return 0; }
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
|
@ -78,6 +78,7 @@ public:
|
||||||
uint8_t get_hard_fork_version(uint64_t height) const { return 0; }
|
uint8_t get_hard_fork_version(uint64_t height) const { return 0; }
|
||||||
cryptonote::difficulty_type get_block_cumulative_difficulty(uint64_t height) const { return 0; }
|
cryptonote::difficulty_type get_block_cumulative_difficulty(uint64_t height) const { return 0; }
|
||||||
bool fluffy_blocks_enabled() const { return false; }
|
bool fluffy_blocks_enabled() const { return false; }
|
||||||
|
uint64_t prevalidate_block_hashes(uint64_t height, const std::list<crypto::hash> &hashes) { return 0; }
|
||||||
};
|
};
|
||||||
|
|
||||||
typedef nodetool::node_server<cryptonote::t_cryptonote_protocol_handler<test_core>> Server;
|
typedef nodetool::node_server<cryptonote::t_cryptonote_protocol_handler<test_core>> Server;
|
||||||
|
|
Loading…
Reference in a new issue