From 5dc590cbdb22d22ca59f6964472da7df4345b3a0 Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Mon, 3 Dec 2018 17:45:18 +0000 Subject: [PATCH] wallet2: speedup output tracking It can get heavy for large wallets --- src/wallet/wallet2.cpp | 55 ++++++++++++++++++++++++++++++++++-------- src/wallet/wallet2.h | 7 +++--- 2 files changed, 49 insertions(+), 13 deletions(-) diff --git a/src/wallet/wallet2.cpp b/src/wallet/wallet2.cpp index f8a11f3ad..98f0a7f68 100644 --- a/src/wallet/wallet2.cpp +++ b/src/wallet/wallet2.cpp @@ -1447,7 +1447,7 @@ void wallet2::cache_tx_data(const cryptonote::transaction& tx, const crypto::has } } //---------------------------------------------------------------------------------------------------- -void wallet2::process_new_transaction(const crypto::hash &txid, const cryptonote::transaction& tx, const std::vector &o_indices, uint64_t height, uint64_t ts, bool miner_tx, bool pool, bool double_spend_seen, const tx_cache_data &tx_cache_data) +void wallet2::process_new_transaction(const crypto::hash &txid, const cryptonote::transaction& tx, const std::vector &o_indices, uint64_t height, uint64_t ts, bool miner_tx, bool pool, bool double_spend_seen, const tx_cache_data &tx_cache_data, std::map, size_t> *output_tracker_cache) { PERF_TIMER(process_new_transaction); // In this function, tx (probably) only contains the base information @@ -1686,6 +1686,8 @@ void wallet2::process_new_transaction(const crypto::hash &txid, const cryptonote if (!m_multisig && !m_watch_only) m_key_images[td.m_key_image] = m_transfers.size()-1; m_pub_keys[tx_scan_info[o].in_ephemeral.pub] = m_transfers.size()-1; + if (output_tracker_cache) + (*output_tracker_cache)[std::make_pair(tx.vout[o].amount, td.m_global_output_index)] = m_transfers.size() - 1; if (m_multisig) { THROW_WALLET_EXCEPTION_IF(!m_multisig_rescan_k && m_multisig_rescan_info, @@ -1751,6 +1753,8 @@ void wallet2::process_new_transaction(const crypto::hash &txid, const cryptonote td.m_mask = rct::identity(); td.m_rct = false; } + if (output_tracker_cache) + (*output_tracker_cache)[std::make_pair(tx.vout[o].amount, td.m_global_output_index)] = kit->second; if (m_multisig) { THROW_WALLET_EXCEPTION_IF(!m_multisig_rescan_k && m_multisig_rescan_info, @@ -1822,10 +1826,24 @@ void wallet2::process_new_transaction(const crypto::hash &txid, const cryptonote if (!pool && m_track_uses) { PERF_TIMER(track_uses); + const uint64_t amount = in_to_key.amount; std::vector offsets = cryptonote::relative_output_offsets_to_absolute(in_to_key.key_offsets); - for (transfer_details &td: m_transfers) + if (output_tracker_cache) { - if ((td.is_rct() ? 0 : td.amount()) != in_to_key.amount) + for (uint64_t offset: offsets) + { + const std::map, size_t>::const_iterator i = output_tracker_cache->find(std::make_pair(amount, offset)); + if (i != output_tracker_cache->end()) + { + size_t idx = i->second; + THROW_WALLET_EXCEPTION_IF(idx >= m_transfers.size(), error::wallet_internal_error, "Output tracker cache index out of range"); + m_transfers[idx].m_uses.push_back(std::make_pair(height, txid)); + } + } + } + else for (transfer_details &td: m_transfers) + { + if (amount != in_to_key.amount) continue; for (uint64_t offset: offsets) if (offset == td.m_global_output_index) @@ -2014,7 +2032,7 @@ void wallet2::process_outgoing(const crypto::hash &txid, const cryptonote::trans add_rings(tx); } //---------------------------------------------------------------------------------------------------- -void wallet2::process_new_blockchain_entry(const cryptonote::block& b, const cryptonote::block_complete_entry& bche, const parsed_block &parsed_block, const crypto::hash& bl_id, uint64_t height, const std::vector &tx_cache_data, size_t tx_cache_data_offset) +void wallet2::process_new_blockchain_entry(const cryptonote::block& b, const cryptonote::block_complete_entry& bche, const parsed_block &parsed_block, const crypto::hash& bl_id, uint64_t height, const std::vector &tx_cache_data, size_t tx_cache_data_offset, std::map, size_t> *output_tracker_cache) { THROW_WALLET_EXCEPTION_IF(bche.txs.size() + 1 != parsed_block.o_indices.indices.size(), error::wallet_internal_error, "block transactions=" + std::to_string(bche.txs.size()) + @@ -2027,7 +2045,7 @@ void wallet2::process_new_blockchain_entry(const cryptonote::block& b, const cry { TIME_MEASURE_START(miner_tx_handle_time); if (m_refresh_type != RefreshNoCoinbase) - process_new_transaction(get_transaction_hash(b.miner_tx), b.miner_tx, parsed_block.o_indices.indices[0].indices, height, b.timestamp, true, false, false, tx_cache_data[tx_cache_data_offset]); + process_new_transaction(get_transaction_hash(b.miner_tx), b.miner_tx, parsed_block.o_indices.indices[0].indices, height, b.timestamp, true, false, false, tx_cache_data[tx_cache_data_offset], output_tracker_cache); ++tx_cache_data_offset; TIME_MEASURE_FINISH(miner_tx_handle_time); @@ -2036,7 +2054,7 @@ void wallet2::process_new_blockchain_entry(const cryptonote::block& b, const cry THROW_WALLET_EXCEPTION_IF(bche.txs.size() != parsed_block.txes.size(), error::wallet_internal_error, "Wrong amount of transactions for block"); for (size_t idx = 0; idx < b.tx_hashes.size(); ++idx) { - process_new_transaction(b.tx_hashes[idx], parsed_block.txes[idx], parsed_block.o_indices.indices[idx+1].indices, height, b.timestamp, false, false, false, tx_cache_data[tx_cache_data_offset++]); + process_new_transaction(b.tx_hashes[idx], parsed_block.txes[idx], parsed_block.o_indices.indices[idx+1].indices, height, b.timestamp, false, false, false, tx_cache_data[tx_cache_data_offset++], output_tracker_cache); } TIME_MEASURE_FINISH(txs_handle_time); m_last_block_reward = cryptonote::get_outs_money_amount(b.miner_tx); @@ -2134,7 +2152,7 @@ void wallet2::pull_hashes(uint64_t start_height, uint64_t &blocks_start_height, hashes = std::move(res.m_block_ids); } //---------------------------------------------------------------------------------------------------- -void wallet2::process_parsed_blocks(uint64_t start_height, const std::vector &blocks, const std::vector &parsed_blocks, uint64_t& blocks_added) +void wallet2::process_parsed_blocks(uint64_t start_height, const std::vector &blocks, const std::vector &parsed_blocks, uint64_t& blocks_added, std::map, size_t> *output_tracker_cache) { size_t current_index = start_height; blocks_added = 0; @@ -2241,7 +2259,7 @@ void wallet2::process_parsed_blocks(uint64_t start_height, const std::vector= m_blockchain.size()) { - process_new_blockchain_entry(bl, blocks[i], parsed_blocks[i], bl_id, current_index, tx_cache_data, tx_cache_data_offset); + process_new_blockchain_entry(bl, blocks[i], parsed_blocks[i], bl_id, current_index, tx_cache_data, tx_cache_data_offset, output_tracker_cache); ++blocks_added; } else if(bl_id != m_blockchain[current_index]) @@ -2253,7 +2271,7 @@ void wallet2::process_parsed_blocks(uint64_t start_height, const std::vector, size_t>> wallet2::create_output_tracker_cache() const +{ + std::shared_ptr, size_t>> cache{new std::map, size_t>()}; + for (size_t i = 0; i < m_transfers.size(); ++i) + { + const transfer_details &td = m_transfers[i]; + (*cache)[std::make_pair(td.is_rct() ? 0 : td.amount(), td.m_global_output_index)] = i; + } + return cache; +} //---------------------------------------------------------------------------------------------------- void wallet2::refresh(bool trusted_daemon, uint64_t start_height, uint64_t & blocks_fetched, bool& received_money) { @@ -2732,6 +2761,7 @@ void wallet2::refresh(bool trusted_daemon, uint64_t start_height, uint64_t & blo std::vector blocks; std::vector parsed_blocks; bool refreshed = false; + std::shared_ptr, size_t>> output_tracker_cache; // pull the first set of blocks get_short_chain_history(short_chain_history, (m_first_refresh_done || trusted_daemon) ? 1 : FIRST_REFRESH_GRANULARITY); @@ -2785,7 +2815,7 @@ void wallet2::refresh(bool trusted_daemon, uint64_t start_height, uint64_t & blo { try { - process_parsed_blocks(blocks_start_height, blocks, parsed_blocks, added_blocks); + process_parsed_blocks(blocks_start_height, blocks, parsed_blocks, added_blocks, output_tracker_cache.get()); } catch (const tools::error::out_of_hashchain_bounds_error&) { @@ -2828,6 +2858,11 @@ void wallet2::refresh(bool trusted_daemon, uint64_t start_height, uint64_t & blo throw std::runtime_error("proxy exception in refresh thread"); } + // if we've got at least 10 blocks to refresh, assume we're starting + // a long refresh, and setup a tracking output cache if we need to + if (m_track_uses && !output_tracker_cache && next_blocks.size() >= 10) + output_tracker_cache = create_output_tracker_cache(); + // switch to the new blocks from the daemon blocks_start_height = next_blocks_start_height; blocks = std::move(next_blocks); diff --git a/src/wallet/wallet2.h b/src/wallet/wallet2.h index 52579376f..5b1988080 100644 --- a/src/wallet/wallet2.h +++ b/src/wallet/wallet2.h @@ -1253,8 +1253,8 @@ namespace tools * \param password Password of wallet file */ bool load_keys(const std::string& keys_file_name, const epee::wipeable_string& password); - void process_new_transaction(const crypto::hash &txid, const cryptonote::transaction& tx, const std::vector &o_indices, uint64_t height, uint64_t ts, bool miner_tx, bool pool, bool double_spend_seen, const tx_cache_data &tx_cache_data); - void process_new_blockchain_entry(const cryptonote::block& b, const cryptonote::block_complete_entry& bche, const parsed_block &parsed_block, const crypto::hash& bl_id, uint64_t height, const std::vector &tx_cache_data, size_t tx_cache_data_offset); + void process_new_transaction(const crypto::hash &txid, const cryptonote::transaction& tx, const std::vector &o_indices, uint64_t height, uint64_t ts, bool miner_tx, bool pool, bool double_spend_seen, const tx_cache_data &tx_cache_data, std::map, size_t> *output_tracker_cache = NULL); + void process_new_blockchain_entry(const cryptonote::block& b, const cryptonote::block_complete_entry& bche, const parsed_block &parsed_block, const crypto::hash& bl_id, uint64_t height, const std::vector &tx_cache_data, size_t tx_cache_data_offset, std::map, size_t> *output_tracker_cache = NULL); void detach_blockchain(uint64_t height); void get_short_chain_history(std::list& ids, uint64_t granularity = 1) const; bool clear(); @@ -1262,7 +1262,7 @@ namespace tools void pull_hashes(uint64_t start_height, uint64_t& blocks_start_height, const std::list &short_chain_history, std::vector &hashes); void fast_refresh(uint64_t stop_height, uint64_t &blocks_start_height, std::list &short_chain_history, bool force = false); void pull_and_parse_next_blocks(uint64_t start_height, uint64_t &blocks_start_height, std::list &short_chain_history, const std::vector &prev_blocks, const std::vector &prev_parsed_blocks, std::vector &blocks, std::vector &parsed_blocks, bool &error); - void process_parsed_blocks(uint64_t start_height, const std::vector &blocks, const std::vector &parsed_blocks, uint64_t& blocks_added); + void process_parsed_blocks(uint64_t start_height, const std::vector &blocks, const std::vector &parsed_blocks, uint64_t& blocks_added, std::map, size_t> *output_tracker_cache = NULL); uint64_t select_transfers(uint64_t needed_money, std::vector unused_transfers_indices, std::vector& selected_transfers) const; bool prepare_file_names(const std::string& file_path); void process_unconfirmed(const crypto::hash &txid, const cryptonote::transaction& tx, uint64_t height); @@ -1316,6 +1316,7 @@ namespace tools std::unordered_set &pkeys) const; void cache_tx_data(const cryptonote::transaction& tx, const crypto::hash &txid, tx_cache_data &tx_cache_data) const; + std::shared_ptr, size_t>> create_output_tracker_cache() const; void setup_new_blockchain(); void create_keys_file(const std::string &wallet_, bool watch_only, const epee::wipeable_string &password, bool create_address_file);