From 558cfc31caeb88398fe5bab9292246739586374d Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Wed, 22 Mar 2017 18:03:23 +0000 Subject: [PATCH] core, wallet: faster tx pool scanning Includes a new RPC to get tx pool hashes fast. --- src/cryptonote_core/cryptonote_core.cpp | 8 +- src/cryptonote_core/cryptonote_core.h | 7 + src/cryptonote_core/tx_pool.cpp | 7 + src/cryptonote_core/tx_pool.h | 7 + src/rpc/core_rpc_server.cpp | 8 + src/rpc/core_rpc_server.h | 2 + src/rpc/core_rpc_server_commands_defs.h | 22 ++- src/wallet/wallet2.cpp | 209 ++++++++++++------------ 8 files changed, 166 insertions(+), 104 deletions(-) diff --git a/src/cryptonote_core/cryptonote_core.cpp b/src/cryptonote_core/cryptonote_core.cpp index 1b20776a5..4d4b9d4d2 100644 --- a/src/cryptonote_core/cryptonote_core.cpp +++ b/src/cryptonote_core/cryptonote_core.cpp @@ -1036,7 +1036,13 @@ namespace cryptonote m_mempool.get_transactions(txs); return true; } - //----------------------------------------------------------------------------------------------- + //----------------------------------------------------------------------------------------------- + bool core::get_pool_transaction_hashes(std::vector& txs) const + { + m_mempool.get_transaction_hashes(txs); + return true; + } + //----------------------------------------------------------------------------------------------- bool core::get_pool_transaction(const crypto::hash &id, transaction& tx) const { return m_mempool.get_transaction(id, tx); diff --git a/src/cryptonote_core/cryptonote_core.h b/src/cryptonote_core/cryptonote_core.h index 02d58691d..24b0f0614 100644 --- a/src/cryptonote_core/cryptonote_core.h +++ b/src/cryptonote_core/cryptonote_core.h @@ -396,6 +396,13 @@ namespace cryptonote */ bool get_pool_transactions(std::list& txs) const; + /** + * @copydoc tx_memory_pool::get_transactions + * + * @note see tx_memory_pool::get_transactions + */ + bool get_pool_transaction_hashes(std::vector& txs) const; + /** * @copydoc tx_memory_pool::get_transaction * diff --git a/src/cryptonote_core/tx_pool.cpp b/src/cryptonote_core/tx_pool.cpp index ea19b6b48..b2a7120cb 100644 --- a/src/cryptonote_core/tx_pool.cpp +++ b/src/cryptonote_core/tx_pool.cpp @@ -423,6 +423,13 @@ namespace cryptonote txs.push_back(tx_vt.second.tx); } //------------------------------------------------------------------ + void tx_memory_pool::get_transaction_hashes(std::vector& txs) const + { + CRITICAL_REGION_LOCAL(m_transactions_lock); + for(const auto& tx_vt: m_transactions) + txs.push_back(get_transaction_hash(tx_vt.second.tx)); + } + //------------------------------------------------------------------ //TODO: investigate whether boolean return is appropriate bool tx_memory_pool::get_transactions_and_spent_keys_info(std::vector& tx_infos, std::vector& key_image_infos) const { diff --git a/src/cryptonote_core/tx_pool.h b/src/cryptonote_core/tx_pool.h index d71032943..d19f83b2e 100644 --- a/src/cryptonote_core/tx_pool.h +++ b/src/cryptonote_core/tx_pool.h @@ -233,6 +233,13 @@ namespace cryptonote */ void get_transactions(std::list& txs) const; + /** + * @brief get a list of all transaction hashes in the pool + * + * @param txs return-by-reference the list of transactions + */ + void get_transaction_hashes(std::vector& txs) const; + /** * @brief get information about all transactions and key images in the pool * diff --git a/src/rpc/core_rpc_server.cpp b/src/rpc/core_rpc_server.cpp index 9f7e8aa38..a800dbb35 100644 --- a/src/rpc/core_rpc_server.cpp +++ b/src/rpc/core_rpc_server.cpp @@ -789,6 +789,14 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ + bool core_rpc_server::on_get_transaction_pool_hashes(const COMMAND_RPC_GET_TRANSACTION_POOL_HASHES::request& req, COMMAND_RPC_GET_TRANSACTION_POOL_HASHES::response& res) + { + CHECK_CORE_BUSY(); + m_core.get_pool_transaction_hashes(res.tx_hashes); + res.status = CORE_RPC_STATUS_OK; + return true; + } + //------------------------------------------------------------------------------------------------------------------------------ bool core_rpc_server::on_stop_daemon(const COMMAND_RPC_STOP_DAEMON::request& req, COMMAND_RPC_STOP_DAEMON::response& res) { // FIXME: replace back to original m_p2p.send_stop_signal() after diff --git a/src/rpc/core_rpc_server.h b/src/rpc/core_rpc_server.h index 900f473a9..f8eff5f8c 100644 --- a/src/rpc/core_rpc_server.h +++ b/src/rpc/core_rpc_server.h @@ -92,6 +92,7 @@ namespace cryptonote MAP_URI_AUTO_JON2_IF("/set_log_level", on_set_log_level, COMMAND_RPC_SET_LOG_LEVEL, !m_restricted) MAP_URI_AUTO_JON2_IF("/set_log_categories", on_set_log_categories, COMMAND_RPC_SET_LOG_CATEGORIES, !m_restricted) MAP_URI_AUTO_JON2("/get_transaction_pool", on_get_transaction_pool, COMMAND_RPC_GET_TRANSACTION_POOL) + MAP_URI_AUTO_JON2("/get_transaction_pool_hashes.bin", on_get_transaction_pool_hashes, COMMAND_RPC_GET_TRANSACTION_POOL_HASHES) MAP_URI_AUTO_JON2_IF("/stop_daemon", on_stop_daemon, COMMAND_RPC_STOP_DAEMON, !m_restricted) MAP_URI_AUTO_JON2("/getinfo", on_get_info, COMMAND_RPC_GET_INFO) MAP_URI_AUTO_JON2_IF("/out_peers", on_out_peers, COMMAND_RPC_OUT_PEERS, !m_restricted) @@ -145,6 +146,7 @@ namespace cryptonote bool on_set_log_level(const COMMAND_RPC_SET_LOG_LEVEL::request& req, COMMAND_RPC_SET_LOG_LEVEL::response& res); bool on_set_log_categories(const COMMAND_RPC_SET_LOG_CATEGORIES::request& req, COMMAND_RPC_SET_LOG_CATEGORIES::response& res); bool on_get_transaction_pool(const COMMAND_RPC_GET_TRANSACTION_POOL::request& req, COMMAND_RPC_GET_TRANSACTION_POOL::response& res); + bool on_get_transaction_pool_hashes(const COMMAND_RPC_GET_TRANSACTION_POOL_HASHES::request& req, COMMAND_RPC_GET_TRANSACTION_POOL_HASHES::response& res); bool on_stop_daemon(const COMMAND_RPC_STOP_DAEMON::request& req, COMMAND_RPC_STOP_DAEMON::response& res); bool on_out_peers(const COMMAND_RPC_OUT_PEERS::request& req, COMMAND_RPC_OUT_PEERS::response& res); bool on_start_save_graph(const COMMAND_RPC_START_SAVE_GRAPH::request& req, COMMAND_RPC_START_SAVE_GRAPH::response& res); diff --git a/src/rpc/core_rpc_server_commands_defs.h b/src/rpc/core_rpc_server_commands_defs.h index 99aa9815e..6f60093ac 100644 --- a/src/rpc/core_rpc_server_commands_defs.h +++ b/src/rpc/core_rpc_server_commands_defs.h @@ -49,7 +49,7 @@ namespace cryptonote // advance which version they will stop working with // Don't go over 32767 for any of these #define CORE_RPC_VERSION_MAJOR 1 -#define CORE_RPC_VERSION_MINOR 7 +#define CORE_RPC_VERSION_MINOR 8 #define MAKE_CORE_RPC_VERSION(major,minor) (((major)<<16)|(minor)) #define CORE_RPC_VERSION MAKE_CORE_RPC_VERSION(CORE_RPC_VERSION_MAJOR, CORE_RPC_VERSION_MINOR) @@ -1025,6 +1025,26 @@ namespace cryptonote }; }; + struct COMMAND_RPC_GET_TRANSACTION_POOL_HASHES + { + struct request + { + BEGIN_KV_SERIALIZE_MAP() + END_KV_SERIALIZE_MAP() + }; + + struct response + { + std::string status; + std::vector tx_hashes; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(status) + KV_SERIALIZE_CONTAINER_POD_AS_BLOB(tx_hashes) + END_KV_SERIALIZE_MAP() + }; + }; + struct COMMAND_RPC_GET_CONNECTIONS { struct request diff --git a/src/wallet/wallet2.cpp b/src/wallet/wallet2.cpp index dce1ee9de..b3791b99c 100644 --- a/src/wallet/wallet2.cpp +++ b/src/wallet/wallet2.cpp @@ -1384,13 +1384,13 @@ void wallet2::update_pool_state() MDEBUG("update_pool_state start"); // get the pool state - cryptonote::COMMAND_RPC_GET_TRANSACTION_POOL::request req; - cryptonote::COMMAND_RPC_GET_TRANSACTION_POOL::response res; + cryptonote::COMMAND_RPC_GET_TRANSACTION_POOL_HASHES::request req; + cryptonote::COMMAND_RPC_GET_TRANSACTION_POOL_HASHES::response res; m_daemon_rpc_mutex.lock(); - bool r = epee::net_utils::invoke_http_json("/get_transaction_pool", req, res, m_http_client, rpc_timeout); + bool r = epee::net_utils::invoke_http_json("/get_transaction_pool_hashes.bin", req, res, m_http_client, rpc_timeout); m_daemon_rpc_mutex.unlock(); - THROW_WALLET_EXCEPTION_IF(!r, error::no_connection_to_daemon, "get_transaction_pool"); - THROW_WALLET_EXCEPTION_IF(res.status == CORE_RPC_STATUS_BUSY, error::daemon_busy, "get_transaction_pool"); + THROW_WALLET_EXCEPTION_IF(!r, error::no_connection_to_daemon, "get_transaction_pool_hashes.bin"); + THROW_WALLET_EXCEPTION_IF(res.status == CORE_RPC_STATUS_BUSY, error::daemon_busy, "get_transaction_pool_hashes.bin"); THROW_WALLET_EXCEPTION_IF(res.status != CORE_RPC_STATUS_OK, error::get_tx_pool_error); MDEBUG("update_pool_state got pool"); @@ -1398,11 +1398,11 @@ void wallet2::update_pool_state() std::unordered_map::iterator it = m_unconfirmed_txs.begin(); while (it != m_unconfirmed_txs.end()) { - const std::string txid = epee::string_tools::pod_to_hex(it->first); + const crypto::hash &txid = it->first; bool found = false; - for (auto it2: res.transactions) + for (const auto &it2: res.tx_hashes) { - if (it2.id_hash == txid) + if (it2 == txid) { found = true; break; @@ -1453,11 +1453,11 @@ void wallet2::update_pool_state() std::unordered_map::iterator uit = m_unconfirmed_payments.begin(); while (uit != m_unconfirmed_payments.end()) { - const std::string txid = string_tools::pod_to_hex(uit->first); + const crypto::hash &txid = uit->first; bool found = false; - for (auto it2: res.transactions) + for (const auto &it2: res.tx_hashes) { - if (it2.id_hash == txid) + if (it2 == txid) { found = true; break; @@ -1466,114 +1466,119 @@ void wallet2::update_pool_state() auto pit = uit++; if (!found) { + MDEBUG("Removing " << txid << " from unconfirmed payments, not found in pool"); m_unconfirmed_payments.erase(pit); } } MDEBUG("update_pool_state done second loop"); - // add new pool txes to us - for (auto it: res.transactions) + // gather txids of new pool txes to us + std::vector txids; + for (const auto &txid: res.tx_hashes) { - cryptonote::blobdata txid_data; - if(epee::string_tools::parse_hexstr_to_binbuff(it.id_hash, txid_data) && txid_data.size() == sizeof(crypto::hash)) + if (m_scanned_pool_txs[0].find(txid) != m_scanned_pool_txs[0].end() || m_scanned_pool_txs[1].find(txid) != m_scanned_pool_txs[1].end()) { - const crypto::hash txid = *reinterpret_cast(txid_data.data()); - if (m_scanned_pool_txs[0].find(txid) != m_scanned_pool_txs[0].end() || m_scanned_pool_txs[1].find(txid) != m_scanned_pool_txs[1].end()) + LOG_PRINT_L2("Already seen " << txid << ", skipped"); + continue; + } + if (m_unconfirmed_payments.find(txid) == m_unconfirmed_payments.end()) + { + LOG_PRINT_L1("Found new pool tx: " << txid); + bool found = false; + for (const auto &i: m_unconfirmed_txs) { - LOG_PRINT_L2("Already seen " << txid << ", skipped"); - continue; + if (i.first == txid) + { + found = true; + break; + } } - if (m_unconfirmed_payments.find(txid) == m_unconfirmed_payments.end()) + if (!found) { - LOG_PRINT_L1("Found new pool tx: " << txid); - bool found = false; - for (const auto &i: m_unconfirmed_txs) - { - if (i.first == txid) - { - found = true; - break; - } - } - if (!found) - { - // not one of those we sent ourselves - cryptonote::COMMAND_RPC_GET_TRANSACTIONS::request req; - cryptonote::COMMAND_RPC_GET_TRANSACTIONS::response res; - req.txs_hashes.push_back(it.id_hash); - MDEBUG("asking for " << it.id_hash); - req.decode_as_json = false; - m_daemon_rpc_mutex.lock(); - bool r = epee::net_utils::invoke_http_json("/gettransactions", req, res, m_http_client, rpc_timeout); - MDEBUG("asked for " << it.id_hash << ", got " << r << " and " << res.status); - m_daemon_rpc_mutex.unlock(); - if (r && res.status == CORE_RPC_STATUS_OK) - { - if (res.txs.size() == 1) - { - // might have just been put in a block - if (res.txs[0].in_pool) - { - cryptonote::transaction tx; - cryptonote::blobdata bd; - crypto::hash tx_hash, tx_prefix_hash; - if (epee::string_tools::parse_hexstr_to_binbuff(res.txs[0].as_hex, bd)) - { - if (cryptonote::parse_and_validate_tx_from_blob(bd, tx, tx_hash, tx_prefix_hash)) - { - if (tx_hash == txid) - { - process_new_transaction(txid, tx, std::vector(), 0, time(NULL), false, true); - m_scanned_pool_txs[0].insert(txid); - if (m_scanned_pool_txs[0].size() > 5000) - { - std::swap(m_scanned_pool_txs[0], m_scanned_pool_txs[1]); - m_scanned_pool_txs[0].clear(); - } - } - else - { - LOG_PRINT_L0("Mismatched txids when processing unconfimed txes from pool"); - } - } - else - { - LOG_PRINT_L0("failed to validate transaction from daemon"); - } - } - else - { - LOG_PRINT_L0("Failed to parse tx " << txid); - } - } - else - { - LOG_PRINT_L1("Tx " << txid << " was in pool, but is no more"); - } - } - else - { - LOG_PRINT_L0("Expected 1 tx, got " << res.txs.size()); - } - } - else - { - LOG_PRINT_L0("Error calling gettransactions daemon RPC: r " << r << ", status " << res.status); - } - } - else - { - LOG_PRINT_L1("We sent that one"); - } + // not one of those we sent ourselves + txids.push_back(txid); } else { - LOG_PRINT_L1("Already saw that one, it's for us"); + LOG_PRINT_L1("We sent that one"); } } else { - LOG_PRINT_L0("Failed to parse txid"); + LOG_PRINT_L1("Already saw that one, it's for us"); + } + } + + // get those txes + if (!txids.empty()) + { + cryptonote::COMMAND_RPC_GET_TRANSACTIONS::request req; + cryptonote::COMMAND_RPC_GET_TRANSACTIONS::response res; + for (const auto &txid: txids) + req.txs_hashes.push_back(epee::string_tools::pod_to_hex(txid)); + MDEBUG("asking for " << txids.size() << " transactions"); + req.decode_as_json = false; + m_daemon_rpc_mutex.lock(); + bool r = epee::net_utils::invoke_http_json("/gettransactions", req, res, m_http_client, rpc_timeout); + m_daemon_rpc_mutex.unlock(); + MDEBUG("Got " << r << " and " << res.status); + if (r && res.status == CORE_RPC_STATUS_OK) + { + if (res.txs.size() == txids.size()) + { + size_t n = 0; + for (const auto &txid: txids) + { + // might have just been put in a block + if (res.txs[n].in_pool) + { + cryptonote::transaction tx; + cryptonote::blobdata bd; + crypto::hash tx_hash, tx_prefix_hash; + if (epee::string_tools::parse_hexstr_to_binbuff(res.txs[n].as_hex, bd)) + { + if (cryptonote::parse_and_validate_tx_from_blob(bd, tx, tx_hash, tx_prefix_hash)) + { + if (tx_hash == txid) + { + process_new_transaction(txid, tx, std::vector(), 0, time(NULL), false, true); + m_scanned_pool_txs[0].insert(txid); + if (m_scanned_pool_txs[0].size() > 5000) + { + std::swap(m_scanned_pool_txs[0], m_scanned_pool_txs[1]); + m_scanned_pool_txs[0].clear(); + } + } + else + { + LOG_PRINT_L0("Mismatched txids when processing unconfimed txes from pool"); + } + } + else + { + LOG_PRINT_L0("failed to validate transaction from daemon"); + } + } + else + { + LOG_PRINT_L0("Failed to parse tx " << txid); + } + } + else + { + LOG_PRINT_L1("Tx " << txid << " was in pool, but is no more"); + } + ++n; + } + } + else + { + LOG_PRINT_L0("Expected " << txids.size() << " tx(es), got " << res.txs.size()); + } + } + else + { + LOG_PRINT_L0("Error calling gettransactions daemon RPC: r " << r << ", status " << res.status); } } MDEBUG("update_pool_state end");