From 4e75860da2befba8970d6530dcf8ec8dc9c4acd1 Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Tue, 1 May 2018 10:52:47 +0100 Subject: [PATCH 1/3] wallet2: fix misc issues when the ringdb can't be initialized --- src/wallet/wallet2.cpp | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/wallet/wallet2.cpp b/src/wallet/wallet2.cpp index 71c7992f3..b7c34b0e3 100755 --- a/src/wallet/wallet2.cpp +++ b/src/wallet/wallet2.cpp @@ -5485,7 +5485,7 @@ void wallet2::set_ring_database(const std::string &filename) bool wallet2::add_rings(const crypto::chacha_key &key, const cryptonote::transaction_prefix &tx) { if (!m_ringdb) - return true; + return false; return m_ringdb->add_rings(key, tx); } @@ -5499,7 +5499,7 @@ bool wallet2::add_rings(const cryptonote::transaction_prefix &tx) bool wallet2::remove_rings(const cryptonote::transaction_prefix &tx) { if (!m_ringdb) - return true; + return false; crypto::chacha_key key; generate_chacha_key_from_secret_keys(key); return m_ringdb->remove_rings(key, tx); @@ -5508,7 +5508,7 @@ bool wallet2::remove_rings(const cryptonote::transaction_prefix &tx) bool wallet2::get_ring(const crypto::chacha_key &key, const crypto::key_image &key_image, std::vector &outs) { if (!m_ringdb) - return true; + return false; return m_ringdb->get_ring(key, key_image, outs); } @@ -5546,7 +5546,7 @@ bool wallet2::get_ring(const crypto::key_image &key_image, std::vector bool wallet2::set_ring(const crypto::key_image &key_image, const std::vector &outs, bool relative) { if (!m_ringdb) - return true; + return false; crypto::chacha_key key; generate_chacha_key_from_secret_keys(key); @@ -5559,7 +5559,7 @@ bool wallet2::find_and_save_rings(bool force) if (!force && m_ring_history_saved) return true; if (!m_ringdb) - return true; + return false; COMMAND_RPC_GET_TRANSACTIONS::request req = AUTO_VAL_INIT(req); COMMAND_RPC_GET_TRANSACTIONS::response res = AUTO_VAL_INIT(res); @@ -5618,14 +5618,14 @@ bool wallet2::find_and_save_rings(bool force) bool wallet2::blackball_output(const crypto::public_key &output) { if (!m_ringdb) - return true; + return false; return m_ringdb->blackball(output); } bool wallet2::set_blackballed_outputs(const std::vector &outputs, bool add) { if (!m_ringdb) - return true; + return false; bool ret = true; if (!add) ret &= m_ringdb->clear_blackballs(); @@ -5637,14 +5637,14 @@ bool wallet2::set_blackballed_outputs(const std::vector &out bool wallet2::unblackball_output(const crypto::public_key &output) { if (!m_ringdb) - return true; + return false; return m_ringdb->unblackball(output); } bool wallet2::is_output_blackballed(const crypto::public_key &output) const { if (!m_ringdb) - return true; + return false; return m_ringdb->blackballed(output); } From 1035e47dd0a22ad283def9f83cd599ccb805b5a4 Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Tue, 1 May 2018 11:08:00 +0100 Subject: [PATCH 2/3] rpc: allow getting pruned blocks from gettransactions and get them pruned in find_and_save_rings, since it does not need the pruned data in the first place. Also set decode_to_json to false where missing, we don't need this either. --- src/rpc/core_rpc_server.cpp | 19 ++++++++++++------- src/rpc/core_rpc_server_commands_defs.h | 2 ++ src/wallet/wallet2.cpp | 13 +++++++++++++ 3 files changed, 27 insertions(+), 7 deletions(-) diff --git a/src/rpc/core_rpc_server.cpp b/src/rpc/core_rpc_server.cpp index c3d1a9d11..2746f2e43 100644 --- a/src/rpc/core_rpc_server.cpp +++ b/src/rpc/core_rpc_server.cpp @@ -209,6 +209,15 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ + static cryptonote::blobdata get_pruned_tx_blob(cryptonote::transaction &tx) + { + std::stringstream ss; + binary_archive ba(ss); + bool r = tx.serialize_base(ba); + CHECK_AND_ASSERT_MES(r, cryptonote::blobdata(), "Failed to serialize rct signatures base"); + return ss.str(); + } + //------------------------------------------------------------------------------------------------------------------------------ static cryptonote::blobdata get_pruned_tx_blob(const cryptonote::blobdata &blobdata) { cryptonote::transaction tx; @@ -216,14 +225,10 @@ namespace cryptonote if (!cryptonote::parse_and_validate_tx_from_blob(blobdata, tx)) { MERROR("Failed to parse and validate tx from blob"); - return blobdata; + return cryptonote::blobdata(); } - std::stringstream ss; - binary_archive ba(ss); - bool r = tx.serialize_base(ba); - CHECK_AND_ASSERT_MES(r, blobdata, "Failed to serialize rct signatures base"); - return ss.str(); + return get_pruned_tx_blob(tx); } //------------------------------------------------------------------------------------------------------------------------------ bool core_rpc_server::on_get_blocks(const COMMAND_RPC_GET_BLOCKS_FAST::request& req, COMMAND_RPC_GET_BLOCKS_FAST::response& res) @@ -633,7 +638,7 @@ namespace cryptonote crypto::hash tx_hash = *vhi++; e.tx_hash = *txhi++; - blobdata blob = t_serializable_object_to_blob(tx); + blobdata blob = req.prune ? get_pruned_tx_blob(tx) : t_serializable_object_to_blob(tx); e.as_hex = string_tools::buff_to_hex_nodelimer(blob); if (req.decode_as_json) e.as_json = obj_to_json_str(tx); diff --git a/src/rpc/core_rpc_server_commands_defs.h b/src/rpc/core_rpc_server_commands_defs.h index 660fb7889..df5b4893f 100644 --- a/src/rpc/core_rpc_server_commands_defs.h +++ b/src/rpc/core_rpc_server_commands_defs.h @@ -563,10 +563,12 @@ namespace cryptonote { std::list txs_hashes; bool decode_as_json; + bool prune; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(txs_hashes) KV_SERIALIZE(decode_as_json) + KV_SERIALIZE_OPT(prune, false) END_KV_SERIALIZE_MAP() }; diff --git a/src/wallet/wallet2.cpp b/src/wallet/wallet2.cpp index b7c34b0e3..0d47b95b2 100755 --- a/src/wallet/wallet2.cpp +++ b/src/wallet/wallet2.cpp @@ -2010,6 +2010,7 @@ void wallet2::update_pool_state(bool refreshed) req.txs_hashes.push_back(epee::string_tools::pod_to_hex(p.first)); MDEBUG("asking for " << txids.size() << " transactions"); req.decode_as_json = false; + req.prune = false; m_daemon_rpc_mutex.lock(); bool r = epee::net_utils::invoke_http_json("/gettransactions", req, res, m_http_client, rpc_timeout); m_daemon_rpc_mutex.unlock(); @@ -5579,6 +5580,7 @@ bool wallet2::find_and_save_rings(bool force) // get those transactions from the daemon req.decode_as_json = false; + req.prune = true; bool r; { const boost::lock_guard lock{m_daemon_rpc_mutex}; @@ -8159,6 +8161,7 @@ std::string wallet2::get_spend_proof(const crypto::hash &txid, const std::string COMMAND_RPC_GET_TRANSACTIONS::request req = AUTO_VAL_INIT(req); req.txs_hashes.push_back(epee::string_tools::pod_to_hex(txid)); req.decode_as_json = false; + req.prune = true; COMMAND_RPC_GET_TRANSACTIONS::response res = AUTO_VAL_INIT(res); bool r; { @@ -8278,6 +8281,7 @@ bool wallet2::check_spend_proof(const crypto::hash &txid, const std::string &mes COMMAND_RPC_GET_TRANSACTIONS::request req = AUTO_VAL_INIT(req); req.txs_hashes.push_back(epee::string_tools::pod_to_hex(txid)); req.decode_as_json = false; + req.prune = true; COMMAND_RPC_GET_TRANSACTIONS::response res = AUTO_VAL_INIT(res); bool r; { @@ -8400,6 +8404,8 @@ void wallet2::check_tx_key_helper(const crypto::hash &txid, const crypto::key_de COMMAND_RPC_GET_TRANSACTIONS::request req; COMMAND_RPC_GET_TRANSACTIONS::response res; req.txs_hashes.push_back(epee::string_tools::pod_to_hex(txid)); + req.decode_as_json = false; + req.prune = false; m_daemon_rpc_mutex.lock(); bool ok = epee::net_utils::invoke_http_json("/gettransactions", req, res, m_http_client); m_daemon_rpc_mutex.unlock(); @@ -8536,6 +8542,8 @@ std::string wallet2::get_tx_proof(const crypto::hash &txid, const cryptonote::ac COMMAND_RPC_GET_TRANSACTIONS::request req; COMMAND_RPC_GET_TRANSACTIONS::response res; req.txs_hashes.push_back(epee::string_tools::pod_to_hex(txid)); + req.decode_as_json = false; + req.prune = false; m_daemon_rpc_mutex.lock(); bool ok = net_utils::invoke_http_json("/gettransactions", req, res, m_http_client); m_daemon_rpc_mutex.unlock(); @@ -8646,6 +8654,8 @@ bool wallet2::check_tx_proof(const crypto::hash &txid, const cryptonote::account COMMAND_RPC_GET_TRANSACTIONS::request req; COMMAND_RPC_GET_TRANSACTIONS::response res; req.txs_hashes.push_back(epee::string_tools::pod_to_hex(txid)); + req.decode_as_json = false; + req.prune = false; m_daemon_rpc_mutex.lock(); bool ok = net_utils::invoke_http_json("/gettransactions", req, res, m_http_client); m_daemon_rpc_mutex.unlock(); @@ -8879,6 +8889,8 @@ bool wallet2::check_reserve_proof(const cryptonote::account_public_address &addr COMMAND_RPC_GET_TRANSACTIONS::response gettx_res; for (size_t i = 0; i < proofs.size(); ++i) gettx_req.txs_hashes.push_back(epee::string_tools::pod_to_hex(proofs[i].txid)); + gettx_req.decode_as_json = false; + gettx_req.prune = false; m_daemon_rpc_mutex.lock(); bool ok = net_utils::invoke_http_json("/gettransactions", gettx_req, gettx_res, m_http_client); m_daemon_rpc_mutex.unlock(); @@ -9453,6 +9465,7 @@ uint64_t wallet2::import_key_images(const std::vector Date: Tue, 1 May 2018 11:13:11 +0100 Subject: [PATCH 3/3] wallet2: request transactions in slices when scanning for known rings This avoid massive memory consumption for huge wallets --- src/wallet/wallet2.cpp | 54 +++++++++++++++++++++++++----------------- 1 file changed, 32 insertions(+), 22 deletions(-) diff --git a/src/wallet/wallet2.cpp b/src/wallet/wallet2.cpp index 0d47b95b2..b93761c88 100755 --- a/src/wallet/wallet2.cpp +++ b/src/wallet/wallet2.cpp @@ -5568,40 +5568,49 @@ bool wallet2::find_and_save_rings(bool force) MDEBUG("Finding and saving rings..."); // get payments we made + std::vector txs_hashes; std::list> payments; get_payments_out(payments, 0, std::numeric_limits::max(), boost::none, std::set()); for (const std::pair &entry: payments) { const crypto::hash &txid = entry.first; - req.txs_hashes.push_back(epee::string_tools::pod_to_hex(txid)); + txs_hashes.push_back(txid); } - MDEBUG("Found " << std::to_string(req.txs_hashes.size()) << " transactions"); - - // get those transactions from the daemon - req.decode_as_json = false; - req.prune = true; - bool r; - { - const boost::lock_guard lock{m_daemon_rpc_mutex}; - r = epee::net_utils::invoke_http_json("/gettransactions", req, res, m_http_client, rpc_timeout); - } - THROW_WALLET_EXCEPTION_IF(!r, error::no_connection_to_daemon, "gettransactions"); - THROW_WALLET_EXCEPTION_IF(res.status == CORE_RPC_STATUS_BUSY, error::daemon_busy, "gettransactions"); - THROW_WALLET_EXCEPTION_IF(res.status != CORE_RPC_STATUS_OK, error::wallet_internal_error, "gettransactions"); - THROW_WALLET_EXCEPTION_IF(res.txs.size() != req.txs_hashes.size(), error::wallet_internal_error, - "daemon returned wrong response for gettransactions, wrong txs count = " + - std::to_string(res.txs.size()) + ", expected " + std::to_string(req.txs_hashes.size())); - - MDEBUG("Scanning " << res.txs.size() << " transactions"); + MDEBUG("Found " << std::to_string(txs_hashes.size()) << " transactions"); crypto::chacha_key key; generate_chacha_key_from_secret_keys(key); - auto it = req.txs_hashes.begin(); - for (size_t i = 0; i < res.txs.size(); ++i, ++it) + // get those transactions from the daemon + static const size_t SLICE_SIZE = 200; + for (size_t slice = 0; slice < txs_hashes.size(); slice += SLICE_SIZE) { + req.decode_as_json = false; + req.prune = true; + req.txs_hashes.clear(); + size_t ntxes = slice + SLICE_SIZE > txs_hashes.size() ? txs_hashes.size() - slice : SLICE_SIZE; + for (size_t s = slice; s < slice + ntxes; ++s) + req.txs_hashes.push_back(epee::string_tools::pod_to_hex(txs_hashes[s])); + bool r; + { + const boost::lock_guard lock{m_daemon_rpc_mutex}; + r = epee::net_utils::invoke_http_json("/gettransactions", req, res, m_http_client, rpc_timeout); + } + THROW_WALLET_EXCEPTION_IF(!r, error::no_connection_to_daemon, "gettransactions"); + THROW_WALLET_EXCEPTION_IF(res.status == CORE_RPC_STATUS_BUSY, error::daemon_busy, "gettransactions"); + THROW_WALLET_EXCEPTION_IF(res.status != CORE_RPC_STATUS_OK, error::wallet_internal_error, "gettransactions"); + THROW_WALLET_EXCEPTION_IF(res.txs.size() != req.txs_hashes.size(), error::wallet_internal_error, + "daemon returned wrong response for gettransactions, wrong txs count = " + + std::to_string(res.txs.size()) + ", expected " + std::to_string(req.txs_hashes.size())); + + MDEBUG("Scanning " << res.txs.size() << " transactions"); + THROW_WALLET_EXCEPTION_IF(slice + res.txs.size() > txs_hashes.size(), error::wallet_internal_error, "Unexpected tx array size"); + auto it = req.txs_hashes.begin(); + for (size_t i = 0; i < res.txs.size(); ++i, ++it) + { const auto &tx_info = res.txs[i]; + THROW_WALLET_EXCEPTION_IF(tx_info.tx_hash != epee::string_tools::pod_to_hex(txs_hashes[slice + i]), error::wallet_internal_error, "Wrong txid received"); THROW_WALLET_EXCEPTION_IF(tx_info.tx_hash != *it, error::wallet_internal_error, "Wrong txid received"); cryptonote::blobdata bd; THROW_WALLET_EXCEPTION_IF(!epee::string_tools::parse_hexstr_to_binbuff(tx_info.as_hex, bd), error::wallet_internal_error, "failed to parse tx from hexstr"); @@ -5610,9 +5619,10 @@ bool wallet2::find_and_save_rings(bool force) THROW_WALLET_EXCEPTION_IF(!cryptonote::parse_and_validate_tx_from_blob(bd, tx, tx_hash, tx_prefix_hash), error::wallet_internal_error, "failed to parse tx from blob"); THROW_WALLET_EXCEPTION_IF(epee::string_tools::pod_to_hex(tx_hash) != tx_info.tx_hash, error::wallet_internal_error, "txid mismatch"); THROW_WALLET_EXCEPTION_IF(!add_rings(key, tx), error::wallet_internal_error, "Failed to save ring"); + } } - MINFO("Found and saved rings for " << res.txs.size() << " transactions"); + MINFO("Found and saved rings for " << txs_hashes.size() << " transactions"); m_ring_history_saved = true; return true; }