wallet2: faster output and key image import/export

This commit is contained in:
moneromooo-monero 2018-09-10 13:08:34 +00:00
parent 7e2483e1d5
commit 769ae42a7b
No known key found for this signature in database
GPG key ID: 686F07454D6CEFC3

View file

@ -69,6 +69,7 @@ using namespace epee;
#include "common/base58.h" #include "common/base58.h"
#include "common/dns_utils.h" #include "common/dns_utils.h"
#include "common/notify.h" #include "common/notify.h"
#include "common/perf_timer.h"
#include "ringct/rctSigs.h" #include "ringct/rctSigs.h"
#include "ringdb.h" #include "ringdb.h"
#include "device/device_cold.hpp" #include "device/device_cold.hpp"
@ -10517,11 +10518,13 @@ crypto::public_key wallet2::get_tx_pub_key_from_received_outs(const tools::walle
bool wallet2::export_key_images(const std::string &filename) const bool wallet2::export_key_images(const std::string &filename) const
{ {
PERF_TIMER(export_key_images);
std::vector<std::pair<crypto::key_image, crypto::signature>> ski = export_key_images(); std::vector<std::pair<crypto::key_image, crypto::signature>> ski = export_key_images();
std::string magic(KEY_IMAGE_EXPORT_FILE_MAGIC, strlen(KEY_IMAGE_EXPORT_FILE_MAGIC)); std::string magic(KEY_IMAGE_EXPORT_FILE_MAGIC, strlen(KEY_IMAGE_EXPORT_FILE_MAGIC));
const cryptonote::account_public_address &keys = get_account().get_keys().m_account_address; const cryptonote::account_public_address &keys = get_account().get_keys().m_account_address;
std::string data; std::string data;
data.reserve(ski.size() * (sizeof(crypto::key_image) + sizeof(crypto::signature)) + 2 * sizeof(crypto::public_key));
data += std::string((const char *)&keys.m_spend_public_key, sizeof(crypto::public_key)); data += std::string((const char *)&keys.m_spend_public_key, sizeof(crypto::public_key));
data += std::string((const char *)&keys.m_view_public_key, sizeof(crypto::public_key)); data += std::string((const char *)&keys.m_view_public_key, sizeof(crypto::public_key));
for (const auto &i: ski) for (const auto &i: ski)
@ -10531,6 +10534,7 @@ bool wallet2::export_key_images(const std::string &filename) const
} }
// encrypt data, keep magic plaintext // encrypt data, keep magic plaintext
PERF_TIMER(export_key_images_encrypt);
std::string ciphertext = encrypt_with_view_secret_key(data); std::string ciphertext = encrypt_with_view_secret_key(data);
return epee::file_io_utils::save_string_to_file(filename, magic + ciphertext); return epee::file_io_utils::save_string_to_file(filename, magic + ciphertext);
} }
@ -10538,6 +10542,7 @@ bool wallet2::export_key_images(const std::string &filename) const
//---------------------------------------------------------------------------------------------------- //----------------------------------------------------------------------------------------------------
std::vector<std::pair<crypto::key_image, crypto::signature>> wallet2::export_key_images() const std::vector<std::pair<crypto::key_image, crypto::signature>> wallet2::export_key_images() const
{ {
PERF_TIMER(export_key_images_raw);
std::vector<std::pair<crypto::key_image, crypto::signature>> ski; std::vector<std::pair<crypto::key_image, crypto::signature>> ski;
ski.reserve(m_transfers.size()); ski.reserve(m_transfers.size());
@ -10590,6 +10595,7 @@ std::vector<std::pair<crypto::key_image, crypto::signature>> wallet2::export_key
uint64_t wallet2::import_key_images(const std::string &filename, uint64_t &spent, uint64_t &unspent) uint64_t wallet2::import_key_images(const std::string &filename, uint64_t &spent, uint64_t &unspent)
{ {
PERF_TIMER(import_key_images_fsu);
std::string data; std::string data;
bool r = epee::file_io_utils::load_file_to_string(filename, data); bool r = epee::file_io_utils::load_file_to_string(filename, data);
@ -10603,6 +10609,7 @@ uint64_t wallet2::import_key_images(const std::string &filename, uint64_t &spent
try try
{ {
PERF_TIMER(import_key_images_decrypt);
data = decrypt_with_view_secret_key(std::string(data, magiclen)); data = decrypt_with_view_secret_key(std::string(data, magiclen));
} }
catch (const std::exception &e) catch (const std::exception &e)
@ -10641,6 +10648,7 @@ uint64_t wallet2::import_key_images(const std::string &filename, uint64_t &spent
//---------------------------------------------------------------------------------------------------- //----------------------------------------------------------------------------------------------------
uint64_t wallet2::import_key_images(const std::vector<std::pair<crypto::key_image, crypto::signature>> &signed_key_images, uint64_t &spent, uint64_t &unspent, bool check_spent) uint64_t wallet2::import_key_images(const std::vector<std::pair<crypto::key_image, crypto::signature>> &signed_key_images, uint64_t &spent, uint64_t &unspent, bool check_spent)
{ {
PERF_TIMER(import_key_images_lots);
COMMAND_RPC_IS_KEY_IMAGE_SPENT::request req = AUTO_VAL_INIT(req); COMMAND_RPC_IS_KEY_IMAGE_SPENT::request req = AUTO_VAL_INIT(req);
COMMAND_RPC_IS_KEY_IMAGE_SPENT::response daemon_resp = AUTO_VAL_INIT(daemon_resp); COMMAND_RPC_IS_KEY_IMAGE_SPENT::response daemon_resp = AUTO_VAL_INIT(daemon_resp);
@ -10654,6 +10662,9 @@ uint64_t wallet2::import_key_images(const std::vector<std::pair<crypto::key_imag
return 0; return 0;
} }
req.key_images.reserve(signed_key_images.size());
PERF_TIMER_START(import_key_images_A);
for (size_t n = 0; n < signed_key_images.size(); ++n) for (size_t n = 0; n < signed_key_images.size(); ++n)
{ {
const transfer_details &td = m_transfers[n]; const transfer_details &td = m_transfers[n];
@ -10667,20 +10678,24 @@ uint64_t wallet2::import_key_images(const std::vector<std::pair<crypto::key_imag
const cryptonote::txout_to_key &o = boost::get<cryptonote::txout_to_key>(out.target); const cryptonote::txout_to_key &o = boost::get<cryptonote::txout_to_key>(out.target);
const crypto::public_key pkey = o.key; const crypto::public_key pkey = o.key;
std::vector<const crypto::public_key*> pkeys; if (!td.m_key_image_known || !(key_image == td.m_key_image))
pkeys.push_back(&pkey); {
THROW_WALLET_EXCEPTION_IF(!(rct::scalarmultKey(rct::ki2rct(key_image), rct::curveOrder()) == rct::identity()), std::vector<const crypto::public_key*> pkeys;
error::wallet_internal_error, "Key image out of validity domain: input " + boost::lexical_cast<std::string>(n) + "/" pkeys.push_back(&pkey);
+ boost::lexical_cast<std::string>(signed_key_images.size()) + ", key image " + epee::string_tools::pod_to_hex(key_image)); THROW_WALLET_EXCEPTION_IF(!(rct::scalarmultKey(rct::ki2rct(key_image), rct::curveOrder()) == rct::identity()),
error::wallet_internal_error, "Key image out of validity domain: input " + boost::lexical_cast<std::string>(n) + "/"
THROW_WALLET_EXCEPTION_IF(!crypto::check_ring_signature((const crypto::hash&)key_image, key_image, pkeys, &signature), + boost::lexical_cast<std::string>(signed_key_images.size()) + ", key image " + epee::string_tools::pod_to_hex(key_image));
error::signature_check_failed, boost::lexical_cast<std::string>(n) + "/"
+ boost::lexical_cast<std::string>(signed_key_images.size()) + ", key image " + epee::string_tools::pod_to_hex(key_image)
+ ", signature " + epee::string_tools::pod_to_hex(signature) + ", pubkey " + epee::string_tools::pod_to_hex(*pkeys[0]));
THROW_WALLET_EXCEPTION_IF(!crypto::check_ring_signature((const crypto::hash&)key_image, key_image, pkeys, &signature),
error::signature_check_failed, boost::lexical_cast<std::string>(n) + "/"
+ boost::lexical_cast<std::string>(signed_key_images.size()) + ", key image " + epee::string_tools::pod_to_hex(key_image)
+ ", signature " + epee::string_tools::pod_to_hex(signature) + ", pubkey " + epee::string_tools::pod_to_hex(*pkeys[0]));
}
req.key_images.push_back(epee::string_tools::pod_to_hex(key_image)); req.key_images.push_back(epee::string_tools::pod_to_hex(key_image));
} }
PERF_TIMER_STOP(import_key_images_A);
PERF_TIMER_START(import_key_images_B);
for (size_t n = 0; n < signed_key_images.size(); ++n) for (size_t n = 0; n < signed_key_images.size(); ++n)
{ {
m_transfers[n].m_key_image = signed_key_images[n].first; m_transfers[n].m_key_image = signed_key_images[n].first;
@ -10688,9 +10703,11 @@ uint64_t wallet2::import_key_images(const std::vector<std::pair<crypto::key_imag
m_transfers[n].m_key_image_known = true; m_transfers[n].m_key_image_known = true;
m_transfers[n].m_key_image_partial = false; m_transfers[n].m_key_image_partial = false;
} }
PERF_TIMER_STOP(import_key_images_B);
if(check_spent) if(check_spent)
{ {
PERF_TIMER(import_key_images_RPC);
m_daemon_rpc_mutex.lock(); m_daemon_rpc_mutex.lock();
bool r = epee::net_utils::invoke_http_json("/is_key_image_spent", req, daemon_resp, m_http_client, rpc_timeout); bool r = epee::net_utils::invoke_http_json("/is_key_image_spent", req, daemon_resp, m_http_client, rpc_timeout);
m_daemon_rpc_mutex.unlock(); m_daemon_rpc_mutex.unlock();
@ -10713,6 +10730,7 @@ uint64_t wallet2::import_key_images(const std::vector<std::pair<crypto::key_imag
// was created by sweep_all, so we can't know the spent height and other detailed info. // was created by sweep_all, so we can't know the spent height and other detailed info.
std::unordered_map<crypto::key_image, crypto::hash> spent_key_images; std::unordered_map<crypto::key_image, crypto::hash> spent_key_images;
PERF_TIMER_START(import_key_images_C);
for (const transfer_details &td: m_transfers) for (const transfer_details &td: m_transfers)
{ {
for (const cryptonote::txin_v& in : td.m_tx.vin) for (const cryptonote::txin_v& in : td.m_tx.vin)
@ -10721,10 +10739,12 @@ uint64_t wallet2::import_key_images(const std::vector<std::pair<crypto::key_imag
spent_key_images.insert(std::make_pair(boost::get<cryptonote::txin_to_key>(in).k_image, td.m_txid)); spent_key_images.insert(std::make_pair(boost::get<cryptonote::txin_to_key>(in).k_image, td.m_txid));
} }
} }
PERF_TIMER_STOP(import_key_images_C);
PERF_TIMER_START(import_key_images_D);
for(size_t i = 0; i < signed_key_images.size(); ++i) for(size_t i = 0; i < signed_key_images.size(); ++i)
{ {
transfer_details &td = m_transfers[i]; const transfer_details &td = m_transfers[i];
uint64_t amount = td.amount(); uint64_t amount = td.amount();
if (td.m_spent) if (td.m_spent)
spent += amount; spent += amount;
@ -10742,6 +10762,8 @@ uint64_t wallet2::import_key_images(const std::vector<std::pair<crypto::key_imag
spent_txids.insert(skii->second); spent_txids.insert(skii->second);
} }
} }
PERF_TIMER_STOP(import_key_images_D);
MDEBUG("Total: " << print_money(spent) << " spent, " << print_money(unspent) << " unspent"); MDEBUG("Total: " << print_money(spent) << " spent, " << print_money(unspent) << " unspent");
if (check_spent) if (check_spent)
@ -10751,8 +10773,12 @@ uint64_t wallet2::import_key_images(const std::vector<std::pair<crypto::key_imag
COMMAND_RPC_GET_TRANSACTIONS::response gettxs_res; COMMAND_RPC_GET_TRANSACTIONS::response gettxs_res;
gettxs_req.decode_as_json = false; gettxs_req.decode_as_json = false;
gettxs_req.prune = false; gettxs_req.prune = false;
gettxs_req.txs_hashes.reserve(spent_txids.size());
for (const crypto::hash& spent_txid : spent_txids) for (const crypto::hash& spent_txid : spent_txids)
gettxs_req.txs_hashes.push_back(epee::string_tools::pod_to_hex(spent_txid)); gettxs_req.txs_hashes.push_back(epee::string_tools::pod_to_hex(spent_txid));
PERF_TIMER_START(import_key_images_E);
m_daemon_rpc_mutex.lock(); m_daemon_rpc_mutex.lock();
bool r = epee::net_utils::invoke_http_json("/gettransactions", gettxs_req, gettxs_res, m_http_client, rpc_timeout); bool r = epee::net_utils::invoke_http_json("/gettransactions", gettxs_req, gettxs_res, m_http_client, rpc_timeout);
m_daemon_rpc_mutex.unlock(); m_daemon_rpc_mutex.unlock();
@ -10760,8 +10786,10 @@ uint64_t wallet2::import_key_images(const std::vector<std::pair<crypto::key_imag
THROW_WALLET_EXCEPTION_IF(gettxs_res.status == CORE_RPC_STATUS_BUSY, error::daemon_busy, "gettransactions"); THROW_WALLET_EXCEPTION_IF(gettxs_res.status == CORE_RPC_STATUS_BUSY, error::daemon_busy, "gettransactions");
THROW_WALLET_EXCEPTION_IF(gettxs_res.txs.size() != spent_txids.size(), error::wallet_internal_error, THROW_WALLET_EXCEPTION_IF(gettxs_res.txs.size() != spent_txids.size(), error::wallet_internal_error,
"daemon returned wrong response for gettransactions, wrong count = " + std::to_string(gettxs_res.txs.size()) + ", expected " + std::to_string(spent_txids.size())); "daemon returned wrong response for gettransactions, wrong count = " + std::to_string(gettxs_res.txs.size()) + ", expected " + std::to_string(spent_txids.size()));
PERF_TIMER_STOP(import_key_images_E);
// process each outgoing tx // process each outgoing tx
PERF_TIMER_START(import_key_images_F);
auto spent_txid = spent_txids.begin(); auto spent_txid = spent_txids.begin();
hw::device &hwdev = m_account.get_device(); hw::device &hwdev = m_account.get_device();
for (const COMMAND_RPC_GET_TRANSACTIONS::entry& e : gettxs_res.txs) for (const COMMAND_RPC_GET_TRANSACTIONS::entry& e : gettxs_res.txs)
@ -10857,7 +10885,9 @@ uint64_t wallet2::import_key_images(const std::vector<std::pair<crypto::key_imag
++spent_txid; ++spent_txid;
} }
PERF_TIMER_STOP(import_key_images_F);
PERF_TIMER_START(import_key_images_G);
for (size_t n : swept_transfers) for (size_t n : swept_transfers)
{ {
const transfer_details& td = m_transfers[n]; const transfer_details& td = m_transfers[n];
@ -10868,6 +10898,7 @@ uint64_t wallet2::import_key_images(const std::vector<std::pair<crypto::key_imag
const crypto::hash &spent_txid = crypto::null_hash; // spent txid is unknown const crypto::hash &spent_txid = crypto::null_hash; // spent txid is unknown
m_confirmed_txs.insert(std::make_pair(spent_txid, pd)); m_confirmed_txs.insert(std::make_pair(spent_txid, pd));
} }
PERF_TIMER_STOP(import_key_images_G);
} }
return m_transfers[signed_key_images.size() - 1].m_block_height; return m_transfers[signed_key_images.size() - 1].m_block_height;
@ -10955,6 +10986,7 @@ void wallet2::import_blockchain(const std::tuple<size_t, crypto::hash, std::vect
//---------------------------------------------------------------------------------------------------- //----------------------------------------------------------------------------------------------------
std::vector<tools::wallet2::transfer_details> wallet2::export_outputs() const std::vector<tools::wallet2::transfer_details> wallet2::export_outputs() const
{ {
PERF_TIMER(export_outputs);
std::vector<tools::wallet2::transfer_details> outs; std::vector<tools::wallet2::transfer_details> outs;
outs.reserve(m_transfers.size()); outs.reserve(m_transfers.size());
@ -10970,29 +11002,53 @@ std::vector<tools::wallet2::transfer_details> wallet2::export_outputs() const
//---------------------------------------------------------------------------------------------------- //----------------------------------------------------------------------------------------------------
std::string wallet2::export_outputs_to_str() const std::string wallet2::export_outputs_to_str() const
{ {
std::vector<tools::wallet2::transfer_details> outs = export_outputs(); PERF_TIMER(export_outputs_to_str);
std::stringstream oss; std::stringstream oss;
boost::archive::portable_binary_oarchive ar(oss); boost::archive::portable_binary_oarchive ar(oss);
ar << outs; ar << m_transfers;
std::string magic(OUTPUT_EXPORT_FILE_MAGIC, strlen(OUTPUT_EXPORT_FILE_MAGIC)); std::string magic(OUTPUT_EXPORT_FILE_MAGIC, strlen(OUTPUT_EXPORT_FILE_MAGIC));
const cryptonote::account_public_address &keys = get_account().get_keys().m_account_address; const cryptonote::account_public_address &keys = get_account().get_keys().m_account_address;
std::string header; std::string header;
header += std::string((const char *)&keys.m_spend_public_key, sizeof(crypto::public_key)); header += std::string((const char *)&keys.m_spend_public_key, sizeof(crypto::public_key));
header += std::string((const char *)&keys.m_view_public_key, sizeof(crypto::public_key)); header += std::string((const char *)&keys.m_view_public_key, sizeof(crypto::public_key));
PERF_TIMER(export_outputs_encryption);
std::string ciphertext = encrypt_with_view_secret_key(header + oss.str()); std::string ciphertext = encrypt_with_view_secret_key(header + oss.str());
return magic + ciphertext; return magic + ciphertext;
} }
//---------------------------------------------------------------------------------------------------- //----------------------------------------------------------------------------------------------------
size_t wallet2::import_outputs(const std::vector<tools::wallet2::transfer_details> &outputs) size_t wallet2::import_outputs(const std::vector<tools::wallet2::transfer_details> &outputs)
{ {
m_transfers.clear(); PERF_TIMER(import_outputs);
m_transfers.reserve(outputs.size()); const size_t original_size = m_transfers.size();
m_transfers.resize(outputs.size());
for (size_t i = 0; i < outputs.size(); ++i) for (size_t i = 0; i < outputs.size(); ++i)
{ {
transfer_details td = outputs[i]; transfer_details td = outputs[i];
// skip those we've already imported, or which have different data
if (i < original_size)
{
// compare the data used to create the key image below
const transfer_details &org_td = m_transfers[i];
if (!org_td.m_key_image_known)
goto process;
#define CMPF(f) if (!(td.f == org_td.f)) goto process
CMPF(m_txid);
CMPF(m_key_image);
CMPF(m_internal_output_index);
#undef CMPF
if (!(get_transaction_prefix_hash(td.m_tx) == get_transaction_prefix_hash(org_td.m_tx)))
goto process;
// copy anyway, since the comparison does not include ancillary fields which may have changed
m_transfers[i] = std::move(td);
continue;
}
process:
// the hot wallet wouldn't have known about key images (except if we already exported them) // the hot wallet wouldn't have known about key images (except if we already exported them)
cryptonote::keypair in_ephemeral; cryptonote::keypair in_ephemeral;
@ -11011,9 +11067,9 @@ size_t wallet2::import_outputs(const std::vector<tools::wallet2::transfer_detail
THROW_WALLET_EXCEPTION_IF(in_ephemeral.pub != out_key, THROW_WALLET_EXCEPTION_IF(in_ephemeral.pub != out_key,
error::wallet_internal_error, "key_image generated ephemeral public key not matched with output_key at index " + boost::lexical_cast<std::string>(i)); error::wallet_internal_error, "key_image generated ephemeral public key not matched with output_key at index " + boost::lexical_cast<std::string>(i));
m_key_images[td.m_key_image] = m_transfers.size(); m_key_images[td.m_key_image] = i;
m_pub_keys[td.get_public_key()] = m_transfers.size(); m_pub_keys[td.get_public_key()] = i;
m_transfers.push_back(std::move(td)); m_transfers[i] = std::move(td);
} }
return m_transfers.size(); return m_transfers.size();
@ -11021,6 +11077,7 @@ size_t wallet2::import_outputs(const std::vector<tools::wallet2::transfer_detail
//---------------------------------------------------------------------------------------------------- //----------------------------------------------------------------------------------------------------
size_t wallet2::import_outputs_from_str(const std::string &outputs_st) size_t wallet2::import_outputs_from_str(const std::string &outputs_st)
{ {
PERF_TIMER(import_outputs_from_str);
std::string data = outputs_st; std::string data = outputs_st;
const size_t magiclen = strlen(OUTPUT_EXPORT_FILE_MAGIC); const size_t magiclen = strlen(OUTPUT_EXPORT_FILE_MAGIC);
if (data.size() < magiclen || memcmp(data.data(), OUTPUT_EXPORT_FILE_MAGIC, magiclen)) if (data.size() < magiclen || memcmp(data.data(), OUTPUT_EXPORT_FILE_MAGIC, magiclen))
@ -11030,6 +11087,7 @@ size_t wallet2::import_outputs_from_str(const std::string &outputs_st)
try try
{ {
PERF_TIMER(import_outputs_decrypt);
data = decrypt_with_view_secret_key(std::string(data, magiclen)); data = decrypt_with_view_secret_key(std::string(data, magiclen));
} }
catch (const std::exception &e) catch (const std::exception &e)