Merge pull request #4821

fc98f7a0 rpc: speedup get_outs.bin (moneromooo-monero)
This commit is contained in:
Riccardo Spagni 2018-11-26 21:51:27 +02:00
commit 58ce16d4d9
No known key found for this signature in database
GPG Key ID: 55432DF31CCD4FCD
8 changed files with 45 additions and 18 deletions

View File

@ -109,6 +109,8 @@ namespace epee
constexpr std::size_t size() const noexcept { return len; } constexpr std::size_t size() const noexcept { return len; }
constexpr std::size_t size_bytes() const noexcept { return size() * sizeof(value_type); } constexpr std::size_t size_bytes() const noexcept { return size() * sizeof(value_type); }
const T &operator[](size_t idx) const { return ptr[idx]; }
private: private:
T* ptr; T* ptr;
std::size_t len; std::size_t len;

View File

@ -1307,11 +1307,11 @@ public:
* get_output_data(const uint64_t& amount, const uint64_t& index) * get_output_data(const uint64_t& amount, const uint64_t& index)
* but for a list of outputs rather than just one. * but for a list of outputs rather than just one.
* *
* @param amount an output amount * @param amounts an output amount, or as many as offsets
* @param offsets a list of amount-specific output indices * @param offsets a list of amount-specific output indices
* @param outputs return-by-reference a list of outputs' metadata * @param outputs return-by-reference a list of outputs' metadata
*/ */
virtual void get_output_key(const uint64_t &amount, const std::vector<uint64_t> &offsets, std::vector<output_data_t> &outputs, bool allow_partial = false) = 0; virtual void get_output_key(const epee::span<const uint64_t> &amounts, const std::vector<uint64_t> &offsets, std::vector<output_data_t> &outputs, bool allow_partial = false) = 0;
/* /*
* FIXME: Need to check with git blame and ask what this does to * FIXME: Need to check with git blame and ask what this does to

View File

@ -3206,8 +3206,11 @@ void BlockchainLMDB::get_output_tx_and_index_from_global(const std::vector<uint6
TXN_POSTFIX_RDONLY(); TXN_POSTFIX_RDONLY();
} }
void BlockchainLMDB::get_output_key(const uint64_t &amount, const std::vector<uint64_t> &offsets, std::vector<output_data_t> &outputs, bool allow_partial) void BlockchainLMDB::get_output_key(const epee::span<const uint64_t> &amounts, const std::vector<uint64_t> &offsets, std::vector<output_data_t> &outputs, bool allow_partial)
{ {
if (amounts.size() != 1 && amounts.size() != offsets.size())
throw0(DB_ERROR("Invalid sizes of amounts and offets"));
LOG_PRINT_L3("BlockchainLMDB::" << __func__); LOG_PRINT_L3("BlockchainLMDB::" << __func__);
TIME_MEASURE_START(db3); TIME_MEASURE_START(db3);
check_open(); check_open();
@ -3218,10 +3221,11 @@ void BlockchainLMDB::get_output_key(const uint64_t &amount, const std::vector<ui
RCURSOR(output_amounts); RCURSOR(output_amounts);
MDB_val_set(k, amount); for (size_t i = 0; i < offsets.size(); ++i)
for (const uint64_t &index : offsets)
{ {
MDB_val_set(v, index); const uint64_t amount = amounts.size() == 1 ? amounts[0] : amounts[i];
MDB_val_set(k, amount);
MDB_val_set(v, offsets[i]);
auto get_result = mdb_cursor_get(m_cur_output_amounts, &k, &v, MDB_GET_BOTH); auto get_result = mdb_cursor_get(m_cur_output_amounts, &k, &v, MDB_GET_BOTH);
if (get_result == MDB_NOTFOUND) if (get_result == MDB_NOTFOUND)
@ -3231,7 +3235,7 @@ void BlockchainLMDB::get_output_key(const uint64_t &amount, const std::vector<ui
MDEBUG("Partial result: " << outputs.size() << "/" << offsets.size()); MDEBUG("Partial result: " << outputs.size() << "/" << offsets.size());
break; break;
} }
throw1(OUTPUT_DNE((std::string("Attempting to get output pubkey by global index (amount ") + boost::lexical_cast<std::string>(amount) + ", index " + boost::lexical_cast<std::string>(index) + ", count " + boost::lexical_cast<std::string>(get_num_outputs(amount)) + "), but key does not exist (current height " + boost::lexical_cast<std::string>(height()) + ")").c_str())); throw1(OUTPUT_DNE((std::string("Attempting to get output pubkey by global index (amount ") + boost::lexical_cast<std::string>(amount) + ", index " + boost::lexical_cast<std::string>(offsets[i]) + ", count " + boost::lexical_cast<std::string>(get_num_outputs(amount)) + "), but key does not exist (current height " + boost::lexical_cast<std::string>(height()) + ")").c_str()));
} }
else if (get_result) else if (get_result)
throw0(DB_ERROR(lmdb_error("Error attempting to retrieve an output pubkey from the db", get_result).c_str())); throw0(DB_ERROR(lmdb_error("Error attempting to retrieve an output pubkey from the db", get_result).c_str()));

View File

@ -243,7 +243,7 @@ public:
virtual uint64_t get_num_outputs(const uint64_t& amount) const; virtual uint64_t get_num_outputs(const uint64_t& amount) const;
virtual output_data_t get_output_key(const uint64_t& amount, const uint64_t& index); virtual output_data_t get_output_key(const uint64_t& amount, const uint64_t& index);
virtual void get_output_key(const uint64_t &amount, const std::vector<uint64_t> &offsets, std::vector<output_data_t> &outputs, bool allow_partial = false); virtual void get_output_key(const epee::span<const uint64_t> &amounts, const std::vector<uint64_t> &offsets, std::vector<output_data_t> &outputs, bool allow_partial = false);
virtual tx_out_index get_output_tx_and_index_from_global(const uint64_t& index) const; virtual tx_out_index get_output_tx_and_index_from_global(const uint64_t& index) const;
virtual void get_output_tx_and_index_from_global(const std::vector<uint64_t> &global_indices, virtual void get_output_tx_and_index_from_global(const std::vector<uint64_t> &global_indices,

View File

@ -229,7 +229,7 @@ bool Blockchain::scan_outputkeys_for_indexes(size_t tx_version, const txin_to_ke
{ {
try try
{ {
m_db->get_output_key(tx_in_to_key.amount, absolute_offsets, outputs, true); m_db->get_output_key(epee::span<const uint64_t>(&tx_in_to_key.amount, 1), absolute_offsets, outputs, true);
if (absolute_offsets.size() != outputs.size()) if (absolute_offsets.size() != outputs.size())
{ {
MERROR_VER("Output does not exist! amount = " << tx_in_to_key.amount); MERROR_VER("Output does not exist! amount = " << tx_in_to_key.amount);
@ -255,7 +255,7 @@ bool Blockchain::scan_outputkeys_for_indexes(size_t tx_version, const txin_to_ke
add_offsets.push_back(absolute_offsets[i]); add_offsets.push_back(absolute_offsets[i]);
try try
{ {
m_db->get_output_key(tx_in_to_key.amount, add_offsets, add_outputs, true); m_db->get_output_key(epee::span<const uint64_t>(&tx_in_to_key.amount, 1), add_offsets, add_outputs, true);
if (add_offsets.size() != add_outputs.size()) if (add_offsets.size() != add_outputs.size())
{ {
MERROR_VER("Output does not exist! amount = " << tx_in_to_key.amount); MERROR_VER("Output does not exist! amount = " << tx_in_to_key.amount);
@ -1767,16 +1767,34 @@ bool Blockchain::get_outs(const COMMAND_RPC_GET_OUTPUTS_BIN::request& req, COMMA
res.outs.clear(); res.outs.clear();
res.outs.reserve(req.outputs.size()); res.outs.reserve(req.outputs.size());
std::vector<cryptonote::output_data_t> data;
try try
{ {
std::vector<uint64_t> amounts, offsets;
amounts.reserve(req.outputs.size());
offsets.reserve(req.outputs.size());
for (const auto &i: req.outputs) for (const auto &i: req.outputs)
{ {
// get tx_hash, tx_out_index from DB amounts.push_back(i.amount);
const output_data_t od = m_db->get_output_key(i.amount, i.index); offsets.push_back(i.index);
tx_out_index toi = m_db->get_output_tx_and_index(i.amount, i.index); }
bool unlocked = is_tx_spendtime_unlocked(m_db->get_tx_unlock_time(toi.first)); m_db->get_output_key(epee::span<const uint64_t>(amounts.data(), amounts.size()), offsets, data);
if (data.size() != req.outputs.size())
{
MERROR("Unexpected output data size: expected " << req.outputs.size() << ", got " << data.size());
return false;
}
for (const auto &t: data)
res.outs.push_back({t.pubkey, t.commitment, is_tx_spendtime_unlocked(t.unlock_time), t.height, crypto::null_hash});
res.outs.push_back({od.pubkey, od.commitment, unlocked, od.height, toi.first}); if (req.get_txid)
{
for (size_t i = 0; i < req.outputs.size(); ++i)
{
tx_out_index toi = m_db->get_output_tx_and_index(req.outputs[i].amount, req.outputs[i].index);
res.outs[i].txid = toi.first;
}
} }
} }
catch (const std::exception &e) catch (const std::exception &e)
@ -3795,7 +3813,7 @@ void Blockchain::output_scan_worker(const uint64_t amount, const std::vector<uin
{ {
try try
{ {
m_db->get_output_key(amount, offsets, outputs, true); m_db->get_output_key(epee::span<const uint64_t>(&amount, 1), offsets, outputs, true);
} }
catch (const std::exception& e) catch (const std::exception& e)
{ {

View File

@ -84,7 +84,7 @@ namespace cryptonote
// advance which version they will stop working with // advance which version they will stop working with
// Don't go over 32767 for any of these // Don't go over 32767 for any of these
#define CORE_RPC_VERSION_MAJOR 2 #define CORE_RPC_VERSION_MAJOR 2
#define CORE_RPC_VERSION_MINOR 1 #define CORE_RPC_VERSION_MINOR 2
#define MAKE_CORE_RPC_VERSION(major,minor) (((major)<<16)|(minor)) #define MAKE_CORE_RPC_VERSION(major,minor) (((major)<<16)|(minor))
#define CORE_RPC_VERSION MAKE_CORE_RPC_VERSION(CORE_RPC_VERSION_MAJOR, CORE_RPC_VERSION_MINOR) #define CORE_RPC_VERSION MAKE_CORE_RPC_VERSION(CORE_RPC_VERSION_MAJOR, CORE_RPC_VERSION_MINOR)
@ -731,9 +731,11 @@ namespace cryptonote
struct request struct request
{ {
std::vector<get_outputs_out> outputs; std::vector<get_outputs_out> outputs;
bool get_txid;
BEGIN_KV_SERIALIZE_MAP() BEGIN_KV_SERIALIZE_MAP()
KV_SERIALIZE(outputs) KV_SERIALIZE(outputs)
KV_SERIALIZE_OPT(get_txid, true)
END_KV_SERIALIZE_MAP() END_KV_SERIALIZE_MAP()
}; };

View File

@ -7188,6 +7188,7 @@ void wallet2::get_outs(std::vector<std::vector<tools::wallet2::get_outs_entry>>
} }
// get the keys for those // get the keys for those
req.get_txid = false;
m_daemon_rpc_mutex.lock(); m_daemon_rpc_mutex.lock();
bool r = epee::net_utils::invoke_http_bin("/get_outs.bin", req, daemon_resp, m_http_client, rpc_timeout); bool r = epee::net_utils::invoke_http_bin("/get_outs.bin", req, daemon_resp, m_http_client, rpc_timeout);
m_daemon_rpc_mutex.unlock(); m_daemon_rpc_mutex.unlock();

View File

@ -93,7 +93,7 @@ public:
virtual cryptonote::tx_out_index get_output_tx_and_index_from_global(const uint64_t& index) const { return cryptonote::tx_out_index(); } virtual cryptonote::tx_out_index get_output_tx_and_index_from_global(const uint64_t& index) const { return cryptonote::tx_out_index(); }
virtual cryptonote::tx_out_index get_output_tx_and_index(const uint64_t& amount, const uint64_t& index) const { return cryptonote::tx_out_index(); } virtual cryptonote::tx_out_index get_output_tx_and_index(const uint64_t& amount, const uint64_t& index) const { return cryptonote::tx_out_index(); }
virtual void get_output_tx_and_index(const uint64_t& amount, const std::vector<uint64_t> &offsets, std::vector<cryptonote::tx_out_index> &indices) const {} virtual void get_output_tx_and_index(const uint64_t& amount, const std::vector<uint64_t> &offsets, std::vector<cryptonote::tx_out_index> &indices) const {}
virtual void get_output_key(const uint64_t &amount, const std::vector<uint64_t> &offsets, std::vector<cryptonote::output_data_t> &outputs, bool allow_partial = false) {} virtual void get_output_key(const epee::span<const uint64_t> &amounts, const std::vector<uint64_t> &offsets, std::vector<cryptonote::output_data_t> &outputs, bool allow_partial = false) {}
virtual bool can_thread_bulk_indices() const { return false; } virtual bool can_thread_bulk_indices() const { return false; }
virtual std::vector<uint64_t> get_tx_output_indices(const crypto::hash& h) const { return std::vector<uint64_t>(); } virtual std::vector<uint64_t> get_tx_output_indices(const crypto::hash& h) const { return std::vector<uint64_t>(); }
virtual std::vector<uint64_t> get_tx_amount_output_indices(const uint64_t tx_index) const { return std::vector<uint64_t>(); } virtual std::vector<uint64_t> get_tx_amount_output_indices(const uint64_t tx_index) const { return std::vector<uint64_t>(); }