db: speedup block addition

by avoiding repeated (de)serialization
This commit is contained in:
moneromooo-monero 2018-11-11 13:39:00 +00:00
parent 5fb4a9719c
commit e69477bf25
No known key found for this signature in database
GPG key ID: 686F07454D6CEFC3
4 changed files with 83 additions and 37 deletions

View file

@ -865,21 +865,26 @@ uint64_t BlockchainLMDB::add_transaction_data(const crypto::hash& blk_hash, cons
cryptonote::blobdata blob = tx_to_blob(tx); cryptonote::blobdata blob = tx_to_blob(tx);
MDB_val_sized(blobval, blob); MDB_val_sized(blobval, blob);
std::stringstream ss; unsigned int unprunable_size = tx.unprunable_size;
binary_archive<true> ba(ss); if (unprunable_size == 0)
bool r = const_cast<cryptonote::transaction&>(tx).serialize_base(ba); {
if (!r) std::stringstream ss;
throw0(DB_ERROR("Failed to serialize pruned tx")); binary_archive<true> ba(ss);
std::string pruned = ss.str(); bool r = const_cast<cryptonote::transaction&>(tx).serialize_base(ba);
MDB_val_sized(pruned_blob, pruned); if (!r)
throw0(DB_ERROR("Failed to serialize pruned tx"));
unprunable_size = ss.str().size();
}
if (unprunable_size > blob.size())
throw0(DB_ERROR("pruned tx size is larger than tx size"));
MDB_val pruned_blob = {unprunable_size, (void*)blob.data()};
result = mdb_cursor_put(m_cur_txs_pruned, &val_tx_id, &pruned_blob, MDB_APPEND); result = mdb_cursor_put(m_cur_txs_pruned, &val_tx_id, &pruned_blob, MDB_APPEND);
if (result) if (result)
throw0(DB_ERROR(lmdb_error("Failed to add pruned tx blob to db transaction: ", result).c_str())); throw0(DB_ERROR(lmdb_error("Failed to add pruned tx blob to db transaction: ", result).c_str()));
if (pruned.size() > blob.size()) MDB_val prunable_blob = {blob.size() - unprunable_size, (void*)(blob.data() + unprunable_size)};
throw0(DB_ERROR("pruned tx size is larger than tx size"));
cryptonote::blobdata prunable(blob.data() + pruned.size(), blob.size() - pruned.size());
MDB_val_sized(prunable_blob, prunable);
result = mdb_cursor_put(m_cur_txs_prunable, &val_tx_id, &prunable_blob, MDB_APPEND); result = mdb_cursor_put(m_cur_txs_prunable, &val_tx_id, &prunable_blob, MDB_APPEND);
if (result) if (result)
throw0(DB_ERROR(lmdb_error("Failed to add prunable tx blob to db transaction: ", result).c_str())); throw0(DB_ERROR(lmdb_error("Failed to add prunable tx blob to db transaction: ", result).c_str()));

View file

@ -152,6 +152,10 @@ namespace cryptonote
}; };
template<typename T> static inline unsigned int getpos(T &ar) { return 0; }
template<> inline unsigned int getpos(binary_archive<true> &ar) { return ar.stream().tellp(); }
template<> inline unsigned int getpos(binary_archive<false> &ar) { return ar.stream().tellg(); }
class transaction_prefix class transaction_prefix
{ {
@ -203,9 +207,12 @@ namespace cryptonote
bool pruned; bool pruned;
std::atomic<unsigned int> unprunable_size;
std::atomic<unsigned int> prefix_size;
transaction(); transaction();
transaction(const transaction &t): transaction_prefix(t), hash_valid(false), blob_size_valid(false), signatures(t.signatures), rct_signatures(t.rct_signatures), pruned(t.pruned) { if (t.is_hash_valid()) { hash = t.hash; set_hash_valid(true); } if (t.is_blob_size_valid()) { blob_size = t.blob_size; set_blob_size_valid(true); } } transaction(const transaction &t): transaction_prefix(t), hash_valid(false), blob_size_valid(false), signatures(t.signatures), rct_signatures(t.rct_signatures), pruned(t.pruned), unprunable_size(t.unprunable_size.load()), prefix_size(t.prefix_size.load()) { if (t.is_hash_valid()) { hash = t.hash; set_hash_valid(true); } if (t.is_blob_size_valid()) { blob_size = t.blob_size; set_blob_size_valid(true); } }
transaction &operator=(const transaction &t) { transaction_prefix::operator=(t); set_hash_valid(false); set_blob_size_valid(false); signatures = t.signatures; rct_signatures = t.rct_signatures; if (t.is_hash_valid()) { hash = t.hash; set_hash_valid(true); } if (t.is_blob_size_valid()) { blob_size = t.blob_size; set_blob_size_valid(true); } pruned = t.pruned; return *this; } transaction &operator=(const transaction &t) { transaction_prefix::operator=(t); set_hash_valid(false); set_blob_size_valid(false); signatures = t.signatures; rct_signatures = t.rct_signatures; if (t.is_hash_valid()) { hash = t.hash; set_hash_valid(true); } if (t.is_blob_size_valid()) { blob_size = t.blob_size; set_blob_size_valid(true); } pruned = t.pruned; unprunable_size = t.unprunable_size.load(); prefix_size = t.prefix_size.load(); return *this; }
virtual ~transaction(); virtual ~transaction();
void set_null(); void set_null();
void invalidate_hashes(); void invalidate_hashes();
@ -223,10 +230,18 @@ namespace cryptonote
set_blob_size_valid(false); set_blob_size_valid(false);
} }
const unsigned int start_pos = getpos(ar);
FIELDS(*static_cast<transaction_prefix *>(this)) FIELDS(*static_cast<transaction_prefix *>(this))
if (std::is_same<Archive<W>, binary_archive<W>>())
prefix_size = getpos(ar) - start_pos;
if (version == 1) if (version == 1)
{ {
if (std::is_same<Archive<W>, binary_archive<W>>())
unprunable_size = getpos(ar) - start_pos;
ar.tag("signatures"); ar.tag("signatures");
ar.begin_array(); ar.begin_array();
PREPARE_CUSTOM_VECTOR_SERIALIZATION(vin.size(), signatures); PREPARE_CUSTOM_VECTOR_SERIALIZATION(vin.size(), signatures);
@ -265,6 +280,10 @@ namespace cryptonote
bool r = rct_signatures.serialize_rctsig_base(ar, vin.size(), vout.size()); bool r = rct_signatures.serialize_rctsig_base(ar, vin.size(), vout.size());
if (!r || !ar.stream().good()) return false; if (!r || !ar.stream().good()) return false;
ar.end_object(); ar.end_object();
if (std::is_same<Archive<W>, binary_archive<W>>())
unprunable_size = getpos(ar) - start_pos;
if (!pruned && rct_signatures.type != rct::RCTTypeNull) if (!pruned && rct_signatures.type != rct::RCTTypeNull)
{ {
ar.tag("rctsig_prunable"); ar.tag("rctsig_prunable");
@ -329,6 +348,8 @@ namespace cryptonote
set_hash_valid(false); set_hash_valid(false);
set_blob_size_valid(false); set_blob_size_valid(false);
pruned = false; pruned = false;
unprunable_size = 0;
prefix_size = 0;
} }
inline inline

View file

@ -878,6 +878,11 @@ namespace cryptonote
return true; return true;
} }
//--------------------------------------------------------------- //---------------------------------------------------------------
void get_blob_hash(const epee::span<const char>& blob, crypto::hash& res)
{
cn_fast_hash(blob.data(), blob.size(), res);
}
//---------------------------------------------------------------
void get_blob_hash(const blobdata& blob, crypto::hash& res) void get_blob_hash(const blobdata& blob, crypto::hash& res)
{ {
cn_fast_hash(blob.data(), blob.size(), res); cn_fast_hash(blob.data(), blob.size(), res);
@ -946,6 +951,13 @@ namespace cryptonote
return h; return h;
} }
//--------------------------------------------------------------- //---------------------------------------------------------------
crypto::hash get_blob_hash(const epee::span<const char>& blob)
{
crypto::hash h = null_hash;
get_blob_hash(blob, h);
return h;
}
//---------------------------------------------------------------
crypto::hash get_transaction_hash(const transaction& t) crypto::hash get_transaction_hash(const transaction& t)
{ {
crypto::hash h = null_hash; crypto::hash h = null_hash;
@ -958,26 +970,35 @@ namespace cryptonote
return get_transaction_hash(t, res, NULL); return get_transaction_hash(t, res, NULL);
} }
//--------------------------------------------------------------- //---------------------------------------------------------------
bool calculate_transaction_prunable_hash(const transaction& t, crypto::hash& res) bool calculate_transaction_prunable_hash(const transaction& t, const cryptonote::blobdata *blob, crypto::hash& res)
{ {
if (t.version == 1) if (t.version == 1)
return false; return false;
transaction &tt = const_cast<transaction&>(t); const unsigned int unprunable_size = t.unprunable_size;
std::stringstream ss; if (blob && unprunable_size)
binary_archive<true> ba(ss); {
const size_t inputs = t.vin.size(); CHECK_AND_ASSERT_MES(unprunable_size <= blob->size(), false, "Inconsistent transaction unprunable and blob sizes");
const size_t outputs = t.vout.size(); cryptonote::get_blob_hash(epee::span<const char>(blob->data() + unprunable_size, blob->size() - unprunable_size), res);
const size_t mixin = t.vin.empty() ? 0 : t.vin[0].type() == typeid(txin_to_key) ? boost::get<txin_to_key>(t.vin[0]).key_offsets.size() - 1 : 0; }
bool r = tt.rct_signatures.p.serialize_rctsig_prunable(ba, t.rct_signatures.type, inputs, outputs, mixin); else
CHECK_AND_ASSERT_MES(r, false, "Failed to serialize rct signatures prunable"); {
cryptonote::get_blob_hash(ss.str(), res); transaction &tt = const_cast<transaction&>(t);
std::stringstream ss;
binary_archive<true> ba(ss);
const size_t inputs = t.vin.size();
const size_t outputs = t.vout.size();
const size_t mixin = t.vin.empty() ? 0 : t.vin[0].type() == typeid(txin_to_key) ? boost::get<txin_to_key>(t.vin[0]).key_offsets.size() - 1 : 0;
bool r = tt.rct_signatures.p.serialize_rctsig_prunable(ba, t.rct_signatures.type, inputs, outputs, mixin);
CHECK_AND_ASSERT_MES(r, false, "Failed to serialize rct signatures prunable");
cryptonote::get_blob_hash(ss.str(), res);
}
return true; return true;
} }
//--------------------------------------------------------------- //---------------------------------------------------------------
crypto::hash get_transaction_prunable_hash(const transaction& t) crypto::hash get_transaction_prunable_hash(const transaction& t, const cryptonote::blobdata *blobdata)
{ {
crypto::hash res; crypto::hash res;
CHECK_AND_ASSERT_THROW_MES(calculate_transaction_prunable_hash(t, res), "Failed to calculate tx prunable hash"); CHECK_AND_ASSERT_THROW_MES(calculate_transaction_prunable_hash(t, blobdata, res), "Failed to calculate tx prunable hash");
return res; return res;
} }
//--------------------------------------------------------------- //---------------------------------------------------------------
@ -1030,16 +1051,13 @@ namespace cryptonote
transaction &tt = const_cast<transaction&>(t); transaction &tt = const_cast<transaction&>(t);
const blobdata blob = tx_to_blob(t);
const unsigned int unprunable_size = t.unprunable_size;
const unsigned int prefix_size = t.prefix_size;
// base rct // base rct
{ CHECK_AND_ASSERT_MES(prefix_size <= unprunable_size && unprunable_size <= blob.size(), false, "Inconsistent transaction prefix, unprunable and blob sizes");
std::stringstream ss; cryptonote::get_blob_hash(epee::span<const char>(blob.data() + prefix_size, unprunable_size - prefix_size), hashes[1]);
binary_archive<true> ba(ss);
const size_t inputs = t.vin.size();
const size_t outputs = t.vout.size();
bool r = tt.rct_signatures.serialize_rctsig_base(ba, inputs, outputs);
CHECK_AND_ASSERT_MES(r, false, "Failed to serialize rct signatures base");
cryptonote::get_blob_hash(ss.str(), hashes[1]);
}
// prunable rct // prunable rct
if (t.rct_signatures.type == rct::RCTTypeNull) if (t.rct_signatures.type == rct::RCTTypeNull)
@ -1048,7 +1066,7 @@ namespace cryptonote
} }
else else
{ {
CHECK_AND_ASSERT_MES(calculate_transaction_prunable_hash(t, hashes[2]), false, "Failed to get tx prunable hash"); CHECK_AND_ASSERT_MES(calculate_transaction_prunable_hash(t, &blob, hashes[2]), false, "Failed to get tx prunable hash");
} }
// the tx hash is the hash of the 3 hashes // the tx hash is the hash of the 3 hashes

View file

@ -98,15 +98,17 @@ namespace cryptonote
bool generate_key_image_helper(const account_keys& ack, const std::unordered_map<crypto::public_key, subaddress_index>& subaddresses, const crypto::public_key& out_key, const crypto::public_key& tx_public_key, const std::vector<crypto::public_key>& additional_tx_public_keys, size_t real_output_index, keypair& in_ephemeral, crypto::key_image& ki, hw::device &hwdev); bool generate_key_image_helper(const account_keys& ack, const std::unordered_map<crypto::public_key, subaddress_index>& subaddresses, const crypto::public_key& out_key, const crypto::public_key& tx_public_key, const std::vector<crypto::public_key>& additional_tx_public_keys, size_t real_output_index, keypair& in_ephemeral, crypto::key_image& ki, hw::device &hwdev);
bool generate_key_image_helper_precomp(const account_keys& ack, const crypto::public_key& out_key, const crypto::key_derivation& recv_derivation, size_t real_output_index, const subaddress_index& received_index, keypair& in_ephemeral, crypto::key_image& ki, hw::device &hwdev); bool generate_key_image_helper_precomp(const account_keys& ack, const crypto::public_key& out_key, const crypto::key_derivation& recv_derivation, size_t real_output_index, const subaddress_index& received_index, keypair& in_ephemeral, crypto::key_image& ki, hw::device &hwdev);
void get_blob_hash(const blobdata& blob, crypto::hash& res); void get_blob_hash(const blobdata& blob, crypto::hash& res);
void get_blob_hash(const epee::span<const char>& blob, crypto::hash& res);
crypto::hash get_blob_hash(const blobdata& blob); crypto::hash get_blob_hash(const blobdata& blob);
crypto::hash get_blob_hash(const epee::span<const char>& blob);
std::string short_hash_str(const crypto::hash& h); std::string short_hash_str(const crypto::hash& h);
crypto::hash get_transaction_hash(const transaction& t); crypto::hash get_transaction_hash(const transaction& t);
bool get_transaction_hash(const transaction& t, crypto::hash& res); bool get_transaction_hash(const transaction& t, crypto::hash& res);
bool get_transaction_hash(const transaction& t, crypto::hash& res, size_t& blob_size); bool get_transaction_hash(const transaction& t, crypto::hash& res, size_t& blob_size);
bool get_transaction_hash(const transaction& t, crypto::hash& res, size_t* blob_size); bool get_transaction_hash(const transaction& t, crypto::hash& res, size_t* blob_size);
bool calculate_transaction_prunable_hash(const transaction& t, crypto::hash& res); bool calculate_transaction_prunable_hash(const transaction& t, const cryptonote::blobdata *blob, crypto::hash& res);
crypto::hash get_transaction_prunable_hash(const transaction& t); crypto::hash get_transaction_prunable_hash(const transaction& t, const cryptonote::blobdata *blob = NULL);
bool calculate_transaction_hash(const transaction& t, crypto::hash& res, size_t* blob_size); bool calculate_transaction_hash(const transaction& t, crypto::hash& res, size_t* blob_size);
crypto::hash get_pruned_transaction_hash(const transaction& t, const crypto::hash &pruned_data_hash); crypto::hash get_pruned_transaction_hash(const transaction& t, const crypto::hash &pruned_data_hash);