From cd972bdcc25e9ed1d4c5bc7663242afa6a61125e Mon Sep 17 00:00:00 2001 From: warptangent Date: Tue, 10 Feb 2015 19:54:27 -0800 Subject: [PATCH 01/16] Update year and formatting in license --- src/blockchain_converter/blockchain_converter.cpp | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/blockchain_converter/blockchain_converter.cpp b/src/blockchain_converter/blockchain_converter.cpp index 57822700f..0dfc1a2c3 100644 --- a/src/blockchain_converter/blockchain_converter.cpp +++ b/src/blockchain_converter/blockchain_converter.cpp @@ -1,21 +1,21 @@ -// Copyright (c) 2014, The Monero Project -// +// Copyright (c) 2014-2015, The Monero Project +// // All rights reserved. -// +// // Redistribution and use in source and binary forms, with or without modification, are // permitted provided that the following conditions are met: -// +// // 1. Redistributions of source code must retain the above copyright notice, this list of // conditions and the following disclaimer. -// +// // 2. Redistributions in binary form must reproduce the above copyright notice, this list // of conditions and the following disclaimer in the documentation and/or other // materials provided with the distribution. -// +// // 3. Neither the name of the copyright holder nor the names of its contributors may be // used to endorse or promote products derived from this software without specific // prior written permission. -// +// // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF // MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL From 59305d3137cbf477c2dc07c26b757d23729ad72b Mon Sep 17 00:00:00 2001 From: warptangent Date: Tue, 10 Feb 2015 15:25:15 -0800 Subject: [PATCH 02/16] Blockchain: match original function declaration from blockchain_storage --- src/cryptonote_core/blockchain.cpp | 2 +- src/cryptonote_core/blockchain.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/cryptonote_core/blockchain.cpp b/src/cryptonote_core/blockchain.cpp index 525c7f36e..b4b1a4c86 100644 --- a/src/cryptonote_core/blockchain.cpp +++ b/src/cryptonote_core/blockchain.cpp @@ -2288,7 +2288,7 @@ bool Blockchain::add_new_block(const block& bl_, block_verification_context& bvc return handle_block_to_main_chain(bl, id, bvc); } //------------------------------------------------------------------ -void Blockchain::check_against_checkpoints(checkpoints& points, bool enforce) +void Blockchain::check_against_checkpoints(const checkpoints& points, bool enforce) { const auto& pts = points.get_points(); diff --git a/src/cryptonote_core/blockchain.h b/src/cryptonote_core/blockchain.h index 6ab963ac7..3e8a2de31 100644 --- a/src/cryptonote_core/blockchain.h +++ b/src/cryptonote_core/blockchain.h @@ -141,7 +141,7 @@ namespace cryptonote void print_blockchain_index(); void print_blockchain_outs(const std::string& file); - void check_against_checkpoints(checkpoints& points, bool enforce); + void check_against_checkpoints(const checkpoints& points, bool enforce); void set_enforce_dns_checkpoints(bool enforce); bool update_checkpoints(const std::string& file_path, bool check_dns); From 2531aa31f80dff871b08371324a43e1363218c59 Mon Sep 17 00:00:00 2001 From: warptangent Date: Wed, 11 Feb 2015 15:55:53 -0800 Subject: [PATCH 03/16] Add and extend log statements --- .../BlockchainDB_impl/db_lmdb.cpp | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp index 4fa16b23e..f5368d6e5 100644 --- a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp +++ b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp @@ -167,8 +167,11 @@ void BlockchainLMDB::add_block( const block& blk MDB_val_copy parent_key(blk.prev_id); MDB_val parent_h; if (mdb_get(*m_write_txn, m_block_heights, &parent_key, &parent_h)) + { + LOG_PRINT_L3("m_height: " << m_height); + LOG_PRINT_L3("parent_key: " << blk.prev_id); throw0(DB_ERROR("Failed to get top block hash to check for new block's parent")); - + } uint64_t parent_height = *(const uint64_t *)parent_h.mv_data; if (parent_height != m_height - 1) throw0(BLOCK_PARENT_DNE("Top block is not new block's parent")); @@ -177,8 +180,9 @@ void BlockchainLMDB::add_block( const block& blk MDB_val_copy key(m_height); MDB_val_copy blob(block_to_blob(blk)); - if (mdb_put(*m_write_txn, m_blocks, &key, &blob, 0)) - throw0(DB_ERROR("Failed to add block blob to db transaction")); + auto res = mdb_put(*m_write_txn, m_blocks, &key, &blob, 0); + if (res) + throw0(DB_ERROR(std::string("Failed to add block blob to db transaction: ").append(mdb_strerror(res)).c_str())); MDB_val_copy sz(block_size); if (mdb_put(*m_write_txn, m_block_sizes, &key, &sz, 0)) @@ -505,8 +509,8 @@ void BlockchainLMDB::add_spent_key(const crypto::key_image& k_image) char anything = '\0'; unused.mv_size = sizeof(char); unused.mv_data = &anything; - if (mdb_put(*m_write_txn, m_spent_keys, &val_key, &unused, 0)) - throw1(DB_ERROR("Error adding spent key image to db transaction")); + if (auto result = mdb_put(*m_write_txn, m_spent_keys, &val_key, &unused, 0)) + throw1(DB_ERROR(std::string("Error adding spent key image to db transaction: ").append(mdb_strerror(result)).c_str())); } void BlockchainLMDB::remove_spent_key(const crypto::key_image& k_image) @@ -910,7 +914,7 @@ size_t BlockchainLMDB::get_block_size(const uint64_t& height) const difficulty_type BlockchainLMDB::get_block_cumulative_difficulty(const uint64_t& height) const { - LOG_PRINT_L3("BlockchainLMDB::" << __func__); + LOG_PRINT_L3("BlockchainLMDB::" << __func__ << " height: " << height); check_open(); txn_safe txn; From 4b90fd389de915767b89c558d7ec8187a5d61d14 Mon Sep 17 00:00:00 2001 From: warptangent Date: Wed, 11 Feb 2015 15:55:53 -0800 Subject: [PATCH 04/16] Add log statement --- src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp index f5368d6e5..b748baddb 100644 --- a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp +++ b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp @@ -1563,6 +1563,7 @@ uint64_t BlockchainLMDB::add_block( const block& blk void BlockchainLMDB::pop_block(block& blk, std::vector& txs) { + LOG_PRINT_L3("BlockchainLMDB::" << __func__); txn_safe txn; if (mdb_txn_begin(m_env, NULL, 0, txn)) throw0(DB_ERROR("Failed to create a transaction for the db")); From 26873db199198bad1892b17a9e7bc0a72c16d5b5 Mon Sep 17 00:00:00 2001 From: warptangent Date: Wed, 11 Feb 2015 15:55:53 -0800 Subject: [PATCH 05/16] Remove unused variable --- src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp index b748baddb..c9741c81d 100644 --- a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp +++ b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp @@ -386,7 +386,6 @@ void BlockchainLMDB::remove_output(const uint64_t& out_index, const uint64_t amo check_open(); MDB_val_copy k(out_index); - MDB_val v; /****** Uncomment if ever outputs actually need to be stored in this manner blobdata b; From aa82f786c730cb27a69bb70d0806ccc384a5890a Mon Sep 17 00:00:00 2001 From: warptangent Date: Wed, 11 Feb 2015 15:55:53 -0800 Subject: [PATCH 06/16] Fix log statement --- src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp index c9741c81d..a11bfdc8c 100644 --- a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp +++ b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp @@ -675,7 +675,7 @@ void BlockchainLMDB::open(const std::string& filename) lmdb_db_open(txn, LMDB_OUTPUT_GINDICES, MDB_CREATE, m_output_gindices, "Failed to open db handle for m_output_gindices"); *************************************************/ - lmdb_db_open(txn, LMDB_SPENT_KEYS, MDB_CREATE, m_spent_keys, "Failed to open db handle for m_outputs"); + lmdb_db_open(txn, LMDB_SPENT_KEYS, MDB_CREATE, m_spent_keys, "Failed to open db handle for m_spent_keys"); mdb_set_dupsort(txn, m_output_amounts, compare_uint64); mdb_set_dupsort(txn, m_tx_outputs, compare_uint64); From 42f8fe5c7f5c3ed7b0d0090a943bffd518c6554d Mon Sep 17 00:00:00 2001 From: warptangent Date: Wed, 11 Feb 2015 15:55:53 -0800 Subject: [PATCH 07/16] Fix formatting --- src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp index a11bfdc8c..9f6bc6b31 100644 --- a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp +++ b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp @@ -756,7 +756,6 @@ void BlockchainLMDB::unlock() check_open(); } - bool BlockchainLMDB::block_exists(const crypto::hash& h) const { LOG_PRINT_L3("BlockchainLMDB::" << __func__); @@ -772,7 +771,7 @@ bool BlockchainLMDB::block_exists(const crypto::hash& h) const if (get_result == MDB_NOTFOUND) { txn.commit(); - LOG_PRINT_L1("Block with hash " << epee::string_tools::pod_to_hex(h) << "not found in db"); + LOG_PRINT_L1("Block with hash " << epee::string_tools::pod_to_hex(h) << " not found in db"); return false; } else if (get_result) @@ -1075,7 +1074,7 @@ bool BlockchainLMDB::tx_exists(const crypto::hash& h) const if (get_result == MDB_NOTFOUND) { txn.commit(); - LOG_PRINT_L1("transaction with hash " << epee::string_tools::pod_to_hex(h) << "not found in db"); + LOG_PRINT_L1("transaction with hash " << epee::string_tools::pod_to_hex(h) << " not found in db"); return false; } else if (get_result) @@ -1097,7 +1096,7 @@ uint64_t BlockchainLMDB::get_tx_unlock_time(const crypto::hash& h) const MDB_val result; auto get_result = mdb_get(txn, m_tx_unlocks, &key, &result); if (get_result == MDB_NOTFOUND) - throw1(TX_DNE(std::string("tx unlock time with hash ").append(epee::string_tools::pod_to_hex(h)).append("not found in db").c_str())); + throw1(TX_DNE(std::string("tx unlock time with hash ").append(epee::string_tools::pod_to_hex(h)).append(" not found in db").c_str())); else if (get_result) throw0(DB_ERROR("DB error attempting to fetch tx unlock time from hash")); @@ -1117,7 +1116,7 @@ transaction BlockchainLMDB::get_tx(const crypto::hash& h) const MDB_val result; auto get_result = mdb_get(txn, m_txs, &key, &result); if (get_result == MDB_NOTFOUND) - throw1(TX_DNE(std::string("tx with hash ").append(epee::string_tools::pod_to_hex(h)).append("not found in db").c_str())); + throw1(TX_DNE(std::string("tx with hash ").append(epee::string_tools::pod_to_hex(h)).append(" not found in db").c_str())); else if (get_result) throw0(DB_ERROR("DB error attempting to fetch tx from hash")); @@ -1177,7 +1176,7 @@ uint64_t BlockchainLMDB::get_tx_block_height(const crypto::hash& h) const auto get_result = mdb_get(txn, m_tx_heights, &key, &result); if (get_result == MDB_NOTFOUND) { - throw1(TX_DNE(std::string("tx height with hash ").append(epee::string_tools::pod_to_hex(h)).append("not found in db").c_str())); + throw1(TX_DNE(std::string("tx height with hash ").append(epee::string_tools::pod_to_hex(h)).append(" not found in db").c_str())); } else if (get_result) throw0(DB_ERROR("DB error attempting to fetch tx height from hash")); From ce71abd0fe9739c3557ba5fce446d1244670976f Mon Sep 17 00:00:00 2001 From: warptangent Date: Thu, 19 Feb 2015 06:37:00 -0800 Subject: [PATCH 08/16] Move LMDB storage to subfolder --- src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp | 10 ++++++++++ src/cryptonote_core/blockchain.cpp | 3 ++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp index 9f6bc6b31..f7e8c5dda 100644 --- a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp +++ b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp @@ -630,6 +630,16 @@ void BlockchainLMDB::open(const std::string& filename) throw0(DB_OPEN_FAILURE(std::string("Failed to create directory ").append(filename).c_str())); } + // check for existing LMDB files in base directory + boost::filesystem::path old_files = direc.parent_path(); + if (boost::filesystem::exists(old_files / "data.mdb") || + boost::filesystem::exists(old_files / "lock.mdb")) + { + LOG_PRINT_L0("Found existing LMDB files in " << old_files.c_str()); + LOG_PRINT_L0("Move data.mdb and/or lock.mdb to " << filename << ", or delete them, and then restart"); + throw DB_ERROR("Database could not be opened"); + } + m_folder = filename; // set up lmdb environment diff --git a/src/cryptonote_core/blockchain.cpp b/src/cryptonote_core/blockchain.cpp index b4b1a4c86..b2d979432 100644 --- a/src/cryptonote_core/blockchain.cpp +++ b/src/cryptonote_core/blockchain.cpp @@ -237,8 +237,9 @@ bool Blockchain::init(const std::string& config_folder, bool testnet) m_testnet = testnet; boost::filesystem::path folder(m_config_folder); + folder /= "lmdb"; - LOG_PRINT_L0("Loading blockchain..."); + LOG_PRINT_L0("Loading blockchain from folder " << folder.c_str() << " ..."); //FIXME: update filename for BlockchainDB const std::string filename = folder.string(); From 3676ac5841b1e433830323b3c51bf78e96947054 Mon Sep 17 00:00:00 2001 From: warptangent Date: Wed, 11 Feb 2015 15:55:53 -0800 Subject: [PATCH 09/16] Add profiling to block and tx processing --- src/cryptonote_core/blockchain_db.cpp | 38 +++++++++++++++++++++++++++ src/cryptonote_core/blockchain_db.h | 10 +++++++ 2 files changed, 48 insertions(+) diff --git a/src/cryptonote_core/blockchain_db.cpp b/src/cryptonote_core/blockchain_db.cpp index 0ee10bd4d..615b08814 100644 --- a/src/cryptonote_core/blockchain_db.cpp +++ b/src/cryptonote_core/blockchain_db.cpp @@ -28,6 +28,7 @@ #include "cryptonote_core/blockchain_db.h" #include "cryptonote_format_utils.h" +#include "profile_tools.h" using epee::string_tools::pod_to_hex; @@ -73,18 +74,29 @@ uint64_t BlockchainDB::add_block( const block& blk , const std::vector& txs ) { + TIME_MEASURE_START(time1); crypto::hash blk_hash = get_block_hash(blk); + TIME_MEASURE_FINISH(time1); + time_blk_hash += time1; // call out to subclass implementation to add the block & metadata + time1 = epee::misc_utils::get_tick_count(); add_block(blk, block_size, cumulative_difficulty, coins_generated); + TIME_MEASURE_FINISH(time1); + time_add_block1 += time1; // call out to add the transactions + time1 = epee::misc_utils::get_tick_count(); add_transaction(blk_hash, blk.miner_tx); for (const transaction& tx : txs) { add_transaction(blk_hash, tx); } + TIME_MEASURE_FINISH(time1); + time_add_transaction += time1; + + ++num_calls; return height(); } @@ -119,4 +131,30 @@ void BlockchainDB::remove_transaction(const crypto::hash& tx_hash) remove_transaction_data(tx_hash, tx); } +void BlockchainDB::reset_stats() +{ + num_calls = 0; + time_blk_hash = 0; + time_add_block1 = 0; + time_add_transaction = 0; +} + +void BlockchainDB::show_stats() +{ + LOG_PRINT_L1(ENDL + << "*********************************" + << ENDL + << "num_calls: " << num_calls + << ENDL + << "time_blk_hash: " << time_blk_hash << "ms" + << ENDL + << "time_add_block1: " << time_add_block1 << "ms" + << ENDL + << "time_add_transaction: " << time_add_transaction << "ms" + << ENDL + << "*********************************" + << ENDL + ); +} + } // namespace cryptonote diff --git a/src/cryptonote_core/blockchain_db.h b/src/cryptonote_core/blockchain_db.h index db56c7c07..531de04bd 100644 --- a/src/cryptonote_core/blockchain_db.h +++ b/src/cryptonote_core/blockchain_db.h @@ -301,12 +301,22 @@ private: // helper function to remove transaction from blockchain void remove_transaction(const crypto::hash& tx_hash); + uint64_t num_calls = 0; + uint64_t time_blk_hash = 0; + uint64_t time_add_block1 = 0; + uint64_t time_add_transaction = 0; + public: // virtual dtor virtual ~BlockchainDB() { }; + // reset profiling stats + void reset_stats(); + + // show profiling stats + void show_stats(); // open the db at location , or create it if there isn't one. virtual void open(const std::string& filename) = 0; From 8909d7d82eaf2b130b7d36fb5db7863170356b15 Mon Sep 17 00:00:00 2001 From: warptangent Date: Wed, 11 Feb 2015 15:55:53 -0800 Subject: [PATCH 10/16] Improve block and tx processing efficiency by less repeat hashing BlockchainLMDB::add_block() BlockchainLMDB::add_transaction_data() BlockchainDB::add_transaction() --- .../BlockchainDB_impl/db_lmdb.cpp | 7 +++--- .../BlockchainDB_impl/db_lmdb.h | 3 ++- src/cryptonote_core/blockchain_db.cpp | 24 +++++++++++++++---- src/cryptonote_core/blockchain_db.h | 5 ++-- 4 files changed, 28 insertions(+), 11 deletions(-) diff --git a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp index f7e8c5dda..849ec8f6d 100644 --- a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp +++ b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp @@ -152,12 +152,13 @@ void BlockchainLMDB::add_block( const block& blk , const size_t& block_size , const difficulty_type& cumulative_difficulty , const uint64_t& coins_generated + , const crypto::hash& blk_hash ) { LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); - MDB_val_copy val_h(get_block_hash(blk)); + MDB_val_copy val_h(blk_hash); MDB_val unused; if (mdb_get(*m_write_txn, m_block_heights, &val_h, &unused) == 0) throw1(BLOCK_EXISTS("Attempting to add block that's already in the db")); @@ -243,12 +244,12 @@ void BlockchainLMDB::remove_block() throw1(DB_ERROR("Failed to add removal of block hash to db transaction")); } -void BlockchainLMDB::add_transaction_data(const crypto::hash& blk_hash, const transaction& tx) +void BlockchainLMDB::add_transaction_data(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash& tx_hash) { LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); - MDB_val_copy val_h(get_transaction_hash(tx)); + MDB_val_copy val_h(tx_hash); MDB_val unused; if (mdb_get(*m_write_txn, m_txs, &val_h, &unused) == 0) throw1(TX_EXISTS("Attempting to add transaction that's already in the db")); diff --git a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.h b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.h index 2513aa48a..29fc52fbd 100644 --- a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.h +++ b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.h @@ -184,11 +184,12 @@ private: , const size_t& block_size , const difficulty_type& cumulative_difficulty , const uint64_t& coins_generated + , const crypto::hash& block_hash ); virtual void remove_block(); - virtual void add_transaction_data(const crypto::hash& blk_hash, const transaction& tx); + virtual void add_transaction_data(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash& tx_hash); virtual void remove_transaction_data(const crypto::hash& tx_hash, const transaction& tx); diff --git a/src/cryptonote_core/blockchain_db.cpp b/src/cryptonote_core/blockchain_db.cpp index 615b08814..af0c9487f 100644 --- a/src/cryptonote_core/blockchain_db.cpp +++ b/src/cryptonote_core/blockchain_db.cpp @@ -42,11 +42,21 @@ void BlockchainDB::pop_block() pop_block(blk, txs); } -void BlockchainDB::add_transaction(const crypto::hash& blk_hash, const transaction& tx) +void BlockchainDB::add_transaction(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash* tx_hash_ptr) { - crypto::hash tx_hash = get_transaction_hash(tx); + crypto::hash tx_hash; + if (!tx_hash_ptr) + { + // should only need to compute hash for miner transactions + tx_hash = get_transaction_hash(tx); + LOG_PRINT_L3("null tx_hash_ptr - needed to compute: " << tx_hash); + } + else + { + tx_hash = *tx_hash_ptr; + } - add_transaction_data(blk_hash, tx); + add_transaction_data(blk_hash, tx, tx_hash); // iterate tx.vout using indices instead of C++11 foreach syntax because // we need the index @@ -81,7 +91,7 @@ uint64_t BlockchainDB::add_block( const block& blk // call out to subclass implementation to add the block & metadata time1 = epee::misc_utils::get_tick_count(); - add_block(blk, block_size, cumulative_difficulty, coins_generated); + add_block(blk, block_size, cumulative_difficulty, coins_generated, blk_hash); TIME_MEASURE_FINISH(time1); time_add_block1 += time1; @@ -89,9 +99,13 @@ uint64_t BlockchainDB::add_block( const block& blk time1 = epee::misc_utils::get_tick_count(); add_transaction(blk_hash, blk.miner_tx); + int tx_i = 0; + crypto::hash tx_hash = null_hash; for (const transaction& tx : txs) { - add_transaction(blk_hash, tx); + tx_hash = blk.tx_hashes[tx_i]; + add_transaction(blk_hash, tx, &tx_hash); + ++tx_i; } TIME_MEASURE_FINISH(time1); time_add_transaction += time1; diff --git a/src/cryptonote_core/blockchain_db.h b/src/cryptonote_core/blockchain_db.h index 531de04bd..02620c406 100644 --- a/src/cryptonote_core/blockchain_db.h +++ b/src/cryptonote_core/blockchain_db.h @@ -265,13 +265,14 @@ private: , const size_t& block_size , const difficulty_type& cumulative_difficulty , const uint64_t& coins_generated + , const crypto::hash& blk_hash ) = 0; // tells the subclass to remove data about the top block virtual void remove_block() = 0; // tells the subclass to store the transaction and its metadata - virtual void add_transaction_data(const crypto::hash& blk_hash, const transaction& tx) = 0; + virtual void add_transaction_data(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash& tx_hash) = 0; // tells the subclass to remove data about a transaction virtual void remove_transaction_data(const crypto::hash& tx_hash, const transaction& tx) = 0; @@ -296,7 +297,7 @@ private: void pop_block(); // helper function for add_transactions, to add each individual tx - void add_transaction(const crypto::hash& blk_hash, const transaction& tx); + void add_transaction(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash* tx_hash_ptr = NULL); // helper function to remove transaction from blockchain void remove_transaction(const crypto::hash& tx_hash); From 58ecc58be13befb8b5d0e6b847987b2fd9635d2c Mon Sep 17 00:00:00 2001 From: warptangent Date: Wed, 11 Feb 2015 15:55:53 -0800 Subject: [PATCH 11/16] BlockchainLMDB: Add support for batch transactions --- .../BlockchainDB_impl/db_lmdb.cpp | 294 ++++++++++++++---- .../BlockchainDB_impl/db_lmdb.h | 50 ++- 2 files changed, 289 insertions(+), 55 deletions(-) diff --git a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp index 849ec8f6d..c30a785ed 100644 --- a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp +++ b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp @@ -600,15 +600,23 @@ void BlockchainLMDB::check_open() const BlockchainLMDB::~BlockchainLMDB() { LOG_PRINT_L3("BlockchainLMDB::" << __func__); + + // batch transaction shouldn't be active at this point. If it is, consider it aborted. + if (m_batch_active) + batch_abort(); } -BlockchainLMDB::BlockchainLMDB() +BlockchainLMDB::BlockchainLMDB(bool batch_transactions) { LOG_PRINT_L3("BlockchainLMDB::" << __func__); // initialize folder to something "safe" just in case // someone accidentally misuses this class... m_folder = "thishsouldnotexistbecauseitisgibberish"; m_open = false; + + m_batch_transactions = batch_transactions; + m_write_txn = nullptr; + m_batch_active = false; m_height = 0; } @@ -720,6 +728,13 @@ void BlockchainLMDB::create(const std::string& filename) void BlockchainLMDB::close() { LOG_PRINT_L3("BlockchainLMDB::" << __func__); + if (m_batch_active) + { + LOG_PRINT_L3("close() first calling batch_abort() due to active batch transaction"); + batch_abort(); + } + this->sync(); + // FIXME: not yet thread safe!!! Use with care. mdb_env_close(m_env); } @@ -727,7 +742,13 @@ void BlockchainLMDB::close() void BlockchainLMDB::sync() { LOG_PRINT_L3("BlockchainLMDB::" << __func__); - // LMDB documentation leads me to believe this is unnecessary + + // Does nothing unless LMDB environment was opened with MDB_NOSYNC or in part + // MDB_NOMETASYNC. Force flush to be synchronous. + if (auto result = mdb_env_sync(m_env, true)) + { + throw0(DB_ERROR(std::string("Failed to sync database").append(mdb_strerror(result)).c_str())); + } } void BlockchainLMDB::reset() @@ -867,12 +888,17 @@ uint64_t BlockchainLMDB::get_block_timestamp(const uint64_t& height) const check_open(); txn_safe txn; - if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) - throw0(DB_ERROR("Failed to create a transaction for the db")); - + txn_safe* txn_ptr = &txn; + if (m_batch_active) + txn_ptr = m_write_txn; + else + { + if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + } MDB_val_copy key(height); MDB_val result; - auto get_result = mdb_get(txn, m_block_timestamps, &key, &result); + auto get_result = mdb_get(*txn_ptr, m_block_timestamps, &key, &result); if (get_result == MDB_NOTFOUND) { throw0(DB_ERROR(std::string("Attempt to get timestamp from height ").append(boost::lexical_cast(height)).append(" failed -- timestamp not in db").c_str())); @@ -880,7 +906,8 @@ uint64_t BlockchainLMDB::get_block_timestamp(const uint64_t& height) const else if (get_result) throw0(DB_ERROR("Error attempting to retrieve a timestamp from the db")); - txn.commit(); + if (! m_batch_active) + txn.commit(); return *(const uint64_t*)result.mv_data; } @@ -904,12 +931,18 @@ size_t BlockchainLMDB::get_block_size(const uint64_t& height) const check_open(); txn_safe txn; - if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) - throw0(DB_ERROR("Failed to create a transaction for the db")); + txn_safe* txn_ptr = &txn; + if (m_batch_active) + txn_ptr = m_write_txn; + else + { + if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + } MDB_val_copy key(height); MDB_val result; - auto get_result = mdb_get(txn, m_block_sizes, &key, &result); + auto get_result = mdb_get(*txn_ptr, m_block_sizes, &key, &result); if (get_result == MDB_NOTFOUND) { throw0(DB_ERROR(std::string("Attempt to get block size from height ").append(boost::lexical_cast(height)).append(" failed -- block size not in db").c_str())); @@ -917,7 +950,8 @@ size_t BlockchainLMDB::get_block_size(const uint64_t& height) const else if (get_result) throw0(DB_ERROR("Error attempting to retrieve a block size from the db")); - txn.commit(); + if (! m_batch_active) + txn.commit(); return *(const size_t*)result.mv_data; } @@ -927,12 +961,17 @@ difficulty_type BlockchainLMDB::get_block_cumulative_difficulty(const uint64_t& check_open(); txn_safe txn; - if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) - throw0(DB_ERROR("Failed to create a transaction for the db")); - + txn_safe* txn_ptr = &txn; + if (m_batch_active) + txn_ptr = m_write_txn; + else + { + if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + } MDB_val_copy key(height); MDB_val result; - auto get_result = mdb_get(txn, m_block_diffs, &key, &result); + auto get_result = mdb_get(*txn_ptr, m_block_diffs, &key, &result); if (get_result == MDB_NOTFOUND) { throw0(DB_ERROR(std::string("Attempt to get cumulative difficulty from height ").append(boost::lexical_cast(height)).append(" failed -- difficulty not in db").c_str())); @@ -940,7 +979,8 @@ difficulty_type BlockchainLMDB::get_block_cumulative_difficulty(const uint64_t& else if (get_result) throw0(DB_ERROR("Error attempting to retrieve a cumulative difficulty from the db")); - txn.commit(); + if (! m_batch_active) + txn.commit(); return *(difficulty_type*)result.mv_data; } @@ -967,12 +1007,18 @@ uint64_t BlockchainLMDB::get_block_already_generated_coins(const uint64_t& heigh check_open(); txn_safe txn; - if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) - throw0(DB_ERROR("Failed to create a transaction for the db")); + txn_safe* txn_ptr = &txn; + if (m_batch_active) + txn_ptr = m_write_txn; + else + { + if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + } MDB_val_copy key(height); MDB_val result; - auto get_result = mdb_get(txn, m_block_coins, &key, &result); + auto get_result = mdb_get(*txn_ptr, m_block_coins, &key, &result); if (get_result == MDB_NOTFOUND) { throw0(DB_ERROR(std::string("Attempt to get generated coins from height ").append(boost::lexical_cast(height)).append(" failed -- block size not in db").c_str())); @@ -980,7 +1026,8 @@ uint64_t BlockchainLMDB::get_block_already_generated_coins(const uint64_t& heigh else if (get_result) throw0(DB_ERROR("Error attempting to retrieve a total generated coins from the db")); - txn.commit(); + if (! m_batch_active) + txn.commit(); return *(const uint64_t*)result.mv_data; } @@ -990,20 +1037,28 @@ crypto::hash BlockchainLMDB::get_block_hash_from_height(const uint64_t& height) check_open(); txn_safe txn; - if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) - throw0(DB_ERROR("Failed to create a transaction for the db")); + txn_safe* txn_ptr = &txn; + if (m_batch_active) + txn_ptr = m_write_txn; + else + { + if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + } MDB_val_copy key(height); MDB_val result; - auto get_result = mdb_get(txn, m_block_hashes, &key, &result); + auto get_result = mdb_get(*txn_ptr, m_block_hashes, &key, &result); if (get_result == MDB_NOTFOUND) { throw0(BLOCK_DNE(std::string("Attempt to get hash from height ").append(boost::lexical_cast(height)).append(" failed -- hash not in db").c_str())); } else if (get_result) - throw0(DB_ERROR("Error attempting to retrieve a block hash from the db")); + throw0(DB_ERROR(std::string("Error attempting to retrieve a block hash from the db: "). + append(mdb_strerror(get_result)).c_str())); - txn.commit(); + if (! m_batch_active) + txn.commit(); return *(crypto::hash*)result.mv_data; } @@ -1075,6 +1130,10 @@ bool BlockchainLMDB::tx_exists(const crypto::hash& h) const LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); + if (m_batch_active) + { + LOG_PRINT_L0("WARNING: active batch transaction while creating a read-only txn in tx_exists()"); + } txn_safe txn; if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) throw0(DB_ERROR("Failed to create a transaction for the db")); @@ -1120,12 +1179,18 @@ transaction BlockchainLMDB::get_tx(const crypto::hash& h) const check_open(); txn_safe txn; - if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) - throw0(DB_ERROR("Failed to create a transaction for the db")); + txn_safe* txn_ptr = &txn; + if (m_batch_active) + txn_ptr = m_write_txn; + else + { + if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + } MDB_val_copy key(h); MDB_val result; - auto get_result = mdb_get(txn, m_txs, &key, &result); + auto get_result = mdb_get(*txn_ptr, m_txs, &key, &result); if (get_result == MDB_NOTFOUND) throw1(TX_DNE(std::string("tx with hash ").append(epee::string_tools::pod_to_hex(h)).append(" not found in db").c_str())); else if (get_result) @@ -1137,6 +1202,8 @@ transaction BlockchainLMDB::get_tx(const crypto::hash& h) const transaction tx; if (!parse_and_validate_tx_from_blob(bd, tx)) throw0(DB_ERROR("Failed to parse tx from blob retrieved from the db")); + if (! m_batch_active) + txn.commit(); return tx; } @@ -1178,13 +1245,27 @@ uint64_t BlockchainLMDB::get_tx_block_height(const crypto::hash& h) const LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); + // If m_batch_active is set, a batch transaction exists beyond this class, + // such as a batch import with verification enabled, or possibly (later) a + // batch network sync. + // + // A regular network sync without batching would be expected to open a new + // read transaction here, as validation is done prior to the write for block + // and tx data. + txn_safe txn; - if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) - throw0(DB_ERROR("Failed to create a transaction for the db")); + txn_safe* txn_ptr = &txn; + if (m_batch_active) + txn_ptr = m_write_txn; + else + { + if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + } MDB_val_copy key(h); MDB_val result; - auto get_result = mdb_get(txn, m_tx_heights, &key, &result); + auto get_result = mdb_get(*txn_ptr, m_tx_heights, &key, &result); if (get_result == MDB_NOTFOUND) { throw1(TX_DNE(std::string("tx height with hash ").append(epee::string_tools::pod_to_hex(h)).append(" not found in db").c_str())); @@ -1192,6 +1273,9 @@ uint64_t BlockchainLMDB::get_tx_block_height(const crypto::hash& h) const else if (get_result) throw0(DB_ERROR("DB error attempting to fetch tx height from hash")); + if (! m_batch_active) + txn.commit(); + return *(const uint64_t*)result.mv_data; } @@ -1334,13 +1418,18 @@ tx_out_index BlockchainLMDB::get_output_tx_and_index_from_global(const uint64_t& check_open(); txn_safe txn; - if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) - throw0(DB_ERROR("Failed to create a transaction for the db")); - + txn_safe* txn_ptr = &txn; + if (m_batch_active) + txn_ptr = m_write_txn; + else + { + if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + } MDB_val_copy k(index); MDB_val v; - auto get_result = mdb_get(txn, m_output_txs, &k, &v); + auto get_result = mdb_get(*txn_ptr, m_output_txs, &k, &v); if (get_result == MDB_NOTFOUND) throw1(OUTPUT_DNE("output with given index not in db")); else if (get_result) @@ -1348,11 +1437,13 @@ tx_out_index BlockchainLMDB::get_output_tx_and_index_from_global(const uint64_t& crypto::hash tx_hash = *(crypto::hash*)v.mv_data; - get_result = mdb_get(txn, m_output_indices, &k, &v); + get_result = mdb_get(*txn_ptr, m_output_indices, &k, &v); if (get_result == MDB_NOTFOUND) throw1(OUTPUT_DNE("output with given index not in db")); else if (get_result) throw0(DB_ERROR("DB error attempting to fetch output tx index")); + if (! m_batch_active) + txn.commit(); return tx_out_index(tx_hash, *(const uint64_t *)v.mv_data); } @@ -1363,10 +1454,15 @@ tx_out_index BlockchainLMDB::get_output_tx_and_index(const uint64_t& amount, con check_open(); txn_safe txn; - if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) - throw0(DB_ERROR("Failed to create a transaction for the db")); - - lmdb_cur cur(txn, m_output_amounts); + txn_safe* txn_ptr = &txn; + if (m_batch_active) + txn_ptr = m_write_txn; + else + { + if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + } + lmdb_cur cur(*txn_ptr, m_output_amounts); MDB_val_copy k(amount); MDB_val v; @@ -1395,7 +1491,8 @@ tx_out_index BlockchainLMDB::get_output_tx_and_index(const uint64_t& amount, con cur.close(); - txn.commit(); + if (! m_batch_active) + txn.commit(); return get_output_tx_and_index_from_global(glob_index); } @@ -1538,6 +1635,85 @@ bool BlockchainLMDB::has_key_image(const crypto::key_image& img) const return false; } +void BlockchainLMDB::batch_start() +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + if (! m_batch_transactions) + throw0(DB_ERROR("batch transactions not enabled")); + if (m_batch_active) + throw0(DB_ERROR("batch transaction already in progress")); + if (m_write_txn) + throw0(DB_ERROR("batch transaction attempted, but m_write_txn already in use")); + check_open(); + // NOTE: need to make sure it's destroyed properly when done + if (mdb_txn_begin(m_env, NULL, 0, m_write_batch_txn)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + // indicates this transaction is for batch transactions, but not whether it's + // active + m_write_batch_txn.m_batch_txn = true; + m_write_txn = &m_write_batch_txn; + m_batch_active = true; + LOG_PRINT_L3("batch transaction: begin"); +} + +void BlockchainLMDB::batch_commit() +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + if (! m_batch_transactions) + throw0(DB_ERROR("batch transactions not enabled")); + if (! m_batch_active) + throw0(DB_ERROR("batch transaction not in progress")); + check_open(); + LOG_PRINT_L3("batch transaction: committing..."); + m_write_txn->commit(); + LOG_PRINT_L3("batch transaction: committed"); + + if (mdb_txn_begin(m_env, NULL, 0, m_write_batch_txn)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + if (! m_write_batch_txn.m_batch_txn) + throw0(DB_ERROR("m_write_batch_txn not marked as a batch transaction")); + m_write_txn = &m_write_batch_txn; +} + +void BlockchainLMDB::batch_stop() +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + if (! m_batch_transactions) + throw0(DB_ERROR("batch transactions not enabled")); + if (! m_batch_active) + throw0(DB_ERROR("batch transaction not in progress")); + check_open(); + LOG_PRINT_L3("batch transaction: committing..."); + m_write_txn->commit(); + // for destruction of batch transaction + m_write_txn = nullptr; + m_batch_active = false; + LOG_PRINT_L3("batch transaction: end"); +} + +void BlockchainLMDB::batch_abort() +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + if (! m_batch_transactions) + throw0(DB_ERROR("batch transactions not enabled")); + if (! m_batch_active) + throw0(DB_ERROR("batch transaction not in progress")); + check_open(); + // for destruction of batch transaction + m_write_txn = nullptr; + // explicitly call in case mdb_env_close() (BlockchainLMDB::close()) called before BlockchainLMDB destructor called. + m_write_batch_txn.abort(); + m_batch_active = false; + LOG_PRINT_L3("batch transaction: aborted"); +} + +void BlockchainLMDB::set_batch_transactions(bool batch_transactions) +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + m_batch_transactions = batch_transactions; + LOG_PRINT_L3("batch transactions " << (m_batch_transactions ? "enabled" : "disabled")); +} + uint64_t BlockchainLMDB::add_block( const block& blk , const size_t& block_size , const difficulty_type& cumulative_difficulty @@ -1547,23 +1723,31 @@ uint64_t BlockchainLMDB::add_block( const block& blk { LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); + txn_safe txn; - if (mdb_txn_begin(m_env, NULL, 0, txn)) - throw0(DB_ERROR("Failed to create a transaction for the db")); - m_write_txn = &txn; + if (! m_batch_active) + { + if (mdb_txn_begin(m_env, NULL, 0, txn)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + m_write_txn = &txn; + } uint64_t num_outputs = m_num_outputs; try { BlockchainDB::add_block(blk, block_size, cumulative_difficulty, coins_generated, txs); - m_write_txn = NULL; + if (! m_batch_active) + { + m_write_txn = NULL; - txn.commit(); + txn.commit(); + } } catch (...) { m_num_outputs = num_outputs; - m_write_txn = NULL; + if (! m_batch_active) + m_write_txn = NULL; throw; } @@ -1574,17 +1758,23 @@ void BlockchainLMDB::pop_block(block& blk, std::vector& txs) { LOG_PRINT_L3("BlockchainLMDB::" << __func__); txn_safe txn; - if (mdb_txn_begin(m_env, NULL, 0, txn)) - throw0(DB_ERROR("Failed to create a transaction for the db")); - m_write_txn = &txn; + if (! m_batch_active) + { + if (mdb_txn_begin(m_env, NULL, 0, txn)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + m_write_txn = &txn; + } uint64_t num_outputs = m_num_outputs; try { BlockchainDB::pop_block(blk, txs); - m_write_txn = NULL; + if (! m_batch_active) + { + m_write_txn = NULL; - txn.commit(); + txn.commit(); + } } catch (...) { diff --git a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.h b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.h index 29fc52fbd..0b4b5ec1a 100644 --- a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.h +++ b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.h @@ -38,8 +38,23 @@ struct txn_safe txn_safe() : m_txn(NULL) { } ~txn_safe() { - if(m_txn != NULL) + LOG_PRINT_L3("txn_safe: destructor"); + if (m_txn != NULL) { + if (m_batch_txn) // this is a batch txn and should have been handled before this point for safety + { + LOG_PRINT_L0("WARNING: txn_safe: m_txn is a batch txn and it's not NULL in destructor - calling mdb_txn_abort()"); + } + else + { + // Example of when this occurs: a lookup fails, so a read-only txn is + // aborted through this destructor. However, successful read-only txns + // ideally should have been committed when done and not end up here. + // + // NOTE: not sure if this is ever reached for a non-batch write + // transaction, but it's probably not ideal if it did. + LOG_PRINT_L3("txn_safe: m_txn not NULL in destructor - calling mdb_txn_abort()"); + } mdb_txn_abort(m_txn); } } @@ -60,6 +75,24 @@ struct txn_safe m_txn = NULL; } + // This should only be needed for batch transaction which must be ensured to + // be aborted before mdb_env_close, not after. So we can't rely on + // BlockchainLMDB destructor to call txn_safe destructor, as that's too late + // to properly abort, since mdb_env_close would have been called earlier. + void abort() + { + LOG_PRINT_L3("txn_safe: abort()"); + if(m_txn != NULL) + { + mdb_txn_abort(m_txn); + m_txn = NULL; + } + else + { + LOG_PRINT_L0("WARNING: txn_safe: abort() called, but m_txn is NULL"); + } + } + operator MDB_txn*() { return m_txn; @@ -71,13 +104,14 @@ struct txn_safe } MDB_txn* m_txn; + bool m_batch_txn = false; }; class BlockchainLMDB : public BlockchainDB { public: - BlockchainLMDB(); + BlockchainLMDB(bool batch_transactions=false); ~BlockchainLMDB(); virtual void open(const std::string& filename); @@ -177,6 +211,12 @@ public: , const std::vector& txs ); + virtual void set_batch_transactions(bool batch_transactions); + virtual void batch_start(); + virtual void batch_commit(); + virtual void batch_stop(); + virtual void batch_abort(); + virtual void pop_block(block& blk, std::vector& txs); private: @@ -264,7 +304,11 @@ private: uint64_t m_height; uint64_t m_num_outputs; std::string m_folder; - txn_safe* m_write_txn; + txn_safe* m_write_txn; // may point to either a short-lived txn or a batch txn + txn_safe m_write_batch_txn; // persist batch txn outside of BlockchainLMDB + + bool m_batch_transactions; // support for batch transactions + bool m_batch_active; // whether batch transaction is in progress }; } // namespace cryptonote From b7a2d84919bc28b3747974509f932e64907e357a Mon Sep 17 00:00:00 2001 From: warptangent Date: Wed, 18 Feb 2015 20:52:44 -0800 Subject: [PATCH 12/16] BlockchainLMDB: Add check for open database to two functions --- src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp index c30a785ed..c3c0724ce 100644 --- a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp +++ b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp @@ -742,6 +742,7 @@ void BlockchainLMDB::close() void BlockchainLMDB::sync() { LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); // Does nothing unless LMDB environment was opened with MDB_NOSYNC or in part // MDB_NOMETASYNC. Force flush to be synchronous. @@ -1757,6 +1758,8 @@ uint64_t BlockchainLMDB::add_block( const block& blk void BlockchainLMDB::pop_block(block& blk, std::vector& txs) { LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + txn_safe txn; if (! m_batch_active) { From 7a66b8bbcfb3f8084141d8ffb1154a3a514a9ee5 Mon Sep 17 00:00:00 2001 From: warptangent Date: Wed, 11 Feb 2015 15:55:53 -0800 Subject: [PATCH 13/16] BlockchainDB: Add virtual function declarations for batch transactions --- src/cryptonote_core/blockchain_db.h | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/cryptonote_core/blockchain_db.h b/src/cryptonote_core/blockchain_db.h index 02620c406..25e9781f8 100644 --- a/src/cryptonote_core/blockchain_db.h +++ b/src/cryptonote_core/blockchain_db.h @@ -347,6 +347,9 @@ public: // release db lock virtual void unlock() = 0; + virtual void batch_start() = 0; + virtual void batch_stop() = 0; + virtual void set_batch_transactions(bool) = 0; // adds a block with the given metadata to the top of the blockchain, returns the new height // NOTE: subclass implementations of this (or the functions it calls) need From 8529c0ea9aa305f097d4a2c89d581163e208e04f Mon Sep 17 00:00:00 2001 From: warptangent Date: Wed, 11 Feb 2015 15:55:53 -0800 Subject: [PATCH 14/16] BlockchainDB, BlockchainLMDB: Add profiling for DB commits --- src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp | 10 ++++++++++ src/cryptonote_core/blockchain_db.cpp | 3 +++ src/cryptonote_core/blockchain_db.h | 5 +++++ 3 files changed, 18 insertions(+) diff --git a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp index c3c0724ce..25c25f399 100644 --- a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp +++ b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp @@ -33,6 +33,7 @@ #include "cryptonote_core/cryptonote_format_utils.h" #include "crypto/crypto.h" +#include "profile_tools.h" using epee::string_tools::pod_to_hex; @@ -1666,7 +1667,10 @@ void BlockchainLMDB::batch_commit() throw0(DB_ERROR("batch transaction not in progress")); check_open(); LOG_PRINT_L3("batch transaction: committing..."); + TIME_MEASURE_START(time1); m_write_txn->commit(); + TIME_MEASURE_FINISH(time1); + time_commit1 += time1; LOG_PRINT_L3("batch transaction: committed"); if (mdb_txn_begin(m_env, NULL, 0, m_write_batch_txn)) @@ -1685,7 +1689,10 @@ void BlockchainLMDB::batch_stop() throw0(DB_ERROR("batch transaction not in progress")); check_open(); LOG_PRINT_L3("batch transaction: committing..."); + TIME_MEASURE_START(time1); m_write_txn->commit(); + TIME_MEASURE_FINISH(time1); + time_commit1 += time1; // for destruction of batch transaction m_write_txn = nullptr; m_batch_active = false; @@ -1741,7 +1748,10 @@ uint64_t BlockchainLMDB::add_block( const block& blk { m_write_txn = NULL; + TIME_MEASURE_START(time1); txn.commit(); + TIME_MEASURE_FINISH(time1); + time_commit1 += time1; } } catch (...) diff --git a/src/cryptonote_core/blockchain_db.cpp b/src/cryptonote_core/blockchain_db.cpp index af0c9487f..74de19585 100644 --- a/src/cryptonote_core/blockchain_db.cpp +++ b/src/cryptonote_core/blockchain_db.cpp @@ -151,6 +151,7 @@ void BlockchainDB::reset_stats() time_blk_hash = 0; time_add_block1 = 0; time_add_transaction = 0; + time_commit1 = 0; } void BlockchainDB::show_stats() @@ -166,6 +167,8 @@ void BlockchainDB::show_stats() << ENDL << "time_add_transaction: " << time_add_transaction << "ms" << ENDL + << "time_commit1: " << time_commit1 << "ms" + << ENDL << "*********************************" << ENDL ); diff --git a/src/cryptonote_core/blockchain_db.h b/src/cryptonote_core/blockchain_db.h index 25e9781f8..ba0e5738b 100644 --- a/src/cryptonote_core/blockchain_db.h +++ b/src/cryptonote_core/blockchain_db.h @@ -308,6 +308,11 @@ private: uint64_t time_add_transaction = 0; +protected: + + uint64_t time_commit1 = 0; + + public: // virtual dtor From 83fb6d8d07c0b473e7b53a35d8e69f586bd1945e Mon Sep 17 00:00:00 2001 From: warptangent Date: Tue, 10 Feb 2015 20:01:02 -0800 Subject: [PATCH 15/16] BlockchainLMDB: Add batch transaction support to tx_exists() --- .../BlockchainDB_impl/db_lmdb.cpp | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp index 25c25f399..4e9f6cb28 100644 --- a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp +++ b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp @@ -1132,20 +1132,23 @@ bool BlockchainLMDB::tx_exists(const crypto::hash& h) const LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); - if (m_batch_active) - { - LOG_PRINT_L0("WARNING: active batch transaction while creating a read-only txn in tx_exists()"); - } txn_safe txn; - if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) - throw0(DB_ERROR("Failed to create a transaction for the db")); + txn_safe* txn_ptr = &txn; + if (m_batch_active) + txn_ptr = m_write_txn; + else + { + if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + } MDB_val_copy key(h); MDB_val result; - auto get_result = mdb_get(txn, m_txs, &key, &result); + auto get_result = mdb_get(*txn_ptr, m_txs, &key, &result); if (get_result == MDB_NOTFOUND) { - txn.commit(); + if (! m_batch_active) + txn.commit(); LOG_PRINT_L1("transaction with hash " << epee::string_tools::pod_to_hex(h) << " not found in db"); return false; } From 6485dacc2fcaab7eb6db63c46716ffdfa0c568c4 Mon Sep 17 00:00:00 2001 From: warptangent Date: Tue, 10 Feb 2015 20:01:02 -0800 Subject: [PATCH 16/16] BlockchainLMDB: Add profiling to tx_exists() --- src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp | 5 ++++- src/cryptonote_core/blockchain_db.cpp | 3 +++ src/cryptonote_core/blockchain_db.h | 1 + 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp index 4e9f6cb28..a09d9331c 100644 --- a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp +++ b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp @@ -1126,7 +1126,6 @@ uint64_t BlockchainLMDB::height() const return m_height; } - bool BlockchainLMDB::tx_exists(const crypto::hash& h) const { LOG_PRINT_L3("BlockchainLMDB::" << __func__); @@ -1144,7 +1143,11 @@ bool BlockchainLMDB::tx_exists(const crypto::hash& h) const MDB_val_copy key(h); MDB_val result; + + TIME_MEASURE_START(time1); auto get_result = mdb_get(*txn_ptr, m_txs, &key, &result); + TIME_MEASURE_FINISH(time1); + time_tx_exists += time1; if (get_result == MDB_NOTFOUND) { if (! m_batch_active) diff --git a/src/cryptonote_core/blockchain_db.cpp b/src/cryptonote_core/blockchain_db.cpp index 74de19585..fc8aeef4e 100644 --- a/src/cryptonote_core/blockchain_db.cpp +++ b/src/cryptonote_core/blockchain_db.cpp @@ -149,6 +149,7 @@ void BlockchainDB::reset_stats() { num_calls = 0; time_blk_hash = 0; + time_tx_exists = 0; time_add_block1 = 0; time_add_transaction = 0; time_commit1 = 0; @@ -163,6 +164,8 @@ void BlockchainDB::show_stats() << ENDL << "time_blk_hash: " << time_blk_hash << "ms" << ENDL + << "time_tx_exists: " << time_tx_exists << "ms" + << ENDL << "time_add_block1: " << time_add_block1 << "ms" << ENDL << "time_add_transaction: " << time_add_transaction << "ms" diff --git a/src/cryptonote_core/blockchain_db.h b/src/cryptonote_core/blockchain_db.h index ba0e5738b..2a7fa8f82 100644 --- a/src/cryptonote_core/blockchain_db.h +++ b/src/cryptonote_core/blockchain_db.h @@ -310,6 +310,7 @@ private: protected: + mutable uint64_t time_tx_exists = 0; uint64_t time_commit1 = 0;