From 7ac1db42c2787605688d3a7bac8308286617ecfe Mon Sep 17 00:00:00 2001 From: luigi1111 Date: Sat, 5 Mar 2016 13:30:48 -0600 Subject: [PATCH 01/23] get_payments short ID Add support for short/integrated/encrypted IDs to get_payments RPC --- src/wallet/wallet_rpc_server.cpp | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/src/wallet/wallet_rpc_server.cpp b/src/wallet/wallet_rpc_server.cpp index e8e062e95..ac13d8021 100644 --- a/src/wallet/wallet_rpc_server.cpp +++ b/src/wallet/wallet_rpc_server.cpp @@ -461,6 +461,7 @@ namespace tools bool wallet_rpc_server::on_get_payments(const wallet_rpc::COMMAND_RPC_GET_PAYMENTS::request& req, wallet_rpc::COMMAND_RPC_GET_PAYMENTS::response& res, epee::json_rpc::error& er) { crypto::hash payment_id; + crypto::hash8 payment_id8; cryptonote::blobdata payment_id_blob; if(!epee::string_tools::parse_hexstr_to_binbuff(req.payment_id, payment_id_blob)) { @@ -469,14 +470,22 @@ namespace tools return false; } - if(sizeof(payment_id) != payment_id_blob.size()) - { - er.code = WALLET_RPC_ERROR_CODE_WRONG_PAYMENT_ID; - er.message = "Payment ID has invalid size"; - return false; - } - - payment_id = *reinterpret_cast(payment_id_blob.data()); + if(sizeof(payment_id) == payment_id_blob.size()) + { + payment_id = *reinterpret_cast(payment_id_blob.data()); + } + else if(sizeof(payment_id8) == payment_id_blob.size()) + { + payment_id8 = *reinterpret_cast(payment_id_blob.data()); + memcpy(payment_id.data, payment_id8.data, 8); + memset(payment_id.data + 8, 0, 24); + } + else + { + er.code = WALLET_RPC_ERROR_CODE_WRONG_PAYMENT_ID; + er.message = "Payment ID has invalid size: " + req.payment_id; + return false; + } res.payments.clear(); std::list payment_list; From 309f8f3d44ad5c1ca85751623f46fc1d676c9e36 Mon Sep 17 00:00:00 2001 From: warptangent Date: Sat, 5 Mar 2016 12:54:31 -0800 Subject: [PATCH 02/23] blockchain_utilities: Update documentation --- src/blockchain_utilities/README.md | 58 +++++++++++++++++++++++++++--- 1 file changed, 54 insertions(+), 4 deletions(-) diff --git a/src/blockchain_utilities/README.md b/src/blockchain_utilities/README.md index d763bc4dd..9c69647ad 100644 --- a/src/blockchain_utilities/README.md +++ b/src/blockchain_utilities/README.md @@ -52,18 +52,68 @@ the `blockchain_import` command again, and it will restart from where it left of ## use default settings to import blockchain.raw into database $ blockchain_import -## fast import with large batch size, verification off -$ blockchain_import --batch-size 100000 --verify off +## fast import with large batch size, database mode "fastest", verification off +$ blockchain_import --batch-size 20000 --database lmdb#fastest --verify off + +``` + +### Import options + +`--input-file` +specifies input file path for importing + +default: `/export/blockchain.raw` + +`--output-file` +specifies output file path to export to + +default: `/export/blockchain.raw` + +`--block-stop` +stop at block number + +`--database ` + +`--database #` + +database type: `lmdb, berkeley, memory` + +flags: + +The flag after the # is interpreted as a composite mode/flag if there's only +one (no comma separated arguments). + +The composite mode represents multiple DB flags and support different database types: + +`safe, fast, fastest` + +Database-specific flags can be set instead. + +LMDB flags (more than one may be specified): + +`nosync, nometasync, writemap, mapasync, nordahead` + +BerkeleyDB flags (takes one): + +`txn_write_nosync, txn_nosync, txn_sync` + +``` +## Examples: +$ blockchain_import --database lmdb#fastest +$ blockchain_import --database berkeley#fastest -## LMDB flags can be set by appending them to the database type: -## flags: nosync, nometasync, writemap, mapasync $ blockchain_import --database lmdb#nosync $ blockchain_import --database lmdb#nosync,nometasync + +$ blockchain_import --database berkeley#txn_nosync ``` + ### Blockchain converter with batching `blockchain_converter` has also been updated and includes batching for faster writes. However, on lower RAM systems, this will be slower than using the exporter and importer utilities. The converter needs to keep the blockchain in memory for the duration of the conversion, like the original bitmonerod, thus leaving less memory available to the destination database to operate. +Due to higher resource use, it is recommended to use the importer with an exported file instead of the converter. + ```bash $ blockchain_converter --batch on --batch-size 20000 ``` From 132c666f67937676bea1cc145c242b857f9852a2 Mon Sep 17 00:00:00 2001 From: warptangent Date: Sun, 31 Jan 2016 05:10:14 -0800 Subject: [PATCH 03/23] Update schema for "tx_outputs" to use array containing amount output indices This speeds up wallet refresh by directly retrieving a tx's amount output indices. It removes the indirection and walking the amount output duplicate list for every amount in each requested tx. "tx_outputs" is used by: Amount output indices are needed for wallet refresh. Global output indices are needed for removing a tx. Both amount output indices and global output indices are now stored in an array of 64-bit unsigned ints: tx_outputs[] -> [ ] Previously it was: tx_outputs[] -> duplicate list of The amount output list had to be walked for every amount in order to find each amount's output index, by comparing the amount's global output index with each one in the duplicate list until a match was found. See also d045dfa7ce0bf131681193c97560da26f9f37900 --- src/blockchain_db/blockchain_db.cpp | 9 +- src/blockchain_db/blockchain_db.h | 24 +++- src/blockchain_db/lmdb/db_lmdb.cpp | 205 ++++++++++++---------------- src/blockchain_db/lmdb/db_lmdb.h | 18 ++- 4 files changed, 134 insertions(+), 122 deletions(-) diff --git a/src/blockchain_db/blockchain_db.cpp b/src/blockchain_db/blockchain_db.cpp index a66f4a403..b1b233b58 100644 --- a/src/blockchain_db/blockchain_db.cpp +++ b/src/blockchain_db/blockchain_db.cpp @@ -84,12 +84,19 @@ void BlockchainDB::add_transaction(const crypto::hash& blk_hash, const transacti add_transaction_data(blk_hash, tx, tx_hash); + std::vector amount_output_indices; + std::vector global_output_indices; + // iterate tx.vout using indices instead of C++11 foreach syntax because // we need the index for (uint64_t i = 0; i < tx.vout.size(); ++i) { - add_output(tx_hash, tx.vout[i], i, tx.unlock_time); + uint64_t amount_output_index, global_output_index; + add_output(tx_hash, tx.vout[i], i, tx.unlock_time, amount_output_index, global_output_index); + amount_output_indices.push_back(amount_output_index); + global_output_indices.push_back(global_output_index); } + add_amount_and_global_output_indices(tx_hash, amount_output_indices, global_output_indices); } uint64_t BlockchainDB::add_block( const block& blk diff --git a/src/blockchain_db/blockchain_db.h b/src/blockchain_db/blockchain_db.h index 3396b8c20..2cdaaea6f 100644 --- a/src/blockchain_db/blockchain_db.h +++ b/src/blockchain_db/blockchain_db.h @@ -297,7 +297,19 @@ private: virtual void remove_transaction_data(const crypto::hash& tx_hash, const transaction& tx) = 0; // tells the subclass to store an output - virtual void add_output(const crypto::hash& tx_hash, const tx_out& tx_output, const uint64_t& local_index, const uint64_t unlock_time) = 0; + virtual void add_output(const crypto::hash& tx_hash, + const tx_out& tx_output, + const uint64_t& local_index, + const uint64_t unlock_time, + uint64_t& amount_output_index, + uint64_t& global_output_index + ) = 0; + + // tells the subclass to store indices for a tx's outputs, both amount output indices and global output indices + virtual void add_amount_and_global_output_indices(const crypto::hash& tx_hash, + const std::vector& amount_output_indices, + const std::vector& global_output_indices + ) = 0; // tells the subclass to remove an output virtual void remove_output(const tx_out& tx_output) = 0; @@ -501,9 +513,13 @@ public: virtual bool can_thread_bulk_indices() const = 0; - // return a vector of indices corresponding to the global output index for - // each output in the transaction with hash - virtual std::vector get_tx_output_indices(const crypto::hash& h) const = 0; + // return two vectors of indices: vector of amount output indices and global + // output indices, corresponding to each output in the transaction with hash + // + virtual void get_amount_and_global_output_indices(const crypto::hash& h, + std::vector& amount_output_indices, + std::vector& global_output_indices) const = 0; + // return a vector of indices corresponding to the amount output index for // each output in the transaction with hash virtual std::vector get_tx_amount_output_indices(const crypto::hash& h) const = 0; diff --git a/src/blockchain_db/lmdb/db_lmdb.cpp b/src/blockchain_db/lmdb/db_lmdb.cpp index 8955072b5..0be2728a9 100644 --- a/src/blockchain_db/lmdb/db_lmdb.cpp +++ b/src/blockchain_db/lmdb/db_lmdb.cpp @@ -687,7 +687,12 @@ void BlockchainLMDB::remove_transaction_data(const crypto::hash& tx_hash, const throw1(DB_ERROR(lmdb_error("Failed to add removal of tx outputs to db transaction: ", result).c_str())); } -void BlockchainLMDB::add_output(const crypto::hash& tx_hash, const tx_out& tx_output, const uint64_t& local_index, const uint64_t unlock_time) +void BlockchainLMDB::add_output(const crypto::hash& tx_hash, + const tx_out& tx_output, + const uint64_t& local_index, + const uint64_t unlock_time, + uint64_t& amount_output_index, + uint64_t& global_output_index) { LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); @@ -696,7 +701,6 @@ void BlockchainLMDB::add_output(const crypto::hash& tx_hash, const tx_out& tx_ou int result = 0; CURSOR(output_txs) - CURSOR(tx_outputs) CURSOR(output_indices) CURSOR(output_amounts) CURSOR(output_keys) @@ -707,9 +711,6 @@ void BlockchainLMDB::add_output(const crypto::hash& tx_hash, const tx_out& tx_ou result = mdb_cursor_put(m_cur_output_txs, &k, &v, MDB_APPEND); if (result) throw0(DB_ERROR(lmdb_error("Failed to add output tx hash to db transaction: ", result).c_str())); - result = mdb_cursor_put(m_cur_tx_outputs, &v, &k, 0); - if (result) - throw0(DB_ERROR(lmdb_error("Failed to add to db transaction: ", result).c_str())); MDB_val_copy val_local_index(local_index); result = mdb_cursor_put(m_cur_output_indices, &k, &val_local_index, MDB_APPEND); @@ -721,6 +722,14 @@ void BlockchainLMDB::add_output(const crypto::hash& tx_hash, const tx_out& tx_ou if (result) throw0(DB_ERROR(lmdb_error("Failed to add output amount to db transaction: ", result).c_str())); + size_t num_elems = 0; + result = mdb_cursor_count(m_cur_output_amounts, &num_elems); + if (result) + throw0(DB_ERROR(std::string("Failed to get number of outputs for amount: ").append(mdb_strerror(result)).c_str())); + + amount_output_index = num_elems - 1; + global_output_index = m_num_outputs; + if (tx_output.target.type() == typeid(txout_to_key)) { output_data_t od; @@ -741,40 +750,57 @@ void BlockchainLMDB::add_output(const crypto::hash& tx_hash, const tx_out& tx_ou m_num_outputs++; } +void BlockchainLMDB::add_amount_and_global_output_indices(const crypto::hash& tx_hash, + const std::vector& amount_output_indices, + const std::vector& global_output_indices) +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + mdb_txn_cursors *m_cursors = &m_wcursors; + CURSOR(tx_outputs) + + int result = 0; + + MDB_val_copy k(tx_hash); + + int num_outputs = amount_output_indices.size(); + std::unique_ptr paired_indices(new uint64_t[2*num_outputs]); + for (int i = 0; i < num_outputs; ++i) + { + paired_indices[2*i] = amount_output_indices[i]; + paired_indices[2*i+1] = global_output_indices[i]; + } + + MDB_val v; + v.mv_data = (void*)paired_indices.get(); + v.mv_size = sizeof(uint64_t) * 2 * num_outputs; + // LOG_PRINT_L1("tx_outputs[tx_hash] size: " << v.mv_size); + + result = mdb_cursor_put(m_cur_tx_outputs, &k, &v, 0); + if (result) + throw0(DB_ERROR(std::string("Failed to add to db transaction: ").append(mdb_strerror(result)).c_str())); +} + void BlockchainLMDB::remove_tx_outputs(const crypto::hash& tx_hash, const transaction& tx) { LOG_PRINT_L3("BlockchainLMDB::" << __func__); - mdb_txn_cursors *m_cursors = &m_wcursors; - MDB_val_copy k(tx_hash); - MDB_val v; - CURSOR(tx_outputs) + // only need global_output_indices + std::vector amount_output_indices, global_output_indices; + get_amount_and_global_output_indices(tx_hash, amount_output_indices, global_output_indices); - auto result = mdb_cursor_get(m_cur_tx_outputs, &k, &v, MDB_SET); - if (result == MDB_NOTFOUND) + if (global_output_indices.empty()) { - LOG_PRINT_L2("tx has no outputs, so no global output indices"); + if (tx.vout.empty()) + LOG_PRINT_L2("tx has no outputs, so no global output indices"); + else + throw0(DB_ERROR("tx has outputs, but no global output indices found")); } - else if (result) - { - throw0(DB_ERROR("DB error attempting to get an output")); - } - else - { - mdb_size_t num_elems = 0; - mdb_cursor_count(m_cur_tx_outputs, &num_elems); - mdb_cursor_get(m_cur_tx_outputs, &k, &v, MDB_LAST_DUP); - - for (uint64_t i = num_elems; i > 0; --i) - { - const tx_out tx_output = tx.vout[i-1]; - remove_output(*(const uint64_t*)v.mv_data, tx_output.amount); - if (i > 1) - { - mdb_cursor_get(m_cur_tx_outputs, &k, &v, MDB_PREV_DUP); - } - } + for (uint64_t i = tx.vout.size(); i > 0; --i) + { + const tx_out tx_output = tx.vout[i-1]; + remove_output(global_output_indices[i-1], tx_output.amount); } } @@ -1066,7 +1092,7 @@ void BlockchainLMDB::open(const std::string& filename, const int mdb_flags) lmdb_db_open(txn, LMDB_TXS, MDB_CREATE, m_txs, "Failed to open db handle for m_txs"); lmdb_db_open(txn, LMDB_TX_UNLOCKS, MDB_CREATE, m_tx_unlocks, "Failed to open db handle for m_tx_unlocks"); lmdb_db_open(txn, LMDB_TX_HEIGHTS, MDB_CREATE, m_tx_heights, "Failed to open db handle for m_tx_heights"); - lmdb_db_open(txn, LMDB_TX_OUTPUTS, MDB_DUPSORT | MDB_CREATE, m_tx_outputs, "Failed to open db handle for m_tx_outputs"); + lmdb_db_open(txn, LMDB_TX_OUTPUTS, MDB_CREATE, m_tx_outputs, "Failed to open db handle for m_tx_outputs"); lmdb_db_open(txn, LMDB_OUTPUT_TXS, MDB_INTEGERKEY | MDB_CREATE, m_output_txs, "Failed to open db handle for m_output_txs"); lmdb_db_open(txn, LMDB_OUTPUT_INDICES, MDB_INTEGERKEY | MDB_CREATE, m_output_indices, "Failed to open db handle for m_output_indices"); @@ -1081,7 +1107,6 @@ void BlockchainLMDB::open(const std::string& filename, const int mdb_flags) lmdb_db_open(txn, LMDB_PROPERTIES, MDB_CREATE, m_properties, "Failed to open db handle for m_properties"); mdb_set_dupsort(txn, m_output_amounts, compare_uint64); - mdb_set_dupsort(txn, m_tx_outputs, compare_uint64); mdb_set_compare(txn, m_spent_keys, compare_hash32); mdb_set_compare(txn, m_block_heights, compare_hash32); mdb_set_compare(txn, m_txs, compare_hash32); @@ -1881,112 +1906,62 @@ tx_out_index BlockchainLMDB::get_output_tx_and_index(const uint64_t& amount, con return indices[0]; } -std::vector BlockchainLMDB::get_tx_output_indices(const crypto::hash& h) const +void BlockchainLMDB::get_amount_and_global_output_indices(const crypto::hash& h, + std::vector& amount_output_indices, + std::vector& global_output_indices) const { LOG_PRINT_L3("BlockchainLMDB::" << __func__); - check_open(); - std::vector index_vec; + check_open(); + + // If a new txn is created, it only needs to read. + // + // This must existence of m_write_txn too (not only m_batch_active), as + // that's what remove_tx_outputs() expected to use instead of creating a new + // txn, regardless of batch mode. Otherwise, remove_tx_outputs() would now + // create a new read-only txn here, which is incorrect. TXN_PREFIX_RDONLY(); const mdb_txn_cursors *m_cursors = m_write_txn ? &m_wcursors : &m_tinfo->m_ti_rcursors; RCURSOR(tx_outputs); + int result = 0; MDB_val_copy k(h); MDB_val v; - auto result = mdb_cursor_get(m_cur_tx_outputs, &k, &v, MDB_SET); + + result = mdb_cursor_get(m_cur_tx_outputs, &k, &v, MDB_SET); if (result == MDB_NOTFOUND) - throw1(OUTPUT_DNE("Attempting to get an output by tx hash and tx index, but output not found")); + LOG_PRINT_L0("WARNING: Unexpected: tx has no amount and global indices stored in " + "tx_outputs, but it should have an empty entry even if it's a tx without " + "outputs"); else if (result) - throw0(DB_ERROR("DB error attempting to get an output")); + throw0(DB_ERROR("DB error attempting to get data for tx_outputs[tx_hash]")); - mdb_size_t num_elems = 0; - mdb_cursor_count(m_cur_tx_outputs, &num_elems); + uint64_t* paired_indices = (uint64_t*)v.mv_data; + int num_elems = v.mv_size / sizeof(uint64_t); + if (num_elems % 2 != 0) + throw0(DB_ERROR("tx_outputs[tx_hash] does not have an even numer of indices")); + int num_outputs = num_elems / 2; - mdb_cursor_get(m_cur_tx_outputs, &k, &v, MDB_FIRST_DUP); - - for (uint64_t i = 0; i < num_elems; ++i) + for (int i = 0; i < num_outputs; ++i) { - mdb_cursor_get(m_cur_tx_outputs, &k, &v, MDB_GET_CURRENT); - index_vec.push_back(*(const uint64_t *)v.mv_data); - mdb_cursor_get(m_cur_tx_outputs, &k, &v, MDB_NEXT_DUP); + // LOG_PRINT_L0("amount output index[" << 2*i << "]" << ": " << paired_indices[2*i] << " global output index: " << paired_indices[2*i+1]); + amount_output_indices.push_back(paired_indices[2*i]); + global_output_indices.push_back(paired_indices[2*i+1]); } + paired_indices = nullptr; TXN_POSTFIX_RDONLY(); - - return index_vec; } std::vector BlockchainLMDB::get_tx_amount_output_indices(const crypto::hash& h) const { LOG_PRINT_L3("BlockchainLMDB::" << __func__); - check_open(); - std::vector index_vec; - std::vector index_vec2; - // get the transaction's global output indices first - index_vec = get_tx_output_indices(h); - // these are next used to obtain the amount output indices + std::vector amount_output_indices, global_output_indices; + // only need amount_output_indices + get_amount_and_global_output_indices(h, amount_output_indices, global_output_indices); - transaction tx = get_tx(h); - - TXN_PREFIX_RDONLY(); - const mdb_txn_cursors *m_cursors = m_write_txn ? &m_wcursors : &m_tinfo->m_ti_rcursors; - RCURSOR(output_amounts); - - uint64_t i = 0; - uint64_t global_index; - BOOST_FOREACH(const auto& vout, tx.vout) - { - uint64_t amount = vout.amount; - - global_index = index_vec[i]; - - MDB_val_copy k(amount); - MDB_val v; - - auto result = mdb_cursor_get(m_cur_output_amounts, &k, &v, MDB_SET); - if (result == MDB_NOTFOUND) - throw1(OUTPUT_DNE("Attempting to get an output index by amount and amount index, but amount not found")); - else if (result) - throw0(DB_ERROR("DB error attempting to get an output")); - - mdb_size_t num_elems = 0; - mdb_cursor_count(m_cur_output_amounts, &num_elems); - - mdb_cursor_get(m_cur_output_amounts, &k, &v, MDB_FIRST_DUP); - - uint64_t amount_output_index = 0; - uint64_t output_index = 0; - bool found_index = false; - for (uint64_t j = 0; j < num_elems; ++j) - { - mdb_cursor_get(m_cur_output_amounts, &k, &v, MDB_GET_CURRENT); - output_index = *(const uint64_t *)v.mv_data; - if (output_index == global_index) - { - amount_output_index = j; - found_index = true; - break; - } - mdb_cursor_get(m_cur_output_amounts, &k, &v, MDB_NEXT_DUP); - } - if (found_index) - { - index_vec2.push_back(amount_output_index); - } - else - { - // not found - TXN_POSTFIX_RDONLY(); - throw1(OUTPUT_DNE("specified output not found in db")); - } - - ++i; - } - - TXN_POSTFIX_RDONLY(); - - return index_vec2; + return amount_output_indices; } diff --git a/src/blockchain_db/lmdb/db_lmdb.h b/src/blockchain_db/lmdb/db_lmdb.h index e58643efa..8f80b8c7e 100644 --- a/src/blockchain_db/lmdb/db_lmdb.h +++ b/src/blockchain_db/lmdb/db_lmdb.h @@ -246,7 +246,10 @@ public: virtual void get_output_tx_and_index(const uint64_t& amount, const std::vector &offsets, std::vector &indices); virtual void get_output_global_indices(const uint64_t& amount, const std::vector &offsets, std::vector &indices); - virtual std::vector get_tx_output_indices(const crypto::hash& h) const; + virtual void get_amount_and_global_output_indices(const crypto::hash& h, + std::vector& amount_output_indices, + std::vector& global_output_indices) const; + virtual std::vector get_tx_amount_output_indices(const crypto::hash& h) const; virtual bool has_key_image(const crypto::key_image& img) const; @@ -298,7 +301,18 @@ private: virtual void remove_transaction_data(const crypto::hash& tx_hash, const transaction& tx); - virtual void add_output(const crypto::hash& tx_hash, const tx_out& tx_output, const uint64_t& local_index, const uint64_t unlock_time); + virtual void add_output(const crypto::hash& tx_hash, + const tx_out& tx_output, + const uint64_t& local_index, + const uint64_t unlock_time, + uint64_t& amount_output_index, + uint64_t& global_output_index + ); + + virtual void add_amount_and_global_output_indices(const crypto::hash& tx_hash, + const std::vector& amount_output_indices, + const std::vector& global_output_indices + ); virtual void remove_output(const tx_out& tx_output); From 8d252a421437c8fe66ff9c230f6b9ebe749f0c09 Mon Sep 17 00:00:00 2001 From: Howard Chu Date: Thu, 3 Mar 2016 04:03:04 +0000 Subject: [PATCH 04/23] Consolidated block info --- src/blockchain_db/lmdb/db_lmdb.cpp | 132 +++++++++++++---------------- src/blockchain_db/lmdb/db_lmdb.h | 24 +----- 2 files changed, 62 insertions(+), 94 deletions(-) diff --git a/src/blockchain_db/lmdb/db_lmdb.cpp b/src/blockchain_db/lmdb/db_lmdb.cpp index 0be2728a9..623b9478a 100644 --- a/src/blockchain_db/lmdb/db_lmdb.cpp +++ b/src/blockchain_db/lmdb/db_lmdb.cpp @@ -152,12 +152,8 @@ int compare_string(const MDB_val *a, const MDB_val *b) } const char* const LMDB_BLOCKS = "blocks"; -const char* const LMDB_BLOCK_TIMESTAMPS = "block_timestamps"; +const char* const LMDB_BLOCK_INFO = "block_info"; const char* const LMDB_BLOCK_HEIGHTS = "block_heights"; -const char* const LMDB_BLOCK_HASHES = "block_hashes"; -const char* const LMDB_BLOCK_SIZES = "block_sizes"; -const char* const LMDB_BLOCK_DIFFS = "block_diffs"; -const char* const LMDB_BLOCK_COINS = "block_coins"; const char* const LMDB_TXS = "txs"; const char* const LMDB_TX_UNLOCKS = "tx_unlocks"; @@ -212,6 +208,17 @@ inline void lmdb_db_open(MDB_txn* txn, const char* name, int flags, MDB_dbi& dbi namespace cryptonote { + +typedef struct mdb_block_info +{ + uint64_t bi_height; + uint64_t bi_timestamp; + uint64_t bi_coins; + uint64_t bi_size; + difficulty_type bi_diff; + crypto::hash bi_hash; +} mdb_block_info; + std::atomic mdb_txn_safe::num_active_txns{0}; std::atomic_flag mdb_txn_safe::creation_gate = ATOMIC_FLAG_INIT; @@ -550,45 +557,31 @@ void BlockchainLMDB::add_block(const block& blk, const size_t& block_size, const MDB_val_copy key(m_height); CURSOR(blocks) - CURSOR(block_sizes) - CURSOR(block_timestamps) - CURSOR(block_diffs) - CURSOR(block_coins) - CURSOR(block_hashes) + CURSOR(block_info) MDB_val_copy blob(block_to_blob(blk)); result = mdb_cursor_put(m_cur_blocks, &key, &blob, MDB_APPEND); if (result) throw0(DB_ERROR(lmdb_error("Failed to add block blob to db transaction: ", result).c_str())); - MDB_val_copy sz(block_size); - result = mdb_cursor_put(m_cur_block_sizes, &key, &sz, MDB_APPEND); - if (result) - throw0(DB_ERROR(lmdb_error("Failed to add block size to db transaction: ", result).c_str())); + mdb_block_info bi; + bi.bi_timestamp = blk.timestamp; + bi.bi_coins = coins_generated; + bi.bi_size = block_size; + bi.bi_diff = cumulative_difficulty; + bi.bi_hash = blk_hash; - MDB_val_copy ts(blk.timestamp); - result = mdb_cursor_put(m_cur_block_timestamps, &key, &ts, MDB_APPEND); + MDB_val val; + val.mv_data = (void *)&bi; + val.mv_size = sizeof(bi); + result = mdb_cursor_put(m_cur_block_info, &key, &val, MDB_APPEND); if (result) - throw0(DB_ERROR(lmdb_error("Failed to add block timestamp to db transaction: ", result).c_str())); - - MDB_val_copy diff(cumulative_difficulty); - result = mdb_cursor_put(m_cur_block_diffs, &key, &diff, MDB_APPEND); - if (result) - throw0(DB_ERROR(lmdb_error("Failed to add block cumulative difficulty to db transaction: ", result).c_str())); - - MDB_val_copy coinsgen(coins_generated); - result = mdb_cursor_put(m_cur_block_coins, &key, &coinsgen, MDB_APPEND); - if (result) - throw0(DB_ERROR(lmdb_error("Failed to add block total generated coins to db transaction: ", result).c_str())); + throw0(DB_ERROR(lmdb_error("Failed to add block info to db transaction: ", result).c_str())); result = mdb_cursor_put(m_cur_block_heights, &val_h, &key, 0); if (result) throw0(DB_ERROR(lmdb_error("Failed to add block height by hash to db transaction: ", result).c_str())); - result = mdb_cursor_put(m_cur_block_hashes, &key, &val_h, MDB_APPEND); - if (result) - throw0(DB_ERROR(lmdb_error("Failed to add block hash to db transaction: ", result).c_str())); - m_cum_size += block_size; m_cum_count++; } @@ -601,31 +594,25 @@ void BlockchainLMDB::remove_block() if (m_height == 0) throw0(BLOCK_DNE ("Attempting to remove block from an empty blockchain")); + mdb_txn_cursors *m_cursors = &m_wcursors; + CURSOR(block_info) MDB_val_copy k(m_height - 1); MDB_val h; - if (mdb_get(*m_write_txn, m_block_hashes, &k, &h)) + if (mdb_cursor_get(m_cur_block_info, &k, &h, MDB_SET)) throw1(BLOCK_DNE("Attempting to remove block that's not in the db")); + // must use h now; deleting from m_block_info will invalidate it + mdb_block_info *bi = (mdb_block_info *)h.mv_data; + h.mv_data = &bi->bi_hash; + h.mv_size = sizeof(bi->bi_hash); + if (mdb_del(*m_write_txn, m_block_heights, &h, NULL)) + throw1(DB_ERROR("Failed to add removal of block height by hash to db transaction")); + if (mdb_del(*m_write_txn, m_blocks, &k, NULL)) throw1(DB_ERROR("Failed to add removal of block to db transaction")); - if (mdb_del(*m_write_txn, m_block_sizes, &k, NULL)) - throw1(DB_ERROR("Failed to add removal of block size to db transaction")); - - if (mdb_del(*m_write_txn, m_block_diffs, &k, NULL)) - throw1(DB_ERROR("Failed to add removal of block cumulative difficulty to db transaction")); - - if (mdb_del(*m_write_txn, m_block_coins, &k, NULL)) - throw1(DB_ERROR("Failed to add removal of block total generated coins to db transaction")); - - if (mdb_del(*m_write_txn, m_block_timestamps, &k, NULL)) - throw1(DB_ERROR("Failed to add removal of block timestamp to db transaction")); - - if (mdb_del(*m_write_txn, m_block_heights, &h, NULL)) - throw1(DB_ERROR("Failed to add removal of block height by hash to db transaction")); - - if (mdb_del(*m_write_txn, m_block_hashes, &k, NULL)) - throw1(DB_ERROR("Failed to add removal of block hash to db transaction")); + if (mdb_cursor_del(m_cur_block_info, 0)) + throw1(DB_ERROR("Failed to add removal of block info to db transaction")); } void BlockchainLMDB::add_transaction_data(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash& tx_hash) @@ -1082,12 +1069,8 @@ void BlockchainLMDB::open(const std::string& filename, const int mdb_flags) // uses macros to avoid having to change things too many places lmdb_db_open(txn, LMDB_BLOCKS, MDB_INTEGERKEY | MDB_CREATE, m_blocks, "Failed to open db handle for m_blocks"); - lmdb_db_open(txn, LMDB_BLOCK_TIMESTAMPS, MDB_INTEGERKEY | MDB_CREATE, m_block_timestamps, "Failed to open db handle for m_block_timestamps"); + lmdb_db_open(txn, LMDB_BLOCK_INFO, MDB_INTEGERKEY | MDB_CREATE, m_block_info, "Failed to open db handle for m_block_info"); lmdb_db_open(txn, LMDB_BLOCK_HEIGHTS, MDB_CREATE, m_block_heights, "Failed to open db handle for m_block_heights"); - lmdb_db_open(txn, LMDB_BLOCK_HASHES, MDB_INTEGERKEY | MDB_CREATE, m_block_hashes, "Failed to open db handle for m_block_hashes"); - lmdb_db_open(txn, LMDB_BLOCK_SIZES, MDB_INTEGERKEY | MDB_CREATE, m_block_sizes, "Failed to open db handle for m_block_sizes"); - lmdb_db_open(txn, LMDB_BLOCK_DIFFS, MDB_INTEGERKEY | MDB_CREATE, m_block_diffs, "Failed to open db handle for m_block_diffs"); - lmdb_db_open(txn, LMDB_BLOCK_COINS, MDB_INTEGERKEY | MDB_CREATE, m_block_coins, "Failed to open db handle for m_block_coins"); lmdb_db_open(txn, LMDB_TXS, MDB_CREATE, m_txs, "Failed to open db handle for m_txs"); lmdb_db_open(txn, LMDB_TX_UNLOCKS, MDB_CREATE, m_tx_unlocks, "Failed to open db handle for m_tx_unlocks"); @@ -1249,12 +1232,8 @@ void BlockchainLMDB::reset() if (mdb_txn_begin(m_env, NULL, 0, txn)) throw0(DB_ERROR("Failed to create a transaction for the db")); mdb_drop(txn, m_blocks, 0); - mdb_drop(txn, m_block_timestamps, 0); + mdb_drop(txn, m_block_info, 0); mdb_drop(txn, m_block_heights, 0); - mdb_drop(txn, m_block_hashes, 0); - mdb_drop(txn, m_block_sizes, 0); - mdb_drop(txn, m_block_diffs, 0); - mdb_drop(txn, m_block_coins, 0); mdb_drop(txn, m_txs, 0); mdb_drop(txn, m_tx_unlocks, 0); mdb_drop(txn, m_tx_heights, 0); @@ -1462,11 +1441,11 @@ uint64_t BlockchainLMDB::get_block_timestamp(const uint64_t& height) const TXN_PREFIX_RDONLY(); const mdb_txn_cursors *m_cursors = m_write_txn ? &m_wcursors : &m_tinfo->m_ti_rcursors; - RCURSOR(block_timestamps); + RCURSOR(block_info); MDB_val_copy key(height); MDB_val result; - auto get_result = mdb_cursor_get(m_cur_block_timestamps, &key, &result, MDB_SET); + auto get_result = mdb_cursor_get(m_cur_block_info, &key, &result, MDB_SET); if (get_result == MDB_NOTFOUND) { throw0(BLOCK_DNE(std::string("Attempt to get timestamp from height ").append(boost::lexical_cast(height)).append(" failed -- timestamp not in db").c_str())); @@ -1474,7 +1453,8 @@ uint64_t BlockchainLMDB::get_block_timestamp(const uint64_t& height) const else if (get_result) throw0(DB_ERROR("Error attempting to retrieve a timestamp from the db")); - uint64_t ret = *(const uint64_t *)result.mv_data; + mdb_block_info *bi = (mdb_block_info *)result.mv_data; + uint64_t ret = bi->bi_timestamp; TXN_POSTFIX_RDONLY(); return ret; } @@ -1500,11 +1480,11 @@ size_t BlockchainLMDB::get_block_size(const uint64_t& height) const TXN_PREFIX_RDONLY(); const mdb_txn_cursors *m_cursors = m_write_txn ? &m_wcursors : &m_tinfo->m_ti_rcursors; - RCURSOR(block_sizes); + RCURSOR(block_info); MDB_val_copy key(height); MDB_val result; - auto get_result = mdb_cursor_get(m_cur_block_sizes, &key, &result, MDB_SET); + auto get_result = mdb_cursor_get(m_cur_block_info, &key, &result, MDB_SET); if (get_result == MDB_NOTFOUND) { throw0(BLOCK_DNE(std::string("Attempt to get block size from height ").append(boost::lexical_cast(height)).append(" failed -- block size not in db").c_str())); @@ -1512,7 +1492,8 @@ size_t BlockchainLMDB::get_block_size(const uint64_t& height) const else if (get_result) throw0(DB_ERROR("Error attempting to retrieve a block size from the db")); - size_t ret = *(const size_t *)result.mv_data; + mdb_block_info *bi = (mdb_block_info *)result.mv_data; + size_t ret = bi->bi_size; TXN_POSTFIX_RDONLY(); return ret; } @@ -1524,11 +1505,11 @@ difficulty_type BlockchainLMDB::get_block_cumulative_difficulty(const uint64_t& TXN_PREFIX_RDONLY(); const mdb_txn_cursors *m_cursors = m_write_txn ? &m_wcursors : &m_tinfo->m_ti_rcursors; - RCURSOR(block_diffs); + RCURSOR(block_info); MDB_val_copy key(height); MDB_val result; - auto get_result = mdb_cursor_get(m_cur_block_diffs, &key, &result, MDB_SET); + auto get_result = mdb_cursor_get(m_cur_block_info, &key, &result, MDB_SET); if (get_result == MDB_NOTFOUND) { throw0(BLOCK_DNE(std::string("Attempt to get cumulative difficulty from height ").append(boost::lexical_cast(height)).append(" failed -- difficulty not in db").c_str())); @@ -1536,7 +1517,8 @@ difficulty_type BlockchainLMDB::get_block_cumulative_difficulty(const uint64_t& else if (get_result) throw0(DB_ERROR("Error attempting to retrieve a cumulative difficulty from the db")); - difficulty_type ret = *(const difficulty_type*)result.mv_data; + mdb_block_info *bi = (mdb_block_info *)result.mv_data; + difficulty_type ret = bi->bi_diff; TXN_POSTFIX_RDONLY(); return ret; } @@ -1565,11 +1547,11 @@ uint64_t BlockchainLMDB::get_block_already_generated_coins(const uint64_t& heigh TXN_PREFIX_RDONLY(); const mdb_txn_cursors *m_cursors = m_write_txn ? &m_wcursors : &m_tinfo->m_ti_rcursors; - RCURSOR(block_coins); + RCURSOR(block_info); MDB_val_copy key(height); MDB_val result; - auto get_result = mdb_cursor_get(m_cur_block_coins, &key, &result, MDB_SET); + auto get_result = mdb_cursor_get(m_cur_block_info, &key, &result, MDB_SET); if (get_result == MDB_NOTFOUND) { throw0(BLOCK_DNE(std::string("Attempt to get generated coins from height ").append(boost::lexical_cast(height)).append(" failed -- block size not in db").c_str())); @@ -1577,7 +1559,8 @@ uint64_t BlockchainLMDB::get_block_already_generated_coins(const uint64_t& heigh else if (get_result) throw0(DB_ERROR("Error attempting to retrieve a total generated coins from the db")); - uint64_t ret = *(const uint64_t*)result.mv_data; + mdb_block_info *bi = (mdb_block_info *)result.mv_data; + uint64_t ret = bi->bi_coins; TXN_POSTFIX_RDONLY(); return ret; } @@ -1589,11 +1572,11 @@ crypto::hash BlockchainLMDB::get_block_hash_from_height(const uint64_t& height) TXN_PREFIX_RDONLY(); const mdb_txn_cursors *m_cursors = m_write_txn ? &m_wcursors : &m_tinfo->m_ti_rcursors; - RCURSOR(block_hashes); + RCURSOR(block_info); MDB_val_copy key(height); MDB_val result; - auto get_result = mdb_cursor_get(m_cur_block_hashes, &key, &result, MDB_SET); + auto get_result = mdb_cursor_get(m_cur_block_info, &key, &result, MDB_SET); if (get_result == MDB_NOTFOUND) { throw0(BLOCK_DNE(std::string("Attempt to get hash from height ").append(boost::lexical_cast(height)).append(" failed -- hash not in db").c_str())); @@ -1601,7 +1584,8 @@ crypto::hash BlockchainLMDB::get_block_hash_from_height(const uint64_t& height) else if (get_result) throw0(DB_ERROR(lmdb_error("Error attempting to retrieve a block hash from the db: ", get_result).c_str())); - crypto::hash ret = *(const crypto::hash*)result.mv_data; + mdb_block_info *bi = (mdb_block_info *)result.mv_data; + crypto::hash ret = bi->bi_hash; TXN_POSTFIX_RDONLY(); return ret; } diff --git a/src/blockchain_db/lmdb/db_lmdb.h b/src/blockchain_db/lmdb/db_lmdb.h index 8f80b8c7e..1181b5b46 100644 --- a/src/blockchain_db/lmdb/db_lmdb.h +++ b/src/blockchain_db/lmdb/db_lmdb.h @@ -43,11 +43,7 @@ typedef struct mdb_txn_cursors { MDB_cursor *m_txc_blocks; MDB_cursor *m_txc_block_heights; - MDB_cursor *m_txc_block_hashes; - MDB_cursor *m_txc_block_timestamps; - MDB_cursor *m_txc_block_sizes; - MDB_cursor *m_txc_block_diffs; - MDB_cursor *m_txc_block_coins; + MDB_cursor *m_txc_block_info; MDB_cursor *m_txc_output_txs; MDB_cursor *m_txc_output_indices; @@ -66,11 +62,7 @@ typedef struct mdb_txn_cursors #define m_cur_blocks m_cursors->m_txc_blocks #define m_cur_block_heights m_cursors->m_txc_block_heights -#define m_cur_block_hashes m_cursors->m_txc_block_hashes -#define m_cur_block_timestamps m_cursors->m_txc_block_timestamps -#define m_cur_block_sizes m_cursors->m_txc_block_sizes -#define m_cur_block_diffs m_cursors->m_txc_block_diffs -#define m_cur_block_coins m_cursors->m_txc_block_coins +#define m_cur_block_info m_cursors->m_txc_block_info #define m_cur_output_txs m_cursors->m_txc_output_txs #define m_cur_output_indices m_cursors->m_txc_output_indices #define m_cur_output_amounts m_cursors->m_txc_output_amounts @@ -87,11 +79,7 @@ typedef struct mdb_rflags bool m_rf_txn; bool m_rf_blocks; bool m_rf_block_heights; - bool m_rf_block_hashes; - bool m_rf_block_timestamps; - bool m_rf_block_sizes; - bool m_rf_block_diffs; - bool m_rf_block_coins; + bool m_rf_block_info; bool m_rf_output_txs; bool m_rf_output_indices; bool m_rf_output_amounts; @@ -372,11 +360,7 @@ private: MDB_dbi m_blocks; MDB_dbi m_block_heights; - MDB_dbi m_block_hashes; - MDB_dbi m_block_timestamps; - MDB_dbi m_block_sizes; - MDB_dbi m_block_diffs; - MDB_dbi m_block_coins; + MDB_dbi m_block_info; MDB_dbi m_txs; MDB_dbi m_tx_unlocks; From ae0854a43163d8480aeb030e2de4bde89b12b7fa Mon Sep 17 00:00:00 2001 From: warptangent Date: Fri, 4 Mar 2016 09:38:15 -0800 Subject: [PATCH 05/23] Schema update: tx_indices --- src/blockchain_db/lmdb/db_lmdb.cpp | 136 ++++++++++++++++++++++------- src/blockchain_db/lmdb/db_lmdb.h | 5 ++ 2 files changed, 109 insertions(+), 32 deletions(-) diff --git a/src/blockchain_db/lmdb/db_lmdb.cpp b/src/blockchain_db/lmdb/db_lmdb.cpp index 623b9478a..d26659ead 100644 --- a/src/blockchain_db/lmdb/db_lmdb.cpp +++ b/src/blockchain_db/lmdb/db_lmdb.cpp @@ -156,6 +156,7 @@ const char* const LMDB_BLOCK_INFO = "block_info"; const char* const LMDB_BLOCK_HEIGHTS = "block_heights"; const char* const LMDB_TXS = "txs"; +const char* const LMDB_TX_INDICES = "tx_indices"; const char* const LMDB_TX_UNLOCKS = "tx_unlocks"; const char* const LMDB_TX_HEIGHTS = "tx_heights"; const char* const LMDB_TX_OUTPUTS = "tx_outputs"; @@ -624,54 +625,76 @@ void BlockchainLMDB::add_transaction_data(const crypto::hash& blk_hash, const tr int result = 0; CURSOR(txs) + CURSOR(tx_indices) CURSOR(tx_heights) CURSOR(tx_unlocks) + MDB_val_copy val_tx_index(m_num_txs); MDB_val_copy val_h(tx_hash); MDB_val unused; - if (mdb_cursor_get(m_cur_txs, &val_h, &unused, MDB_SET) == 0) - throw1(TX_EXISTS("Attempting to add transaction that's already in the db")); + result = mdb_cursor_get(m_cur_tx_indices, &val_h, &unused, MDB_SET); + if (result == 0) + throw1(TX_EXISTS(std::string("Attempting to add transaction that's already in the db (tx index ").append(boost::lexical_cast(*(const uint64_t*)unused.mv_data)).append(")").c_str())); + else if (result != MDB_NOTFOUND) + throw1(DB_ERROR(lmdb_error(std::string("Error checking if tx index exists for tx hash ") + epee::string_tools::pod_to_hex(tx_hash) + ": ", result).c_str())); + + result = mdb_cursor_put(m_cur_tx_indices, &val_h, &val_tx_index, 0); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to add tx index to db transaction: ", result).c_str())); MDB_val_copy blob(tx_to_blob(tx)); - result = mdb_cursor_put(m_cur_txs, &val_h, &blob, 0); + result = mdb_cursor_put(m_cur_txs, &val_tx_index, &blob, 0); if (result) throw0(DB_ERROR(lmdb_error("Failed to add tx blob to db transaction: ", result).c_str())); MDB_val_copy height(m_height); - result = mdb_cursor_put(m_cur_tx_heights, &val_h, &height, 0); + result = mdb_cursor_put(m_cur_tx_heights, &val_tx_index, &height, 0); if (result) throw0(DB_ERROR(lmdb_error("Failed to add tx block height to db transaction: ", result).c_str())); MDB_val_copy unlock_time(tx.unlock_time); - result = mdb_cursor_put(m_cur_tx_unlocks, &val_h, &unlock_time, 0); + result = mdb_cursor_put(m_cur_tx_unlocks, &val_tx_index, &unlock_time, 0); if (result) throw0(DB_ERROR(lmdb_error("Failed to add tx unlock time to db transaction: ", result).c_str())); + + m_num_txs++; } +// TODO: compare pros and cons of looking up the tx hash's tx index once and +// passing it in to functions like this void BlockchainLMDB::remove_transaction_data(const crypto::hash& tx_hash, const transaction& tx) { LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); MDB_val_copy val_h(tx_hash); - MDB_val unused; - if (mdb_get(*m_write_txn, m_txs, &val_h, &unused)) + MDB_val val_tx_index; + + if (mdb_get(*m_write_txn, m_tx_indices, &val_h, &val_tx_index)) throw1(TX_DNE("Attempting to remove transaction that isn't in the db")); - if (mdb_del(*m_write_txn, m_txs, &val_h, NULL)) + if (mdb_del(*m_write_txn, m_txs, &val_tx_index, NULL)) throw1(DB_ERROR("Failed to add removal of tx to db transaction")); - if (mdb_del(*m_write_txn, m_tx_unlocks, &val_h, NULL)) + if (mdb_del(*m_write_txn, m_tx_unlocks, &val_tx_index, NULL)) throw1(DB_ERROR("Failed to add removal of tx unlock time to db transaction")); - if (mdb_del(*m_write_txn, m_tx_heights, &val_h, NULL)) + if (mdb_del(*m_write_txn, m_tx_heights, &val_tx_index, NULL)) throw1(DB_ERROR("Failed to add removal of tx block height to db transaction")); remove_tx_outputs(tx_hash, tx); - auto result = mdb_del(*m_write_txn, m_tx_outputs, &val_h, NULL); + int result = mdb_del(*m_write_txn, m_tx_outputs, &val_tx_index, NULL); if (result == MDB_NOTFOUND) LOG_PRINT_L1("tx has no outputs to remove: " << tx_hash); else if (result) throw1(DB_ERROR(lmdb_error("Failed to add removal of tx outputs to db transaction: ", result).c_str())); + + // Though other things could change, so long as earlier functions (like + // remove_tx_outputs) need to do the lookup of tx hash -> tx index, don't + // delete the tx_indices entry until the end. + if (mdb_del(*m_write_txn, m_tx_indices, &val_h, NULL)) + throw1(DB_ERROR("Failed to add removal of tx index to db transaction")); + + m_num_txs--; } void BlockchainLMDB::add_output(const crypto::hash& tx_hash, @@ -763,7 +786,12 @@ void BlockchainLMDB::add_amount_and_global_output_indices(const crypto::hash& tx v.mv_size = sizeof(uint64_t) * 2 * num_outputs; // LOG_PRINT_L1("tx_outputs[tx_hash] size: " << v.mv_size); - result = mdb_cursor_put(m_cur_tx_outputs, &k, &v, 0); + MDB_val val_tx_index; + result = mdb_cursor_get(m_cur_tx_indices, &k, &val_tx_index, MDB_SET); + if (result) + throw0(DB_ERROR(lmdb_error(std::string("Failed to get tx index for tx hash ") + epee::string_tools::pod_to_hex(tx_hash), result).c_str())); + + result = mdb_cursor_put(m_cur_tx_outputs, &val_tx_index, &v, 0); if (result) throw0(DB_ERROR(std::string("Failed to add to db transaction: ").append(mdb_strerror(result)).c_str())); } @@ -1072,10 +1100,11 @@ void BlockchainLMDB::open(const std::string& filename, const int mdb_flags) lmdb_db_open(txn, LMDB_BLOCK_INFO, MDB_INTEGERKEY | MDB_CREATE, m_block_info, "Failed to open db handle for m_block_info"); lmdb_db_open(txn, LMDB_BLOCK_HEIGHTS, MDB_CREATE, m_block_heights, "Failed to open db handle for m_block_heights"); - lmdb_db_open(txn, LMDB_TXS, MDB_CREATE, m_txs, "Failed to open db handle for m_txs"); - lmdb_db_open(txn, LMDB_TX_UNLOCKS, MDB_CREATE, m_tx_unlocks, "Failed to open db handle for m_tx_unlocks"); - lmdb_db_open(txn, LMDB_TX_HEIGHTS, MDB_CREATE, m_tx_heights, "Failed to open db handle for m_tx_heights"); - lmdb_db_open(txn, LMDB_TX_OUTPUTS, MDB_CREATE, m_tx_outputs, "Failed to open db handle for m_tx_outputs"); + lmdb_db_open(txn, LMDB_TXS, MDB_INTEGERKEY | MDB_CREATE, m_txs, "Failed to open db handle for m_txs"); + lmdb_db_open(txn, LMDB_TX_INDICES, MDB_CREATE, m_tx_indices, "Failed to open db handle for m_tx_indices"); + lmdb_db_open(txn, LMDB_TX_UNLOCKS, MDB_INTEGERKEY | MDB_CREATE, m_tx_unlocks, "Failed to open db handle for m_tx_unlocks"); + lmdb_db_open(txn, LMDB_TX_HEIGHTS, MDB_INTEGERKEY | MDB_CREATE, m_tx_heights, "Failed to open db handle for m_tx_heights"); + lmdb_db_open(txn, LMDB_TX_OUTPUTS, MDB_INTEGERKEY | MDB_CREATE, m_tx_outputs, "Failed to open db handle for m_tx_outputs"); lmdb_db_open(txn, LMDB_OUTPUT_TXS, MDB_INTEGERKEY | MDB_CREATE, m_output_txs, "Failed to open db handle for m_output_txs"); lmdb_db_open(txn, LMDB_OUTPUT_INDICES, MDB_INTEGERKEY | MDB_CREATE, m_output_indices, "Failed to open db handle for m_output_indices"); @@ -1092,9 +1121,7 @@ void BlockchainLMDB::open(const std::string& filename, const int mdb_flags) mdb_set_dupsort(txn, m_output_amounts, compare_uint64); mdb_set_compare(txn, m_spent_keys, compare_hash32); mdb_set_compare(txn, m_block_heights, compare_hash32); - mdb_set_compare(txn, m_txs, compare_hash32); - mdb_set_compare(txn, m_tx_unlocks, compare_hash32); - mdb_set_compare(txn, m_tx_heights, compare_hash32); + mdb_set_compare(txn, m_tx_indices, compare_hash32); mdb_set_compare(txn, m_hf_starting_heights, compare_uint8); mdb_set_compare(txn, m_hf_versions, compare_uint64); mdb_set_compare(txn, m_properties, compare_string); @@ -1106,6 +1133,11 @@ void BlockchainLMDB::open(const std::string& filename, const int mdb_flags) LOG_PRINT_L2("Setting m_height to: " << db_stats.ms_entries); m_height = db_stats.ms_entries; + // get and keep current number of txs + if (mdb_stat(txn, m_tx_indices, &db_stats)) + throw0(DB_ERROR("Failed to query m_tx_indices")); + m_num_txs = db_stats.ms_entries; + // get and keep current number of outputs if (mdb_stat(txn, m_output_indices, &db_stats)) throw0(DB_ERROR("Failed to query m_output_indices")); @@ -1659,25 +1691,37 @@ bool BlockchainLMDB::tx_exists(const crypto::hash& h) const TXN_PREFIX_RDONLY(); const mdb_txn_cursors *m_cursors = m_write_txn ? &m_wcursors : &m_tinfo->m_ti_rcursors; + RCURSOR(tx_indices); RCURSOR(txs); MDB_val_copy key(h); + MDB_val val_tx_index; MDB_val result; + bool tx_found = false; TIME_MEASURE_START(time1); - auto get_result = mdb_cursor_get(m_cur_txs, &key, &result, MDB_SET); + auto get_result = mdb_cursor_get(m_cur_tx_indices, &key, &val_tx_index, MDB_SET); + if (get_result == 0) + tx_found = true; + else if (get_result != MDB_NOTFOUND) + throw0(DB_ERROR(lmdb_error(std::string("DB error attempting to fetch transaction index from hash ") + epee::string_tools::pod_to_hex(h) + ": ", get_result).c_str())); + + get_result = mdb_cursor_get(m_cur_txs, &val_tx_index, &result, MDB_SET); TIME_MEASURE_FINISH(time1); time_tx_exists += time1; TXN_POSTFIX_RDONLY(); - if (get_result == MDB_NOTFOUND) + if (! tx_found) { LOG_PRINT_L1("transaction with hash " << epee::string_tools::pod_to_hex(h) << " not found in db"); return false; } + + if (get_result == MDB_NOTFOUND) + throw0(DB_ERROR(std::string("transaction with hash ").append(epee::string_tools::pod_to_hex(h)).append(" not found at index").c_str())); else if (get_result) - throw0(DB_ERROR("DB error attempting to fetch transaction from hash")); + throw0(DB_ERROR(lmdb_error(std::string("DB error attempting to fetch transaction ") + epee::string_tools::pod_to_hex(h) + " at index: ", get_result).c_str())); return true; } @@ -1689,15 +1733,23 @@ uint64_t BlockchainLMDB::get_tx_unlock_time(const crypto::hash& h) const TXN_PREFIX_RDONLY(); const mdb_txn_cursors *m_cursors = m_write_txn ? &m_wcursors : &m_tinfo->m_ti_rcursors; + RCURSOR(tx_indices); RCURSOR(tx_unlocks); MDB_val_copy key(h); + MDB_val val_tx_index; MDB_val result; - auto get_result = mdb_cursor_get(m_cur_tx_unlocks, &key, &result, MDB_SET); + auto get_result = mdb_cursor_get(m_cur_tx_indices, &key, &val_tx_index, MDB_SET); if (get_result == MDB_NOTFOUND) - throw1(TX_DNE(std::string("tx unlock time with hash ").append(epee::string_tools::pod_to_hex(h)).append(" not found in db").c_str())); + throw1(TX_DNE(lmdb_error(std::string("tx index with hash ") + epee::string_tools::pod_to_hex(h) + " not found in db: ", get_result).c_str())); else if (get_result) - throw0(DB_ERROR("DB error attempting to fetch tx unlock time from hash")); + throw0(DB_ERROR(lmdb_error("DB error attempting to fetch tx index from hash: ", get_result).c_str())); + + get_result = mdb_cursor_get(m_cur_tx_unlocks, &val_tx_index, &result, MDB_SET); + if (get_result == MDB_NOTFOUND) + throw1(TX_DNE(lmdb_error(std::string("tx unlock time with hash ") + epee::string_tools::pod_to_hex(h) + " not found in db: ", get_result).c_str())); + else if (get_result) + throw0(DB_ERROR(lmdb_error("DB error attempting to fetch tx unlock time from index: ", get_result).c_str())); uint64_t ret = *(const uint64_t*)result.mv_data; TXN_POSTFIX_RDONLY(); @@ -1711,11 +1763,15 @@ transaction BlockchainLMDB::get_tx(const crypto::hash& h) const TXN_PREFIX_RDONLY(); const mdb_txn_cursors *m_cursors = m_write_txn ? &m_wcursors : &m_tinfo->m_ti_rcursors; + RCURSOR(tx_indices); RCURSOR(txs); MDB_val_copy key(h); + MDB_val val_tx_index; MDB_val result; - auto get_result = mdb_cursor_get(m_cur_txs, &key, &result, MDB_SET); + auto get_result = mdb_cursor_get(m_cur_tx_indices, &key, &val_tx_index, MDB_SET); + if (get_result == 0) + get_result = mdb_cursor_get(m_cur_txs, &val_tx_index, &result, MDB_SET); if (get_result == MDB_NOTFOUND) throw1(TX_DNE(std::string("tx with hash ").append(epee::string_tools::pod_to_hex(h)).append(" not found in db").c_str())); else if (get_result) @@ -1741,8 +1797,8 @@ uint64_t BlockchainLMDB::get_tx_count() const TXN_PREFIX_RDONLY(); MDB_stat db_stats; - if (mdb_stat(m_txn, m_txs, &db_stats)) - throw0(DB_ERROR("Failed to query m_txs")); + if (mdb_stat(m_txn, m_tx_indices, &db_stats)) + throw0(DB_ERROR("Failed to query m_tx_indices")); TXN_POSTFIX_RDONLY(); @@ -1770,11 +1826,15 @@ uint64_t BlockchainLMDB::get_tx_block_height(const crypto::hash& h) const TXN_PREFIX_RDONLY(); const mdb_txn_cursors *m_cursors = m_write_txn ? &m_wcursors : &m_tinfo->m_ti_rcursors; + RCURSOR(tx_indices); RCURSOR(tx_heights); MDB_val_copy key(h); + MDB_val val_tx_index; MDB_val result; - auto get_result = mdb_cursor_get(m_cur_tx_heights, &key, &result, MDB_SET); + auto get_result = mdb_cursor_get(m_cur_tx_indices, &key, &val_tx_index, MDB_SET); + if (get_result == 0) + get_result = mdb_cursor_get(m_cur_tx_heights, &val_tx_index, &result, MDB_SET); if (get_result == MDB_NOTFOUND) { throw1(TX_DNE(std::string("tx height with hash ").append(epee::string_tools::pod_to_hex(h)).append(" not found in db").c_str())); @@ -1906,24 +1966,32 @@ void BlockchainLMDB::get_amount_and_global_output_indices(const crypto::hash& h, // create a new read-only txn here, which is incorrect. TXN_PREFIX_RDONLY(); const mdb_txn_cursors *m_cursors = m_write_txn ? &m_wcursors : &m_tinfo->m_ti_rcursors; + RCURSOR(tx_indices); RCURSOR(tx_outputs); int result = 0; MDB_val_copy k(h); + MDB_val val_tx_index; MDB_val v; - result = mdb_cursor_get(m_cur_tx_outputs, &k, &v, MDB_SET); + result = mdb_cursor_get(m_cur_tx_indices, &k, &val_tx_index, MDB_SET); + if (result == MDB_NOTFOUND) + throw1(OUTPUT_DNE(std::string("Failed to get tx index for tx hash ").append(epee::string_tools::pod_to_hex(h)).c_str())); + else if (result) + throw0(DB_ERROR(lmdb_error(std::string("DB error attempting to fetch transaction index from hash ") + epee::string_tools::pod_to_hex(h) + ": ", result).c_str())); + + result = mdb_cursor_get(m_cur_tx_outputs, &val_tx_index, &v, MDB_SET); if (result == MDB_NOTFOUND) LOG_PRINT_L0("WARNING: Unexpected: tx has no amount and global indices stored in " "tx_outputs, but it should have an empty entry even if it's a tx without " "outputs"); else if (result) - throw0(DB_ERROR("DB error attempting to get data for tx_outputs[tx_hash]")); + throw0(DB_ERROR("DB error attempting to get data for tx_outputs[tx_index]")); uint64_t* paired_indices = (uint64_t*)v.mv_data; int num_elems = v.mv_size / sizeof(uint64_t); if (num_elems % 2 != 0) - throw0(DB_ERROR("tx_outputs[tx_hash] does not have an even numer of indices")); + throw0(DB_ERROR("tx_outputs[tx_index] does not have an even numer of indices")); int num_outputs = num_elems / 2; for (int i = 0; i < num_outputs; ++i) @@ -2380,6 +2448,7 @@ uint64_t BlockchainLMDB::add_block(const block& blk, const size_t& block_size, c } } + uint64_t num_txs = m_num_txs; uint64_t num_outputs = m_num_outputs; try { @@ -2391,6 +2460,7 @@ uint64_t BlockchainLMDB::add_block(const block& blk, const size_t& block_size, c } catch (...) { + m_num_txs = num_txs; m_num_outputs = num_outputs; block_txn_abort(); throw; @@ -2406,6 +2476,7 @@ void BlockchainLMDB::pop_block(block& blk, std::vector& txs) block_txn_start(false); + uint64_t num_txs = m_num_txs; uint64_t num_outputs = m_num_outputs; try { @@ -2414,6 +2485,7 @@ void BlockchainLMDB::pop_block(block& blk, std::vector& txs) } catch (...) { + m_num_txs = num_txs; m_num_outputs = num_outputs; block_txn_abort(); throw; diff --git a/src/blockchain_db/lmdb/db_lmdb.h b/src/blockchain_db/lmdb/db_lmdb.h index 1181b5b46..41dccfede 100644 --- a/src/blockchain_db/lmdb/db_lmdb.h +++ b/src/blockchain_db/lmdb/db_lmdb.h @@ -51,6 +51,7 @@ typedef struct mdb_txn_cursors MDB_cursor *m_txc_output_keys; MDB_cursor *m_txc_txs; + MDB_cursor *m_txc_tx_indices; MDB_cursor *m_txc_tx_heights; MDB_cursor *m_txc_tx_unlocks; MDB_cursor *m_txc_tx_outputs; @@ -68,6 +69,7 @@ typedef struct mdb_txn_cursors #define m_cur_output_amounts m_cursors->m_txc_output_amounts #define m_cur_output_keys m_cursors->m_txc_output_keys #define m_cur_txs m_cursors->m_txc_txs +#define m_cur_tx_indices m_cursors->m_txc_tx_indices #define m_cur_tx_heights m_cursors->m_txc_tx_heights #define m_cur_tx_unlocks m_cursors->m_txc_tx_unlocks #define m_cur_tx_outputs m_cursors->m_txc_tx_outputs @@ -85,6 +87,7 @@ typedef struct mdb_rflags bool m_rf_output_amounts; bool m_rf_output_keys; bool m_rf_txs; + bool m_rf_tx_indices; bool m_rf_tx_heights; bool m_rf_tx_unlocks; bool m_rf_tx_outputs; @@ -363,6 +366,7 @@ private: MDB_dbi m_block_info; MDB_dbi m_txs; + MDB_dbi m_tx_indices; MDB_dbi m_tx_unlocks; MDB_dbi m_tx_heights; MDB_dbi m_tx_outputs; @@ -380,6 +384,7 @@ private: MDB_dbi m_properties; uint64_t m_height; + uint64_t m_num_txs; uint64_t m_num_outputs; mutable uint64_t m_cum_size; // used in batch size estimation mutable int m_cum_count; From 7c013f66e95199ec15f413aa8acf66f61a60ffe2 Mon Sep 17 00:00:00 2001 From: warptangent Date: Fri, 4 Mar 2016 09:40:22 -0800 Subject: [PATCH 06/23] Add batch warning for further review --- src/blockchain_db/lmdb/db_lmdb.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/blockchain_db/lmdb/db_lmdb.cpp b/src/blockchain_db/lmdb/db_lmdb.cpp index d26659ead..4c377544c 100644 --- a/src/blockchain_db/lmdb/db_lmdb.cpp +++ b/src/blockchain_db/lmdb/db_lmdb.cpp @@ -2293,6 +2293,10 @@ void BlockchainLMDB::batch_abort() void BlockchainLMDB::set_batch_transactions(bool batch_transactions) { LOG_PRINT_L3("BlockchainLMDB::" << __func__); + if ((batch_transactions) && (m_batch_transactions)) + { + LOG_PRINT_L0("WARNING: batch transaction mode already enabled, but asked to enable batch mode"); + } m_batch_transactions = batch_transactions; LOG_PRINT_L3("batch transactions " << (m_batch_transactions ? "enabled" : "disabled")); } From 8d12a8df2cf27713509c039c91fb9180e010011a Mon Sep 17 00:00:00 2001 From: warptangent Date: Fri, 4 Mar 2016 10:59:20 -0800 Subject: [PATCH 07/23] Schema update: tx_indices - improve further with less indirection --- src/blockchain_db/blockchain_db.h | 5 ++- src/blockchain_db/lmdb/db_lmdb.cpp | 60 +++++++++++++++++++++++------- src/blockchain_db/lmdb/db_lmdb.h | 7 ++-- src/cryptonote_core/blockchain.cpp | 5 ++- 4 files changed, 56 insertions(+), 21 deletions(-) diff --git a/src/blockchain_db/blockchain_db.h b/src/blockchain_db/blockchain_db.h index 2cdaaea6f..933cc34f5 100644 --- a/src/blockchain_db/blockchain_db.h +++ b/src/blockchain_db/blockchain_db.h @@ -473,6 +473,7 @@ public: // return true if a transaction with hash exists virtual bool tx_exists(const crypto::hash& h) const = 0; + virtual bool tx_exists(const crypto::hash& h, uint64_t& tx_index) const = 0; // return unlock time of tx with hash virtual uint64_t get_tx_unlock_time(const crypto::hash& h) const = 0; @@ -516,13 +517,13 @@ public: // return two vectors of indices: vector of amount output indices and global // output indices, corresponding to each output in the transaction with hash // - virtual void get_amount_and_global_output_indices(const crypto::hash& h, + virtual void get_amount_and_global_output_indices(const uint64_t tx_index, std::vector& amount_output_indices, std::vector& global_output_indices) const = 0; // return a vector of indices corresponding to the amount output index for // each output in the transaction with hash - virtual std::vector get_tx_amount_output_indices(const crypto::hash& h) const = 0; + virtual std::vector get_tx_amount_output_indices(const uint64_t tx_index) const = 0; // returns true if key image is present in spent key images storage virtual bool has_key_image(const crypto::key_image& img) const = 0; diff --git a/src/blockchain_db/lmdb/db_lmdb.cpp b/src/blockchain_db/lmdb/db_lmdb.cpp index 4c377544c..a07b257a9 100644 --- a/src/blockchain_db/lmdb/db_lmdb.cpp +++ b/src/blockchain_db/lmdb/db_lmdb.cpp @@ -672,6 +672,7 @@ void BlockchainLMDB::remove_transaction_data(const crypto::hash& tx_hash, const if (mdb_get(*m_write_txn, m_tx_indices, &val_h, &val_tx_index)) throw1(TX_DNE("Attempting to remove transaction that isn't in the db")); + uint64_t tx_index = *(uint64_t*)val_tx_index.mv_data; if (mdb_del(*m_write_txn, m_txs, &val_tx_index, NULL)) throw1(DB_ERROR("Failed to add removal of tx to db transaction")); @@ -680,7 +681,7 @@ void BlockchainLMDB::remove_transaction_data(const crypto::hash& tx_hash, const if (mdb_del(*m_write_txn, m_tx_heights, &val_tx_index, NULL)) throw1(DB_ERROR("Failed to add removal of tx block height to db transaction")); - remove_tx_outputs(tx_hash, tx); + remove_tx_outputs(tx_index, tx); int result = mdb_del(*m_write_txn, m_tx_outputs, &val_tx_index, NULL); if (result == MDB_NOTFOUND) @@ -796,13 +797,13 @@ void BlockchainLMDB::add_amount_and_global_output_indices(const crypto::hash& tx throw0(DB_ERROR(std::string("Failed to add to db transaction: ").append(mdb_strerror(result)).c_str())); } -void BlockchainLMDB::remove_tx_outputs(const crypto::hash& tx_hash, const transaction& tx) +void BlockchainLMDB::remove_tx_outputs(const uint64_t tx_index, const transaction& tx) { LOG_PRINT_L3("BlockchainLMDB::" << __func__); // only need global_output_indices std::vector amount_output_indices, global_output_indices; - get_amount_and_global_output_indices(tx_hash, amount_output_indices, global_output_indices); + get_amount_and_global_output_indices(tx_index, amount_output_indices, global_output_indices); if (global_output_indices.empty()) { @@ -1122,6 +1123,7 @@ void BlockchainLMDB::open(const std::string& filename, const int mdb_flags) mdb_set_compare(txn, m_spent_keys, compare_hash32); mdb_set_compare(txn, m_block_heights, compare_hash32); mdb_set_compare(txn, m_tx_indices, compare_hash32); + mdb_set_compare(txn, m_hf_starting_heights, compare_uint8); mdb_set_compare(txn, m_hf_versions, compare_uint64); mdb_set_compare(txn, m_properties, compare_string); @@ -1726,6 +1728,43 @@ bool BlockchainLMDB::tx_exists(const crypto::hash& h) const return true; } +bool BlockchainLMDB::tx_exists(const crypto::hash& h, uint64_t& tx_index) const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + TXN_PREFIX_RDONLY(); + const mdb_txn_cursors *m_cursors = m_write_txn ? &m_wcursors : &m_tinfo->m_ti_rcursors; + RCURSOR(tx_indices); + RCURSOR(txs); + + MDB_val_copy key(h); + MDB_val val_tx_index; + MDB_val result; + + TIME_MEASURE_START(time1); + auto get_result = mdb_cursor_get(m_cur_tx_indices, &key, &val_tx_index, MDB_SET); + if (get_result == 0) { + get_result = mdb_cursor_get(m_cur_txs, &val_tx_index, &result, MDB_SET); + } + TIME_MEASURE_FINISH(time1); + time_tx_exists += time1; + + TXN_POSTFIX_RDONLY(); + + if (get_result == MDB_NOTFOUND) + { + LOG_PRINT_L1("transaction with hash " << epee::string_tools::pod_to_hex(h) << " not found in db"); + return false; + } + else if (get_result) + throw0(DB_ERROR("DB error attempting to fetch transaction from hash")); + else + tx_index = *(uint64_t*)val_tx_index.mv_data; + + return true; +} + uint64_t BlockchainLMDB::get_tx_unlock_time(const crypto::hash& h) const { LOG_PRINT_L3("BlockchainLMDB::" << __func__); @@ -1950,7 +1989,7 @@ tx_out_index BlockchainLMDB::get_output_tx_and_index(const uint64_t& amount, con return indices[0]; } -void BlockchainLMDB::get_amount_and_global_output_indices(const crypto::hash& h, +void BlockchainLMDB::get_amount_and_global_output_indices(const uint64_t tx_index, std::vector& amount_output_indices, std::vector& global_output_indices) const { @@ -1970,16 +2009,9 @@ void BlockchainLMDB::get_amount_and_global_output_indices(const crypto::hash& h, RCURSOR(tx_outputs); int result = 0; - MDB_val_copy k(h); - MDB_val val_tx_index; + MDB_val_copy val_tx_index(tx_index); MDB_val v; - result = mdb_cursor_get(m_cur_tx_indices, &k, &val_tx_index, MDB_SET); - if (result == MDB_NOTFOUND) - throw1(OUTPUT_DNE(std::string("Failed to get tx index for tx hash ").append(epee::string_tools::pod_to_hex(h)).c_str())); - else if (result) - throw0(DB_ERROR(lmdb_error(std::string("DB error attempting to fetch transaction index from hash ") + epee::string_tools::pod_to_hex(h) + ": ", result).c_str())); - result = mdb_cursor_get(m_cur_tx_outputs, &val_tx_index, &v, MDB_SET); if (result == MDB_NOTFOUND) LOG_PRINT_L0("WARNING: Unexpected: tx has no amount and global indices stored in " @@ -2005,13 +2037,13 @@ void BlockchainLMDB::get_amount_and_global_output_indices(const crypto::hash& h, TXN_POSTFIX_RDONLY(); } -std::vector BlockchainLMDB::get_tx_amount_output_indices(const crypto::hash& h) const +std::vector BlockchainLMDB::get_tx_amount_output_indices(const uint64_t tx_index) const { LOG_PRINT_L3("BlockchainLMDB::" << __func__); std::vector amount_output_indices, global_output_indices; // only need amount_output_indices - get_amount_and_global_output_indices(h, amount_output_indices, global_output_indices); + get_amount_and_global_output_indices(tx_index, amount_output_indices, global_output_indices); return amount_output_indices; } diff --git a/src/blockchain_db/lmdb/db_lmdb.h b/src/blockchain_db/lmdb/db_lmdb.h index 41dccfede..ef47ea5f7 100644 --- a/src/blockchain_db/lmdb/db_lmdb.h +++ b/src/blockchain_db/lmdb/db_lmdb.h @@ -212,6 +212,7 @@ public: virtual uint64_t height() const; virtual bool tx_exists(const crypto::hash& h) const; + virtual bool tx_exists(const crypto::hash& h, uint64_t& tx_index) const; virtual uint64_t get_tx_unlock_time(const crypto::hash& h) const; @@ -237,11 +238,11 @@ public: virtual void get_output_tx_and_index(const uint64_t& amount, const std::vector &offsets, std::vector &indices); virtual void get_output_global_indices(const uint64_t& amount, const std::vector &offsets, std::vector &indices); - virtual void get_amount_and_global_output_indices(const crypto::hash& h, + virtual void get_amount_and_global_output_indices(const uint64_t tx_index, std::vector& amount_output_indices, std::vector& global_output_indices) const; - virtual std::vector get_tx_amount_output_indices(const crypto::hash& h) const; + virtual std::vector get_tx_amount_output_indices(const uint64_t tx_index) const; virtual bool has_key_image(const crypto::key_image& img) const; @@ -307,7 +308,7 @@ private: virtual void remove_output(const tx_out& tx_output); - void remove_tx_outputs(const crypto::hash& tx_hash, const transaction& tx); + void remove_tx_outputs(const uint64_t tx_index, const transaction& tx); void remove_output(const uint64_t& out_index, const uint64_t amount); void remove_amount_output_index(const uint64_t amount, const uint64_t global_output_index); diff --git a/src/cryptonote_core/blockchain.cpp b/src/cryptonote_core/blockchain.cpp index 2aa8ecec4..f5c912d76 100644 --- a/src/cryptonote_core/blockchain.cpp +++ b/src/cryptonote_core/blockchain.cpp @@ -1949,14 +1949,15 @@ bool Blockchain::get_tx_outputs_gindexs(const crypto::hash& tx_id, std::vectortx_exists(tx_id)) + uint64_t tx_index; + if (!m_db->tx_exists(tx_id, tx_index)) { LOG_PRINT_RED_L1("warning: get_tx_outputs_gindexs failed to find transaction with id = " << tx_id); return false; } // get amount output indexes, currently referred to in parts as "output global indices", but they are actually specific to amounts - indexs = m_db->get_tx_amount_output_indices(tx_id); + indexs = m_db->get_tx_amount_output_indices(tx_index); CHECK_AND_ASSERT_MES(indexs.size(), false, "internal error: global indexes for transaction " << tx_id << " is empty"); return true; From a2f518aa014163e63e39ce5e83a4f0d4d5be5c3f Mon Sep 17 00:00:00 2001 From: warptangent Date: Fri, 4 Mar 2016 11:37:41 -0800 Subject: [PATCH 08/23] Schema update: tx_indices - yet less indirection --- src/blockchain_db/blockchain_db.cpp | 4 ++-- src/blockchain_db/blockchain_db.h | 4 ++-- src/blockchain_db/lmdb/db_lmdb.cpp | 23 +++++++++-------------- src/blockchain_db/lmdb/db_lmdb.h | 4 ++-- 4 files changed, 15 insertions(+), 20 deletions(-) diff --git a/src/blockchain_db/blockchain_db.cpp b/src/blockchain_db/blockchain_db.cpp index b1b233b58..8cde4f138 100644 --- a/src/blockchain_db/blockchain_db.cpp +++ b/src/blockchain_db/blockchain_db.cpp @@ -82,7 +82,7 @@ void BlockchainDB::add_transaction(const crypto::hash& blk_hash, const transacti } } - add_transaction_data(blk_hash, tx, tx_hash); + uint64_t tx_index = add_transaction_data(blk_hash, tx, tx_hash); std::vector amount_output_indices; std::vector global_output_indices; @@ -96,7 +96,7 @@ void BlockchainDB::add_transaction(const crypto::hash& blk_hash, const transacti amount_output_indices.push_back(amount_output_index); global_output_indices.push_back(global_output_index); } - add_amount_and_global_output_indices(tx_hash, amount_output_indices, global_output_indices); + add_amount_and_global_output_indices(tx_index, amount_output_indices, global_output_indices); } uint64_t BlockchainDB::add_block( const block& blk diff --git a/src/blockchain_db/blockchain_db.h b/src/blockchain_db/blockchain_db.h index 933cc34f5..b71bb4416 100644 --- a/src/blockchain_db/blockchain_db.h +++ b/src/blockchain_db/blockchain_db.h @@ -291,7 +291,7 @@ private: virtual void remove_block() = 0; // tells the subclass to store the transaction and its metadata - virtual void add_transaction_data(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash& tx_hash) = 0; + virtual uint64_t add_transaction_data(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash& tx_hash) = 0; // tells the subclass to remove data about a transaction virtual void remove_transaction_data(const crypto::hash& tx_hash, const transaction& tx) = 0; @@ -306,7 +306,7 @@ private: ) = 0; // tells the subclass to store indices for a tx's outputs, both amount output indices and global output indices - virtual void add_amount_and_global_output_indices(const crypto::hash& tx_hash, + virtual void add_amount_and_global_output_indices(const uint64_t tx_index, const std::vector& amount_output_indices, const std::vector& global_output_indices ) = 0; diff --git a/src/blockchain_db/lmdb/db_lmdb.cpp b/src/blockchain_db/lmdb/db_lmdb.cpp index a07b257a9..a1ca2e716 100644 --- a/src/blockchain_db/lmdb/db_lmdb.cpp +++ b/src/blockchain_db/lmdb/db_lmdb.cpp @@ -616,20 +616,21 @@ void BlockchainLMDB::remove_block() throw1(DB_ERROR("Failed to add removal of block info to db transaction")); } -void BlockchainLMDB::add_transaction_data(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash& tx_hash) +uint64_t BlockchainLMDB::add_transaction_data(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash& tx_hash) { LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); mdb_txn_cursors *m_cursors = &m_wcursors; int result = 0; + uint64_t tx_index = m_num_txs; CURSOR(txs) CURSOR(tx_indices) CURSOR(tx_heights) CURSOR(tx_unlocks) - MDB_val_copy val_tx_index(m_num_txs); + MDB_val_copy val_tx_index(tx_index); MDB_val_copy val_h(tx_hash); MDB_val unused; result = mdb_cursor_get(m_cur_tx_indices, &val_h, &unused, MDB_SET); @@ -658,6 +659,7 @@ void BlockchainLMDB::add_transaction_data(const crypto::hash& blk_hash, const tr throw0(DB_ERROR(lmdb_error("Failed to add tx unlock time to db transaction: ", result).c_str())); m_num_txs++; + return tx_index; } // TODO: compare pros and cons of looking up the tx hash's tx index once and @@ -761,7 +763,7 @@ void BlockchainLMDB::add_output(const crypto::hash& tx_hash, m_num_outputs++; } -void BlockchainLMDB::add_amount_and_global_output_indices(const crypto::hash& tx_hash, +void BlockchainLMDB::add_amount_and_global_output_indices(const uint64_t tx_index, const std::vector& amount_output_indices, const std::vector& global_output_indices) { @@ -772,8 +774,6 @@ void BlockchainLMDB::add_amount_and_global_output_indices(const crypto::hash& tx int result = 0; - MDB_val_copy k(tx_hash); - int num_outputs = amount_output_indices.size(); std::unique_ptr paired_indices(new uint64_t[2*num_outputs]); for (int i = 0; i < num_outputs; ++i) @@ -782,17 +782,13 @@ void BlockchainLMDB::add_amount_and_global_output_indices(const crypto::hash& tx paired_indices[2*i+1] = global_output_indices[i]; } + MDB_val_copy k_tx_index(tx_index); MDB_val v; v.mv_data = (void*)paired_indices.get(); v.mv_size = sizeof(uint64_t) * 2 * num_outputs; // LOG_PRINT_L1("tx_outputs[tx_hash] size: " << v.mv_size); - MDB_val val_tx_index; - result = mdb_cursor_get(m_cur_tx_indices, &k, &val_tx_index, MDB_SET); - if (result) - throw0(DB_ERROR(lmdb_error(std::string("Failed to get tx index for tx hash ") + epee::string_tools::pod_to_hex(tx_hash), result).c_str())); - - result = mdb_cursor_put(m_cur_tx_outputs, &val_tx_index, &v, 0); + result = mdb_cursor_put(m_cur_tx_outputs, &k_tx_index, &v, 0); if (result) throw0(DB_ERROR(std::string("Failed to add to db transaction: ").append(mdb_strerror(result)).c_str())); } @@ -1724,7 +1720,6 @@ bool BlockchainLMDB::tx_exists(const crypto::hash& h) const throw0(DB_ERROR(std::string("transaction with hash ").append(epee::string_tools::pod_to_hex(h)).append(" not found at index").c_str())); else if (get_result) throw0(DB_ERROR(lmdb_error(std::string("DB error attempting to fetch transaction ") + epee::string_tools::pod_to_hex(h) + " at index: ", get_result).c_str())); - return true; } @@ -2009,10 +2004,10 @@ void BlockchainLMDB::get_amount_and_global_output_indices(const uint64_t tx_inde RCURSOR(tx_outputs); int result = 0; - MDB_val_copy val_tx_index(tx_index); + MDB_val_copy k_tx_index(tx_index); MDB_val v; - result = mdb_cursor_get(m_cur_tx_outputs, &val_tx_index, &v, MDB_SET); + result = mdb_cursor_get(m_cur_tx_outputs, &k_tx_index, &v, MDB_SET); if (result == MDB_NOTFOUND) LOG_PRINT_L0("WARNING: Unexpected: tx has no amount and global indices stored in " "tx_outputs, but it should have an empty entry even if it's a tx without " diff --git a/src/blockchain_db/lmdb/db_lmdb.h b/src/blockchain_db/lmdb/db_lmdb.h index ef47ea5f7..e60103712 100644 --- a/src/blockchain_db/lmdb/db_lmdb.h +++ b/src/blockchain_db/lmdb/db_lmdb.h @@ -289,7 +289,7 @@ private: virtual void remove_block(); - virtual void add_transaction_data(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash& tx_hash); + virtual uint64_t add_transaction_data(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash& tx_hash); virtual void remove_transaction_data(const crypto::hash& tx_hash, const transaction& tx); @@ -301,7 +301,7 @@ private: uint64_t& global_output_index ); - virtual void add_amount_and_global_output_indices(const crypto::hash& tx_hash, + virtual void add_amount_and_global_output_indices(const uint64_t tx_index, const std::vector& amount_output_indices, const std::vector& global_output_indices ); From 9aadedb1d0eb4f922b33edfa0ede1f4d1ac128d1 Mon Sep 17 00:00:00 2001 From: warptangent Date: Fri, 4 Mar 2016 11:56:36 -0800 Subject: [PATCH 09/23] Schema update: tx_indices - consolidate the tx subdbs from 5 to 3 --- src/blockchain_db/blockchain_db.h | 9 +++ src/blockchain_db/lmdb/db_lmdb.cpp | 114 ++++++++++++----------------- src/blockchain_db/lmdb/db_lmdb.h | 8 -- 3 files changed, 56 insertions(+), 75 deletions(-) diff --git a/src/blockchain_db/blockchain_db.h b/src/blockchain_db/blockchain_db.h index b71bb4416..665cccd40 100644 --- a/src/blockchain_db/blockchain_db.h +++ b/src/blockchain_db/blockchain_db.h @@ -148,6 +148,15 @@ struct output_data_t }; #pragma pack(pop) +#pragma pack(push, 1) +struct tx_data_t +{ + uint64_t tx_index; + uint64_t unlock_time; + uint64_t height; +}; +#pragma pack(pop) + /*********************************** * Exception Definitions ***********************************/ diff --git a/src/blockchain_db/lmdb/db_lmdb.cpp b/src/blockchain_db/lmdb/db_lmdb.cpp index a1ca2e716..eaba95cad 100644 --- a/src/blockchain_db/lmdb/db_lmdb.cpp +++ b/src/blockchain_db/lmdb/db_lmdb.cpp @@ -157,8 +157,6 @@ const char* const LMDB_BLOCK_HEIGHTS = "block_heights"; const char* const LMDB_TXS = "txs"; const char* const LMDB_TX_INDICES = "tx_indices"; -const char* const LMDB_TX_UNLOCKS = "tx_unlocks"; -const char* const LMDB_TX_HEIGHTS = "tx_heights"; const char* const LMDB_TX_OUTPUTS = "tx_outputs"; const char* const LMDB_OUTPUT_TXS = "output_txs"; @@ -627,8 +625,6 @@ uint64_t BlockchainLMDB::add_transaction_data(const crypto::hash& blk_hash, cons CURSOR(txs) CURSOR(tx_indices) - CURSOR(tx_heights) - CURSOR(tx_unlocks) MDB_val_copy val_tx_index(tx_index); MDB_val_copy val_h(tx_hash); @@ -639,25 +635,22 @@ uint64_t BlockchainLMDB::add_transaction_data(const crypto::hash& blk_hash, cons else if (result != MDB_NOTFOUND) throw1(DB_ERROR(lmdb_error(std::string("Error checking if tx index exists for tx hash ") + epee::string_tools::pod_to_hex(tx_hash) + ": ", result).c_str())); - result = mdb_cursor_put(m_cur_tx_indices, &val_h, &val_tx_index, 0); + tx_data_t td; + td.tx_index = tx_index; + td.unlock_time = tx.unlock_time; + td.height = m_height; + + MDB_val_copy tx_data(td); + + result = mdb_cursor_put(m_cur_tx_indices, &val_h, &tx_data, 0); if (result) - throw0(DB_ERROR(lmdb_error("Failed to add tx index to db transaction: ", result).c_str())); + throw0(DB_ERROR(lmdb_error("Failed to add tx data to db transaction: ", result).c_str())); MDB_val_copy blob(tx_to_blob(tx)); result = mdb_cursor_put(m_cur_txs, &val_tx_index, &blob, 0); if (result) throw0(DB_ERROR(lmdb_error("Failed to add tx blob to db transaction: ", result).c_str())); - MDB_val_copy height(m_height); - result = mdb_cursor_put(m_cur_tx_heights, &val_tx_index, &height, 0); - if (result) - throw0(DB_ERROR(lmdb_error("Failed to add tx block height to db transaction: ", result).c_str())); - - MDB_val_copy unlock_time(tx.unlock_time); - result = mdb_cursor_put(m_cur_tx_unlocks, &val_tx_index, &unlock_time, 0); - if (result) - throw0(DB_ERROR(lmdb_error("Failed to add tx unlock time to db transaction: ", result).c_str())); - m_num_txs++; return tx_index; } @@ -670,18 +663,16 @@ void BlockchainLMDB::remove_transaction_data(const crypto::hash& tx_hash, const check_open(); MDB_val_copy val_h(tx_hash); - MDB_val val_tx_index; + MDB_val v; - if (mdb_get(*m_write_txn, m_tx_indices, &val_h, &val_tx_index)) + if (mdb_get(*m_write_txn, m_tx_indices, &val_h, &v)) throw1(TX_DNE("Attempting to remove transaction that isn't in the db")); - uint64_t tx_index = *(uint64_t*)val_tx_index.mv_data; + tx_data_t td = *(const tx_data_t*)v.mv_data; + uint64_t tx_index = td.tx_index; + MDB_val_copy val_tx_index(tx_index); if (mdb_del(*m_write_txn, m_txs, &val_tx_index, NULL)) throw1(DB_ERROR("Failed to add removal of tx to db transaction")); - if (mdb_del(*m_write_txn, m_tx_unlocks, &val_tx_index, NULL)) - throw1(DB_ERROR("Failed to add removal of tx unlock time to db transaction")); - if (mdb_del(*m_write_txn, m_tx_heights, &val_tx_index, NULL)) - throw1(DB_ERROR("Failed to add removal of tx block height to db transaction")); remove_tx_outputs(tx_index, tx); @@ -1099,8 +1090,6 @@ void BlockchainLMDB::open(const std::string& filename, const int mdb_flags) lmdb_db_open(txn, LMDB_TXS, MDB_INTEGERKEY | MDB_CREATE, m_txs, "Failed to open db handle for m_txs"); lmdb_db_open(txn, LMDB_TX_INDICES, MDB_CREATE, m_tx_indices, "Failed to open db handle for m_tx_indices"); - lmdb_db_open(txn, LMDB_TX_UNLOCKS, MDB_INTEGERKEY | MDB_CREATE, m_tx_unlocks, "Failed to open db handle for m_tx_unlocks"); - lmdb_db_open(txn, LMDB_TX_HEIGHTS, MDB_INTEGERKEY | MDB_CREATE, m_tx_heights, "Failed to open db handle for m_tx_heights"); lmdb_db_open(txn, LMDB_TX_OUTPUTS, MDB_INTEGERKEY | MDB_CREATE, m_tx_outputs, "Failed to open db handle for m_tx_outputs"); lmdb_db_open(txn, LMDB_OUTPUT_TXS, MDB_INTEGERKEY | MDB_CREATE, m_output_txs, "Failed to open db handle for m_output_txs"); @@ -1265,8 +1254,6 @@ void BlockchainLMDB::reset() mdb_drop(txn, m_block_info, 0); mdb_drop(txn, m_block_heights, 0); mdb_drop(txn, m_txs, 0); - mdb_drop(txn, m_tx_unlocks, 0); - mdb_drop(txn, m_tx_heights, 0); mdb_drop(txn, m_tx_outputs, 0); mdb_drop(txn, m_output_txs, 0); mdb_drop(txn, m_output_indices, 0); @@ -1693,18 +1680,18 @@ bool BlockchainLMDB::tx_exists(const crypto::hash& h) const RCURSOR(txs); MDB_val_copy key(h); - MDB_val val_tx_index; - MDB_val result; + MDB_val v; bool tx_found = false; TIME_MEASURE_START(time1); - auto get_result = mdb_cursor_get(m_cur_tx_indices, &key, &val_tx_index, MDB_SET); + auto get_result = mdb_cursor_get(m_cur_tx_indices, &key, &v, MDB_SET); if (get_result == 0) tx_found = true; else if (get_result != MDB_NOTFOUND) throw0(DB_ERROR(lmdb_error(std::string("DB error attempting to fetch transaction index from hash ") + epee::string_tools::pod_to_hex(h) + ": ", get_result).c_str())); - get_result = mdb_cursor_get(m_cur_txs, &val_tx_index, &result, MDB_SET); + // This isn't needed as part of the check. we're not checking consistency of db. + // get_result = mdb_cursor_get(m_cur_txs, &val_tx_index, &result, MDB_SET); TIME_MEASURE_FINISH(time1); time_tx_exists += time1; @@ -1716,10 +1703,11 @@ bool BlockchainLMDB::tx_exists(const crypto::hash& h) const return false; } - if (get_result == MDB_NOTFOUND) - throw0(DB_ERROR(std::string("transaction with hash ").append(epee::string_tools::pod_to_hex(h)).append(" not found at index").c_str())); - else if (get_result) - throw0(DB_ERROR(lmdb_error(std::string("DB error attempting to fetch transaction ") + epee::string_tools::pod_to_hex(h) + " at index: ", get_result).c_str())); + // Below not needed due to above comment. + // if (get_result == MDB_NOTFOUND) + // throw0(DB_ERROR(std::string("transaction with hash ").append(epee::string_tools::pod_to_hex(h)).append(" not found at index").c_str())); + // else if (get_result) + // throw0(DB_ERROR(lmdb_error(std::string("DB error attempting to fetch transaction ") + epee::string_tools::pod_to_hex(h) + " at index: ", get_result).c_str())); return true; } @@ -1731,17 +1719,12 @@ bool BlockchainLMDB::tx_exists(const crypto::hash& h, uint64_t& tx_index) const TXN_PREFIX_RDONLY(); const mdb_txn_cursors *m_cursors = m_write_txn ? &m_wcursors : &m_tinfo->m_ti_rcursors; RCURSOR(tx_indices); - RCURSOR(txs); MDB_val_copy key(h); - MDB_val val_tx_index; - MDB_val result; + MDB_val v; TIME_MEASURE_START(time1); - auto get_result = mdb_cursor_get(m_cur_tx_indices, &key, &val_tx_index, MDB_SET); - if (get_result == 0) { - get_result = mdb_cursor_get(m_cur_txs, &val_tx_index, &result, MDB_SET); - } + auto get_result = mdb_cursor_get(m_cur_tx_indices, &key, &v, MDB_SET); TIME_MEASURE_FINISH(time1); time_tx_exists += time1; @@ -1755,8 +1738,10 @@ bool BlockchainLMDB::tx_exists(const crypto::hash& h, uint64_t& tx_index) const else if (get_result) throw0(DB_ERROR("DB error attempting to fetch transaction from hash")); else - tx_index = *(uint64_t*)val_tx_index.mv_data; - + { + tx_data_t td = *(const tx_data_t*)v.mv_data; + tx_index = td.tx_index; + } return true; } @@ -1768,24 +1753,17 @@ uint64_t BlockchainLMDB::get_tx_unlock_time(const crypto::hash& h) const TXN_PREFIX_RDONLY(); const mdb_txn_cursors *m_cursors = m_write_txn ? &m_wcursors : &m_tinfo->m_ti_rcursors; RCURSOR(tx_indices); - RCURSOR(tx_unlocks); MDB_val_copy key(h); - MDB_val val_tx_index; - MDB_val result; - auto get_result = mdb_cursor_get(m_cur_tx_indices, &key, &val_tx_index, MDB_SET); + MDB_val v; + auto get_result = mdb_cursor_get(m_cur_tx_indices, &key, &v, MDB_SET); if (get_result == MDB_NOTFOUND) - throw1(TX_DNE(lmdb_error(std::string("tx index with hash ") + epee::string_tools::pod_to_hex(h) + " not found in db: ", get_result).c_str())); + throw1(TX_DNE(lmdb_error(std::string("tx data with hash ") + epee::string_tools::pod_to_hex(h) + " not found in db: ", get_result).c_str())); else if (get_result) - throw0(DB_ERROR(lmdb_error("DB error attempting to fetch tx index from hash: ", get_result).c_str())); + throw0(DB_ERROR(lmdb_error("DB error attempting to fetch tx data from hash: ", get_result).c_str())); - get_result = mdb_cursor_get(m_cur_tx_unlocks, &val_tx_index, &result, MDB_SET); - if (get_result == MDB_NOTFOUND) - throw1(TX_DNE(lmdb_error(std::string("tx unlock time with hash ") + epee::string_tools::pod_to_hex(h) + " not found in db: ", get_result).c_str())); - else if (get_result) - throw0(DB_ERROR(lmdb_error("DB error attempting to fetch tx unlock time from index: ", get_result).c_str())); - - uint64_t ret = *(const uint64_t*)result.mv_data; + tx_data_t td = *(const tx_data_t*)v.mv_data; + uint64_t ret = td.unlock_time; TXN_POSTFIX_RDONLY(); return ret; } @@ -1801,11 +1779,16 @@ transaction BlockchainLMDB::get_tx(const crypto::hash& h) const RCURSOR(txs); MDB_val_copy key(h); - MDB_val val_tx_index; + MDB_val v; MDB_val result; - auto get_result = mdb_cursor_get(m_cur_tx_indices, &key, &val_tx_index, MDB_SET); + auto get_result = mdb_cursor_get(m_cur_tx_indices, &key, &v, MDB_SET); if (get_result == 0) + { + tx_data_t td = *(const tx_data_t*)v.mv_data; + uint64_t tx_index = td.tx_index; + MDB_val_copy val_tx_index(tx_index); get_result = mdb_cursor_get(m_cur_txs, &val_tx_index, &result, MDB_SET); + } if (get_result == MDB_NOTFOUND) throw1(TX_DNE(std::string("tx with hash ").append(epee::string_tools::pod_to_hex(h)).append(" not found in db").c_str())); else if (get_result) @@ -1861,22 +1844,19 @@ uint64_t BlockchainLMDB::get_tx_block_height(const crypto::hash& h) const TXN_PREFIX_RDONLY(); const mdb_txn_cursors *m_cursors = m_write_txn ? &m_wcursors : &m_tinfo->m_ti_rcursors; RCURSOR(tx_indices); - RCURSOR(tx_heights); MDB_val_copy key(h); - MDB_val val_tx_index; - MDB_val result; - auto get_result = mdb_cursor_get(m_cur_tx_indices, &key, &val_tx_index, MDB_SET); - if (get_result == 0) - get_result = mdb_cursor_get(m_cur_tx_heights, &val_tx_index, &result, MDB_SET); + MDB_val v; + auto get_result = mdb_cursor_get(m_cur_tx_indices, &key, &v, MDB_SET); if (get_result == MDB_NOTFOUND) { - throw1(TX_DNE(std::string("tx height with hash ").append(epee::string_tools::pod_to_hex(h)).append(" not found in db").c_str())); + throw1(TX_DNE(std::string("tx_data_t with hash ").append(epee::string_tools::pod_to_hex(h)).append(" not found in db").c_str())); } else if (get_result) throw0(DB_ERROR("DB error attempting to fetch tx height from hash")); - uint64_t ret = *(const uint64_t*)result.mv_data; + tx_data_t res = *(const tx_data_t *)v.mv_data; + uint64_t ret = res.height; TXN_POSTFIX_RDONLY(); return ret; } diff --git a/src/blockchain_db/lmdb/db_lmdb.h b/src/blockchain_db/lmdb/db_lmdb.h index e60103712..0eda3bc4d 100644 --- a/src/blockchain_db/lmdb/db_lmdb.h +++ b/src/blockchain_db/lmdb/db_lmdb.h @@ -52,8 +52,6 @@ typedef struct mdb_txn_cursors MDB_cursor *m_txc_txs; MDB_cursor *m_txc_tx_indices; - MDB_cursor *m_txc_tx_heights; - MDB_cursor *m_txc_tx_unlocks; MDB_cursor *m_txc_tx_outputs; MDB_cursor *m_txc_spent_keys; @@ -70,8 +68,6 @@ typedef struct mdb_txn_cursors #define m_cur_output_keys m_cursors->m_txc_output_keys #define m_cur_txs m_cursors->m_txc_txs #define m_cur_tx_indices m_cursors->m_txc_tx_indices -#define m_cur_tx_heights m_cursors->m_txc_tx_heights -#define m_cur_tx_unlocks m_cursors->m_txc_tx_unlocks #define m_cur_tx_outputs m_cursors->m_txc_tx_outputs #define m_cur_spent_keys m_cursors->m_txc_spent_keys #define m_cur_hf_versions m_cursors->m_txc_hf_versions @@ -88,8 +84,6 @@ typedef struct mdb_rflags bool m_rf_output_keys; bool m_rf_txs; bool m_rf_tx_indices; - bool m_rf_tx_heights; - bool m_rf_tx_unlocks; bool m_rf_tx_outputs; bool m_rf_spent_keys; bool m_rf_hf_versions; @@ -368,8 +362,6 @@ private: MDB_dbi m_txs; MDB_dbi m_tx_indices; - MDB_dbi m_tx_unlocks; - MDB_dbi m_tx_heights; MDB_dbi m_tx_outputs; MDB_dbi m_output_txs; From 46b991b3629856b4d5c8c59fb18e7a55d23f78a1 Mon Sep 17 00:00:00 2001 From: warptangent Date: Fri, 4 Mar 2016 11:58:30 -0800 Subject: [PATCH 10/23] Use MDB_APPEND mode with two tx subdbs This is possible on those using a tx index as a key. --- src/blockchain_db/lmdb/db_lmdb.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/blockchain_db/lmdb/db_lmdb.cpp b/src/blockchain_db/lmdb/db_lmdb.cpp index eaba95cad..f97c2209f 100644 --- a/src/blockchain_db/lmdb/db_lmdb.cpp +++ b/src/blockchain_db/lmdb/db_lmdb.cpp @@ -647,7 +647,7 @@ uint64_t BlockchainLMDB::add_transaction_data(const crypto::hash& blk_hash, cons throw0(DB_ERROR(lmdb_error("Failed to add tx data to db transaction: ", result).c_str())); MDB_val_copy blob(tx_to_blob(tx)); - result = mdb_cursor_put(m_cur_txs, &val_tx_index, &blob, 0); + result = mdb_cursor_put(m_cur_txs, &val_tx_index, &blob, MDB_APPEND); if (result) throw0(DB_ERROR(lmdb_error("Failed to add tx blob to db transaction: ", result).c_str())); @@ -779,7 +779,7 @@ void BlockchainLMDB::add_amount_and_global_output_indices(const uint64_t tx_inde v.mv_size = sizeof(uint64_t) * 2 * num_outputs; // LOG_PRINT_L1("tx_outputs[tx_hash] size: " << v.mv_size); - result = mdb_cursor_put(m_cur_tx_outputs, &k_tx_index, &v, 0); + result = mdb_cursor_put(m_cur_tx_outputs, &k_tx_index, &v, MDB_APPEND); if (result) throw0(DB_ERROR(std::string("Failed to add to db transaction: ").append(mdb_strerror(result)).c_str())); } From bf769c32ba6b08ca7315c7b99a3b0ee1eb8da7df Mon Sep 17 00:00:00 2001 From: warptangent Date: Fri, 4 Mar 2016 12:05:51 -0800 Subject: [PATCH 11/23] Add back changes from revert. m_tx_outputs doesn't need to be changed, as it's no longer dup list. --- src/blockchain_db/lmdb/db_lmdb.cpp | 21 ++------------------- 1 file changed, 2 insertions(+), 19 deletions(-) diff --git a/src/blockchain_db/lmdb/db_lmdb.cpp b/src/blockchain_db/lmdb/db_lmdb.cpp index f97c2209f..48bde6038 100644 --- a/src/blockchain_db/lmdb/db_lmdb.cpp +++ b/src/blockchain_db/lmdb/db_lmdb.cpp @@ -108,21 +108,6 @@ private: std::unique_ptr data; }; -int compare_uint64(const MDB_val *a, const MDB_val *b) -{ -#ifdef MISALIGNED_OK - const uint64_t va = *(const uint64_t*)a->mv_data; - const uint64_t vb = *(const uint64_t*)b->mv_data; -#else - uint64_t va, vb; - memcpy(&va, a->mv_data, sizeof(uint64_t)); - memcpy(&vb, b->mv_data, sizeof(uint64_t)); -#endif - if (va < vb) return -1; - else if (va == vb) return 0; - else return 1; -}; - int compare_uint8(const MDB_val *a, const MDB_val *b) { const uint8_t va = *(const uint8_t*)a->mv_data; @@ -1094,23 +1079,21 @@ void BlockchainLMDB::open(const std::string& filename, const int mdb_flags) lmdb_db_open(txn, LMDB_OUTPUT_TXS, MDB_INTEGERKEY | MDB_CREATE, m_output_txs, "Failed to open db handle for m_output_txs"); lmdb_db_open(txn, LMDB_OUTPUT_INDICES, MDB_INTEGERKEY | MDB_CREATE, m_output_indices, "Failed to open db handle for m_output_indices"); - lmdb_db_open(txn, LMDB_OUTPUT_AMOUNTS, MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED | MDB_CREATE, m_output_amounts, "Failed to open db handle for m_output_amounts"); + lmdb_db_open(txn, LMDB_OUTPUT_AMOUNTS, MDB_INTEGERKEY | MDB_INTEGERDUP | MDB_DUPSORT | MDB_DUPFIXED | MDB_CREATE, m_output_amounts, "Failed to open db handle for m_output_amounts"); lmdb_db_open(txn, LMDB_OUTPUT_KEYS, MDB_INTEGERKEY | MDB_CREATE, m_output_keys, "Failed to open db handle for m_output_keys"); lmdb_db_open(txn, LMDB_SPENT_KEYS, MDB_CREATE, m_spent_keys, "Failed to open db handle for m_spent_keys"); lmdb_db_open(txn, LMDB_HF_STARTING_HEIGHTS, MDB_CREATE, m_hf_starting_heights, "Failed to open db handle for m_hf_starting_heights"); - lmdb_db_open(txn, LMDB_HF_VERSIONS, MDB_CREATE, m_hf_versions, "Failed to open db handle for m_hf_versions"); + lmdb_db_open(txn, LMDB_HF_VERSIONS, MDB_INTEGERKEY | MDB_CREATE, m_hf_versions, "Failed to open db handle for m_hf_versions"); lmdb_db_open(txn, LMDB_PROPERTIES, MDB_CREATE, m_properties, "Failed to open db handle for m_properties"); - mdb_set_dupsort(txn, m_output_amounts, compare_uint64); mdb_set_compare(txn, m_spent_keys, compare_hash32); mdb_set_compare(txn, m_block_heights, compare_hash32); mdb_set_compare(txn, m_tx_indices, compare_hash32); mdb_set_compare(txn, m_hf_starting_heights, compare_uint8); - mdb_set_compare(txn, m_hf_versions, compare_uint64); mdb_set_compare(txn, m_properties, compare_string); // get and keep current height From 38c2277d6f821ae92ab0109157b1a7980802391c Mon Sep 17 00:00:00 2001 From: Howard Chu Date: Sun, 6 Mar 2016 05:05:49 +0000 Subject: [PATCH 12/23] Use DUPFIXED for spent_keys --- src/blockchain_db/lmdb/db_lmdb.cpp | 41 +++++++++++++----------------- 1 file changed, 18 insertions(+), 23 deletions(-) diff --git a/src/blockchain_db/lmdb/db_lmdb.cpp b/src/blockchain_db/lmdb/db_lmdb.cpp index 48bde6038..46e62eef3 100644 --- a/src/blockchain_db/lmdb/db_lmdb.cpp +++ b/src/blockchain_db/lmdb/db_lmdb.cpp @@ -155,6 +155,8 @@ const char* const LMDB_HF_VERSIONS = "hf_versions"; const char* const LMDB_PROPERTIES = "properties"; +const char zerokey[8] = {0}; +const MDB_val zerokval = { sizeof(zerokey), (void *)zerokey }; const std::string lmdb_error(const std::string& error_string, int mdb_res) { @@ -901,16 +903,13 @@ void BlockchainLMDB::add_spent_key(const crypto::key_image& k_image) CURSOR(spent_keys) - MDB_val_copy val_key(k_image); - MDB_val unused; - if (mdb_cursor_get(m_cur_spent_keys, &val_key, &unused, MDB_SET) == 0) + MDB_val k = {sizeof(k_image), (void *)&k_image}; + if (auto result = mdb_cursor_put(m_cur_spent_keys, (MDB_val *)&zerokval, &k, MDB_NODUPDATA)) { + if (result == MDB_KEYEXIST) throw1(KEY_IMAGE_EXISTS("Attempting to add spent key image that's already in the db")); - - char anything = '\0'; - unused.mv_size = sizeof(char); - unused.mv_data = &anything; - if (auto result = mdb_cursor_put(m_cur_spent_keys, &val_key, &unused, 0)) - throw1(DB_ERROR(lmdb_error("Error adding spent key image to db transaction: ", result).c_str())); + else + throw1(DB_ERROR(lmdb_error("Error adding spent key image to db transaction: ", result).c_str())); + } } void BlockchainLMDB::remove_spent_key(const crypto::key_image& k_image) @@ -918,8 +917,8 @@ void BlockchainLMDB::remove_spent_key(const crypto::key_image& k_image) LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); - MDB_val_copy k(k_image); - auto result = mdb_del(*m_write_txn, m_spent_keys, &k, NULL); + MDB_val k = {sizeof(k_image), (void *)&k_image}; + auto result = mdb_del(*m_write_txn, m_spent_keys, (MDB_val *)&zerokval, &k); if (result != 0 && result != MDB_NOTFOUND) throw1(DB_ERROR("Error adding removal of key image to db transaction")); } @@ -1082,14 +1081,14 @@ void BlockchainLMDB::open(const std::string& filename, const int mdb_flags) lmdb_db_open(txn, LMDB_OUTPUT_AMOUNTS, MDB_INTEGERKEY | MDB_INTEGERDUP | MDB_DUPSORT | MDB_DUPFIXED | MDB_CREATE, m_output_amounts, "Failed to open db handle for m_output_amounts"); lmdb_db_open(txn, LMDB_OUTPUT_KEYS, MDB_INTEGERKEY | MDB_CREATE, m_output_keys, "Failed to open db handle for m_output_keys"); - lmdb_db_open(txn, LMDB_SPENT_KEYS, MDB_CREATE, m_spent_keys, "Failed to open db handle for m_spent_keys"); + lmdb_db_open(txn, LMDB_SPENT_KEYS, MDB_CREATE | MDB_DUPSORT | MDB_DUPFIXED, m_spent_keys, "Failed to open db handle for m_spent_keys"); lmdb_db_open(txn, LMDB_HF_STARTING_HEIGHTS, MDB_CREATE, m_hf_starting_heights, "Failed to open db handle for m_hf_starting_heights"); lmdb_db_open(txn, LMDB_HF_VERSIONS, MDB_INTEGERKEY | MDB_CREATE, m_hf_versions, "Failed to open db handle for m_hf_versions"); lmdb_db_open(txn, LMDB_PROPERTIES, MDB_CREATE, m_properties, "Failed to open db handle for m_properties"); - mdb_set_compare(txn, m_spent_keys, compare_hash32); + mdb_set_dupsort(txn, m_spent_keys, compare_hash32); mdb_set_compare(txn, m_block_heights, compare_hash32); mdb_set_compare(txn, m_tx_indices, compare_hash32); @@ -2013,19 +2012,16 @@ bool BlockchainLMDB::has_key_image(const crypto::key_image& img) const LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); + bool ret; + TXN_PREFIX_RDONLY(); const mdb_txn_cursors *m_cursors = m_write_txn ? &m_wcursors : &m_tinfo->m_ti_rcursors; RCURSOR(spent_keys); - MDB_val_copy val_key(img); - if (mdb_cursor_get(m_cur_spent_keys, &val_key, NULL, MDB_SET) == 0) - { - TXN_POSTFIX_RDONLY(); - return true; - } - + MDB_val k = {sizeof(img), (void *)&img}; + ret = (mdb_cursor_get(m_cur_spent_keys, (MDB_val *)&zerokval, &k, MDB_GET_BOTH) == 0); TXN_POSTFIX_RDONLY(); - return false; + return ret; } bool BlockchainLMDB::for_all_key_images(std::function f) const @@ -2038,13 +2034,12 @@ bool BlockchainLMDB::for_all_key_images(std::function Date: Sun, 6 Mar 2016 06:21:15 +0000 Subject: [PATCH 13/23] Use DUPFIXED for block_heights Only a small savings... --- src/blockchain_db/lmdb/db_lmdb.cpp | 51 +++++++++++++++++------------- 1 file changed, 29 insertions(+), 22 deletions(-) diff --git a/src/blockchain_db/lmdb/db_lmdb.cpp b/src/blockchain_db/lmdb/db_lmdb.cpp index 46e62eef3..e4ec1b2df 100644 --- a/src/blockchain_db/lmdb/db_lmdb.cpp +++ b/src/blockchain_db/lmdb/db_lmdb.cpp @@ -158,6 +158,11 @@ const char* const LMDB_PROPERTIES = "properties"; const char zerokey[8] = {0}; const MDB_val zerokval = { sizeof(zerokey), (void *)zerokey }; +typedef struct blk_height { + crypto::hash key; + uint64_t height; +} blk_height; + const std::string lmdb_error(const std::string& error_string, int mdb_res) { const std::string full_string = error_string + mdb_strerror(mdb_res); @@ -519,22 +524,23 @@ void BlockchainLMDB::add_block(const block& blk, const size_t& block_size, const mdb_txn_cursors *m_cursors = &m_wcursors; CURSOR(block_heights) - MDB_val_copy val_h(blk_hash); - if (mdb_cursor_get(m_cur_block_heights, &val_h, NULL, MDB_SET) == 0) + blk_height bh = {blk_hash, m_height}; + MDB_val val_h = {sizeof(bh), (void *)&bh}; + if (mdb_cursor_get(m_cur_block_heights, (MDB_val *)&zerokval, &val_h, MDB_GET_BOTH) == 0) throw1(BLOCK_EXISTS("Attempting to add block that's already in the db")); if (m_height > 0) { - MDB_val_copy parent_key(blk.prev_id); - MDB_val parent_h; - if (mdb_cursor_get(m_cur_block_heights, &parent_key, &parent_h, MDB_SET)) + blk_height ph = {blk.prev_id, 0}; + MDB_val parent_key = {sizeof(ph), (void *)&ph}; + if (mdb_cursor_get(m_cur_block_heights, (MDB_val *)&zerokval, &parent_key, MDB_GET_BOTH)) { LOG_PRINT_L3("m_height: " << m_height); LOG_PRINT_L3("parent_key: " << blk.prev_id); throw0(DB_ERROR("Failed to get top block hash to check for new block's parent")); } - uint64_t parent_height = *(const uint64_t *)parent_h.mv_data; - if (parent_height != m_height - 1) + blk_height *prev = (blk_height *)parent_key.mv_data; + if (prev->height != m_height - 1) throw0(BLOCK_PARENT_DNE("Top block is not new block's parent")); } @@ -557,14 +563,12 @@ void BlockchainLMDB::add_block(const block& blk, const size_t& block_size, const bi.bi_diff = cumulative_difficulty; bi.bi_hash = blk_hash; - MDB_val val; - val.mv_data = (void *)&bi; - val.mv_size = sizeof(bi); + MDB_val val = {sizeof(bi), (void *)&bi}; result = mdb_cursor_put(m_cur_block_info, &key, &val, MDB_APPEND); if (result) throw0(DB_ERROR(lmdb_error("Failed to add block info to db transaction: ", result).c_str())); - result = mdb_cursor_put(m_cur_block_heights, &val_h, &key, 0); + result = mdb_cursor_put(m_cur_block_heights, (MDB_val *)&zerokval, &val_h, 0); if (result) throw0(DB_ERROR(lmdb_error("Failed to add block height by hash to db transaction: ", result).c_str())); @@ -589,9 +593,10 @@ void BlockchainLMDB::remove_block() // must use h now; deleting from m_block_info will invalidate it mdb_block_info *bi = (mdb_block_info *)h.mv_data; - h.mv_data = &bi->bi_hash; - h.mv_size = sizeof(bi->bi_hash); - if (mdb_del(*m_write_txn, m_block_heights, &h, NULL)) + blk_height bh = {bi->bi_hash, 0}; + h.mv_data = (void *)&bh; + h.mv_size = sizeof(bh); + if (mdb_del(*m_write_txn, m_block_heights, (MDB_val *)&zerokval, &h)) throw1(DB_ERROR("Failed to add removal of block height by hash to db transaction")); if (mdb_del(*m_write_txn, m_blocks, &k, NULL)) @@ -1070,7 +1075,7 @@ void BlockchainLMDB::open(const std::string& filename, const int mdb_flags) lmdb_db_open(txn, LMDB_BLOCKS, MDB_INTEGERKEY | MDB_CREATE, m_blocks, "Failed to open db handle for m_blocks"); lmdb_db_open(txn, LMDB_BLOCK_INFO, MDB_INTEGERKEY | MDB_CREATE, m_block_info, "Failed to open db handle for m_block_info"); - lmdb_db_open(txn, LMDB_BLOCK_HEIGHTS, MDB_CREATE, m_block_heights, "Failed to open db handle for m_block_heights"); + lmdb_db_open(txn, LMDB_BLOCK_HEIGHTS, MDB_CREATE | MDB_DUPSORT | MDB_DUPFIXED, m_block_heights, "Failed to open db handle for m_block_heights"); lmdb_db_open(txn, LMDB_TXS, MDB_INTEGERKEY | MDB_CREATE, m_txs, "Failed to open db handle for m_txs"); lmdb_db_open(txn, LMDB_TX_INDICES, MDB_CREATE, m_tx_indices, "Failed to open db handle for m_tx_indices"); @@ -1089,7 +1094,7 @@ void BlockchainLMDB::open(const std::string& filename, const int mdb_flags) lmdb_db_open(txn, LMDB_PROPERTIES, MDB_CREATE, m_properties, "Failed to open db handle for m_properties"); mdb_set_dupsort(txn, m_spent_keys, compare_hash32); - mdb_set_compare(txn, m_block_heights, compare_hash32); + mdb_set_dupsort(txn, m_block_heights, compare_hash32); mdb_set_compare(txn, m_tx_indices, compare_hash32); mdb_set_compare(txn, m_hf_starting_heights, compare_uint8); @@ -1348,8 +1353,9 @@ bool BlockchainLMDB::block_exists(const crypto::hash& h) const const mdb_txn_cursors *m_cursors = m_write_txn ? &m_wcursors : &m_tinfo->m_ti_rcursors; RCURSOR(block_heights); - MDB_val_copy key(h); - auto get_result = mdb_cursor_get(m_cur_block_heights, &key, NULL, MDB_SET); + blk_height bh = {h, 0}; + MDB_val key = {sizeof(bh), (void *)&bh}; + auto get_result = mdb_cursor_get(m_cur_block_heights, (MDB_val *)&zerokval, &key, MDB_GET_BOTH); if (get_result == MDB_NOTFOUND) { TXN_POSTFIX_RDONLY(); @@ -1380,15 +1386,16 @@ uint64_t BlockchainLMDB::get_block_height(const crypto::hash& h) const const mdb_txn_cursors *m_cursors = m_write_txn ? &m_wcursors : &m_tinfo->m_ti_rcursors; RCURSOR(block_heights); - MDB_val_copy key(h); - MDB_val result; - auto get_result = mdb_cursor_get(m_cur_block_heights, &key, &result, MDB_SET); + blk_height bh = {h, 0}; + MDB_val key = {sizeof(bh), (void *)&bh}; + auto get_result = mdb_cursor_get(m_cur_block_heights, (MDB_val *)&zerokval, &key, MDB_GET_BOTH); if (get_result == MDB_NOTFOUND) throw1(BLOCK_DNE("Attempted to retrieve non-existent block height")); else if (get_result) throw0(DB_ERROR("Error attempting to retrieve a block height from the db")); - uint64_t ret = *(const uint64_t *)result.mv_data; + blk_height *bhp = (blk_height *)key.mv_data; + uint64_t ret = bhp->height; TXN_POSTFIX_RDONLY(); return ret; } From 8e9d8e33640f5ac653de9df834f149b98893f0b8 Mon Sep 17 00:00:00 2001 From: Howard Chu Date: Sun, 6 Mar 2016 07:08:22 +0000 Subject: [PATCH 14/23] Use DUPFIXED for tx_indices Small space savings, no measurable speedup --- src/blockchain_db/lmdb/db_lmdb.cpp | 110 ++++++++++++++++------------- 1 file changed, 60 insertions(+), 50 deletions(-) diff --git a/src/blockchain_db/lmdb/db_lmdb.cpp b/src/blockchain_db/lmdb/db_lmdb.cpp index e4ec1b2df..ea4a111d4 100644 --- a/src/blockchain_db/lmdb/db_lmdb.cpp +++ b/src/blockchain_db/lmdb/db_lmdb.cpp @@ -158,11 +158,6 @@ const char* const LMDB_PROPERTIES = "properties"; const char zerokey[8] = {0}; const MDB_val zerokval = { sizeof(zerokey), (void *)zerokey }; -typedef struct blk_height { - crypto::hash key; - uint64_t height; -} blk_height; - const std::string lmdb_error(const std::string& error_string, int mdb_res) { const std::string full_string = error_string + mdb_strerror(mdb_res); @@ -210,6 +205,16 @@ typedef struct mdb_block_info crypto::hash bi_hash; } mdb_block_info; +typedef struct blk_height { + crypto::hash key; + uint64_t height; +} blk_height; + +typedef struct txindex { + crypto::hash key; + tx_data_t data; +} txindex; + std::atomic mdb_txn_safe::num_active_txns{0}; std::atomic_flag mdb_txn_safe::creation_gate = ATOMIC_FLAG_INIT; @@ -618,23 +623,25 @@ uint64_t BlockchainLMDB::add_transaction_data(const crypto::hash& blk_hash, cons CURSOR(txs) CURSOR(tx_indices) + txindex ti = {tx_hash}; MDB_val_copy val_tx_index(tx_index); - MDB_val_copy val_h(tx_hash); - MDB_val unused; - result = mdb_cursor_get(m_cur_tx_indices, &val_h, &unused, MDB_SET); - if (result == 0) - throw1(TX_EXISTS(std::string("Attempting to add transaction that's already in the db (tx index ").append(boost::lexical_cast(*(const uint64_t*)unused.mv_data)).append(")").c_str())); - else if (result != MDB_NOTFOUND) + MDB_val val_h = {sizeof(ti), (void *)&ti}; + result = mdb_cursor_get(m_cur_tx_indices, (MDB_val *)&zerokval, &val_h, MDB_GET_BOTH); + if (result == 0) { + txindex *tip = (txindex *)val_h.mv_data; + throw1(TX_EXISTS(std::string("Attempting to add transaction that's already in the db (tx index ").append(boost::lexical_cast(tip->data.tx_index)).append(")").c_str())); + } else if (result != MDB_NOTFOUND) { throw1(DB_ERROR(lmdb_error(std::string("Error checking if tx index exists for tx hash ") + epee::string_tools::pod_to_hex(tx_hash) + ": ", result).c_str())); + } - tx_data_t td; - td.tx_index = tx_index; - td.unlock_time = tx.unlock_time; - td.height = m_height; + ti.data.tx_index = tx_index; + ti.data.unlock_time = tx.unlock_time; + ti.data.height = m_height; - MDB_val_copy tx_data(td); + val_h.mv_size = sizeof(ti); + val_h.mv_data = (void *)&ti; - result = mdb_cursor_put(m_cur_tx_indices, &val_h, &tx_data, 0); + result = mdb_cursor_put(m_cur_tx_indices, (MDB_val *)&zerokval, &val_h, 0); if (result) throw0(DB_ERROR(lmdb_error("Failed to add tx data to db transaction: ", result).c_str())); @@ -654,13 +661,17 @@ void BlockchainLMDB::remove_transaction_data(const crypto::hash& tx_hash, const LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); - MDB_val_copy val_h(tx_hash); + mdb_txn_cursors *m_cursors = &m_wcursors; + CURSOR(tx_indices) + + txindex ti = {tx_hash}; + MDB_val val_h = {sizeof(ti), (void *)&ti}; MDB_val v; - if (mdb_get(*m_write_txn, m_tx_indices, &val_h, &v)) + if (mdb_cursor_get(m_cur_tx_indices, (MDB_val *)&zerokval, &val_h, MDB_GET_BOTH)) throw1(TX_DNE("Attempting to remove transaction that isn't in the db")); - tx_data_t td = *(const tx_data_t*)v.mv_data; - uint64_t tx_index = td.tx_index; + txindex *tip = (txindex *)val_h.mv_data; + uint64_t tx_index = tip->data.tx_index; MDB_val_copy val_tx_index(tx_index); if (mdb_del(*m_write_txn, m_txs, &val_tx_index, NULL)) @@ -677,7 +688,9 @@ void BlockchainLMDB::remove_transaction_data(const crypto::hash& tx_hash, const // Though other things could change, so long as earlier functions (like // remove_tx_outputs) need to do the lookup of tx hash -> tx index, don't // delete the tx_indices entry until the end. - if (mdb_del(*m_write_txn, m_tx_indices, &val_h, NULL)) + if (mdb_cursor_get(m_cur_tx_indices, (MDB_val *)&zerokval, &val_h, MDB_GET_BOTH)) + throw1(TX_DNE("Attempting to remove transaction that isn't in the db")); + if (mdb_cursor_del(m_cur_tx_indices, 0)) throw1(DB_ERROR("Failed to add removal of tx index to db transaction")); m_num_txs--; @@ -1078,7 +1091,7 @@ void BlockchainLMDB::open(const std::string& filename, const int mdb_flags) lmdb_db_open(txn, LMDB_BLOCK_HEIGHTS, MDB_CREATE | MDB_DUPSORT | MDB_DUPFIXED, m_block_heights, "Failed to open db handle for m_block_heights"); lmdb_db_open(txn, LMDB_TXS, MDB_INTEGERKEY | MDB_CREATE, m_txs, "Failed to open db handle for m_txs"); - lmdb_db_open(txn, LMDB_TX_INDICES, MDB_CREATE, m_tx_indices, "Failed to open db handle for m_tx_indices"); + lmdb_db_open(txn, LMDB_TX_INDICES, MDB_CREATE | MDB_DUPSORT | MDB_DUPFIXED, m_tx_indices, "Failed to open db handle for m_tx_indices"); lmdb_db_open(txn, LMDB_TX_OUTPUTS, MDB_INTEGERKEY | MDB_CREATE, m_tx_outputs, "Failed to open db handle for m_tx_outputs"); lmdb_db_open(txn, LMDB_OUTPUT_TXS, MDB_INTEGERKEY | MDB_CREATE, m_output_txs, "Failed to open db handle for m_output_txs"); @@ -1095,7 +1108,7 @@ void BlockchainLMDB::open(const std::string& filename, const int mdb_flags) mdb_set_dupsort(txn, m_spent_keys, compare_hash32); mdb_set_dupsort(txn, m_block_heights, compare_hash32); - mdb_set_compare(txn, m_tx_indices, compare_hash32); + mdb_set_dupsort(txn, m_tx_indices, compare_hash32); mdb_set_compare(txn, m_hf_starting_heights, compare_uint8); mdb_set_compare(txn, m_properties, compare_string); @@ -1669,11 +1682,10 @@ bool BlockchainLMDB::tx_exists(const crypto::hash& h) const RCURSOR(txs); MDB_val_copy key(h); - MDB_val v; bool tx_found = false; TIME_MEASURE_START(time1); - auto get_result = mdb_cursor_get(m_cur_tx_indices, &key, &v, MDB_SET); + auto get_result = mdb_cursor_get(m_cur_tx_indices, (MDB_val *)&zerokval, &key, MDB_GET_BOTH); if (get_result == 0) tx_found = true; else if (get_result != MDB_NOTFOUND) @@ -1709,13 +1721,17 @@ bool BlockchainLMDB::tx_exists(const crypto::hash& h, uint64_t& tx_index) const const mdb_txn_cursors *m_cursors = m_write_txn ? &m_wcursors : &m_tinfo->m_ti_rcursors; RCURSOR(tx_indices); - MDB_val_copy key(h); - MDB_val v; + txindex ti = {h}; + MDB_val v = {sizeof(ti), (void *)&ti}; TIME_MEASURE_START(time1); - auto get_result = mdb_cursor_get(m_cur_tx_indices, &key, &v, MDB_SET); + auto get_result = mdb_cursor_get(m_cur_tx_indices, (MDB_val *)&zerokval, &v, MDB_GET_BOTH); TIME_MEASURE_FINISH(time1); time_tx_exists += time1; + if (!get_result) { + txindex *tip = (txindex *)v.mv_data; + tx_index = tip->data.tx_index; + } TXN_POSTFIX_RDONLY(); @@ -1726,11 +1742,6 @@ bool BlockchainLMDB::tx_exists(const crypto::hash& h, uint64_t& tx_index) const } else if (get_result) throw0(DB_ERROR("DB error attempting to fetch transaction from hash")); - else - { - tx_data_t td = *(const tx_data_t*)v.mv_data; - tx_index = td.tx_index; - } return true; } @@ -1743,16 +1754,16 @@ uint64_t BlockchainLMDB::get_tx_unlock_time(const crypto::hash& h) const const mdb_txn_cursors *m_cursors = m_write_txn ? &m_wcursors : &m_tinfo->m_ti_rcursors; RCURSOR(tx_indices); - MDB_val_copy key(h); - MDB_val v; - auto get_result = mdb_cursor_get(m_cur_tx_indices, &key, &v, MDB_SET); + txindex ti = {h}; + MDB_val v = {sizeof(ti), (void *)&ti}; + auto get_result = mdb_cursor_get(m_cur_tx_indices, (MDB_val *)&zerokval, &v, MDB_GET_BOTH); if (get_result == MDB_NOTFOUND) throw1(TX_DNE(lmdb_error(std::string("tx data with hash ") + epee::string_tools::pod_to_hex(h) + " not found in db: ", get_result).c_str())); else if (get_result) throw0(DB_ERROR(lmdb_error("DB error attempting to fetch tx data from hash: ", get_result).c_str())); - tx_data_t td = *(const tx_data_t*)v.mv_data; - uint64_t ret = td.unlock_time; + txindex *tip = (txindex *)v.mv_data; + uint64_t ret = tip->data.unlock_time; TXN_POSTFIX_RDONLY(); return ret; } @@ -1767,14 +1778,14 @@ transaction BlockchainLMDB::get_tx(const crypto::hash& h) const RCURSOR(tx_indices); RCURSOR(txs); - MDB_val_copy key(h); - MDB_val v; + txindex ti = {h}; + MDB_val v = {sizeof(ti), (void *)&ti}; MDB_val result; - auto get_result = mdb_cursor_get(m_cur_tx_indices, &key, &v, MDB_SET); + auto get_result = mdb_cursor_get(m_cur_tx_indices, (MDB_val *)&zerokval, &v, MDB_GET_BOTH); if (get_result == 0) { - tx_data_t td = *(const tx_data_t*)v.mv_data; - uint64_t tx_index = td.tx_index; + txindex *tip = (txindex *)v.mv_data; + uint64_t tx_index = tip->data.tx_index; MDB_val_copy val_tx_index(tx_index); get_result = mdb_cursor_get(m_cur_txs, &val_tx_index, &result, MDB_SET); } @@ -1834,9 +1845,9 @@ uint64_t BlockchainLMDB::get_tx_block_height(const crypto::hash& h) const const mdb_txn_cursors *m_cursors = m_write_txn ? &m_wcursors : &m_tinfo->m_ti_rcursors; RCURSOR(tx_indices); - MDB_val_copy key(h); - MDB_val v; - auto get_result = mdb_cursor_get(m_cur_tx_indices, &key, &v, MDB_SET); + txindex ti = {h}; + MDB_val v = {sizeof(ti), (void *)&ti}; + auto get_result = mdb_cursor_get(m_cur_tx_indices, (MDB_val *)&zerokval, &v, MDB_GET_BOTH); if (get_result == MDB_NOTFOUND) { throw1(TX_DNE(std::string("tx_data_t with hash ").append(epee::string_tools::pod_to_hex(h)).append(" not found in db").c_str())); @@ -1844,8 +1855,8 @@ uint64_t BlockchainLMDB::get_tx_block_height(const crypto::hash& h) const else if (get_result) throw0(DB_ERROR("DB error attempting to fetch tx height from hash")); - tx_data_t res = *(const tx_data_t *)v.mv_data; - uint64_t ret = res.height; + txindex *tip = (txindex *)v.mv_data; + uint64_t ret = tip->data.height; TXN_POSTFIX_RDONLY(); return ret; } @@ -1969,7 +1980,6 @@ void BlockchainLMDB::get_amount_and_global_output_indices(const uint64_t tx_inde // create a new read-only txn here, which is incorrect. TXN_PREFIX_RDONLY(); const mdb_txn_cursors *m_cursors = m_write_txn ? &m_wcursors : &m_tinfo->m_ti_rcursors; - RCURSOR(tx_indices); RCURSOR(tx_outputs); int result = 0; From 7c5abdc3a3059c4f013bfa63a897171540b34ee5 Mon Sep 17 00:00:00 2001 From: Howard Chu Date: Sun, 6 Mar 2016 09:18:51 +0000 Subject: [PATCH 15/23] Use DUPFIXED for output_keys Saves another 90MB on 200000 block import. Had to bring back compare_uint64 for this, but it's safe since this table is always 64-bit aligned. --- src/blockchain_db/lmdb/db_lmdb.cpp | 60 ++++++++++++++++++------------ 1 file changed, 37 insertions(+), 23 deletions(-) diff --git a/src/blockchain_db/lmdb/db_lmdb.cpp b/src/blockchain_db/lmdb/db_lmdb.cpp index ea4a111d4..cd35286c5 100644 --- a/src/blockchain_db/lmdb/db_lmdb.cpp +++ b/src/blockchain_db/lmdb/db_lmdb.cpp @@ -108,6 +108,13 @@ private: std::unique_ptr data; }; +int compare_uint64(const MDB_val *a, const MDB_val *b) +{ + const uint64_t va = *(const uint64_t *)a->mv_data; + const uint64_t vb = *(const uint64_t *)b->mv_data; + return (va < vb) ? -1 : va > vb; +} + int compare_uint8(const MDB_val *a, const MDB_val *b) { const uint8_t va = *(const uint8_t*)a->mv_data; @@ -215,6 +222,11 @@ typedef struct txindex { tx_data_t data; } txindex; +typedef struct outkey { + uint64_t num_outputs; + output_data_t data; +} outkey; + std::atomic mdb_txn_safe::num_active_txns{0}; std::atomic_flag mdb_txn_safe::creation_gate = ATOMIC_FLAG_INIT; @@ -741,14 +753,14 @@ void BlockchainLMDB::add_output(const crypto::hash& tx_hash, if (tx_output.target.type() == typeid(txout_to_key)) { - output_data_t od; - od.pubkey = boost::get < txout_to_key > (tx_output.target).key; - od.unlock_time = unlock_time; - od.height = m_height; + outkey ok; + ok.num_outputs = m_num_outputs; + ok.data.pubkey = boost::get < txout_to_key > (tx_output.target).key; + ok.data.unlock_time = unlock_time; + ok.data.height = m_height; - MDB_val_copy data(od); - //MDB_val_copy val_pubkey(boost::get(tx_output.target).key); - if (mdb_cursor_put(m_cur_output_keys, &k, &data, MDB_APPEND)) + MDB_val data = {sizeof(ok), (void *)&ok}; + if (mdb_cursor_put(m_cur_output_keys, (MDB_val *)&zerokval, &data, MDB_APPENDDUP)) throw0(DB_ERROR("Failed to add output pubkey to db transaction")); } else @@ -848,7 +860,7 @@ void BlockchainLMDB::remove_output(const uint64_t& out_index, const uint64_t amo throw1(DB_ERROR("Error adding removal of output tx hash to db transaction")); } - result = mdb_del(*m_write_txn, m_output_keys, &k, NULL); + result = mdb_del(*m_write_txn, m_output_keys, (MDB_val *)&zerokval, &k); if (result == MDB_NOTFOUND) { LOG_PRINT_L0("Unexpected: global output index not found in m_output_keys"); @@ -1088,18 +1100,18 @@ void BlockchainLMDB::open(const std::string& filename, const int mdb_flags) lmdb_db_open(txn, LMDB_BLOCKS, MDB_INTEGERKEY | MDB_CREATE, m_blocks, "Failed to open db handle for m_blocks"); lmdb_db_open(txn, LMDB_BLOCK_INFO, MDB_INTEGERKEY | MDB_CREATE, m_block_info, "Failed to open db handle for m_block_info"); - lmdb_db_open(txn, LMDB_BLOCK_HEIGHTS, MDB_CREATE | MDB_DUPSORT | MDB_DUPFIXED, m_block_heights, "Failed to open db handle for m_block_heights"); + lmdb_db_open(txn, LMDB_BLOCK_HEIGHTS, MDB_INTEGERKEY | MDB_CREATE | MDB_DUPSORT | MDB_DUPFIXED, m_block_heights, "Failed to open db handle for m_block_heights"); lmdb_db_open(txn, LMDB_TXS, MDB_INTEGERKEY | MDB_CREATE, m_txs, "Failed to open db handle for m_txs"); - lmdb_db_open(txn, LMDB_TX_INDICES, MDB_CREATE | MDB_DUPSORT | MDB_DUPFIXED, m_tx_indices, "Failed to open db handle for m_tx_indices"); + lmdb_db_open(txn, LMDB_TX_INDICES, MDB_INTEGERKEY | MDB_CREATE | MDB_DUPSORT | MDB_DUPFIXED, m_tx_indices, "Failed to open db handle for m_tx_indices"); lmdb_db_open(txn, LMDB_TX_OUTPUTS, MDB_INTEGERKEY | MDB_CREATE, m_tx_outputs, "Failed to open db handle for m_tx_outputs"); lmdb_db_open(txn, LMDB_OUTPUT_TXS, MDB_INTEGERKEY | MDB_CREATE, m_output_txs, "Failed to open db handle for m_output_txs"); lmdb_db_open(txn, LMDB_OUTPUT_INDICES, MDB_INTEGERKEY | MDB_CREATE, m_output_indices, "Failed to open db handle for m_output_indices"); lmdb_db_open(txn, LMDB_OUTPUT_AMOUNTS, MDB_INTEGERKEY | MDB_INTEGERDUP | MDB_DUPSORT | MDB_DUPFIXED | MDB_CREATE, m_output_amounts, "Failed to open db handle for m_output_amounts"); - lmdb_db_open(txn, LMDB_OUTPUT_KEYS, MDB_INTEGERKEY | MDB_CREATE, m_output_keys, "Failed to open db handle for m_output_keys"); + lmdb_db_open(txn, LMDB_OUTPUT_KEYS, MDB_INTEGERKEY | MDB_CREATE | MDB_DUPSORT | MDB_DUPFIXED, m_output_keys, "Failed to open db handle for m_output_keys"); - lmdb_db_open(txn, LMDB_SPENT_KEYS, MDB_CREATE | MDB_DUPSORT | MDB_DUPFIXED, m_spent_keys, "Failed to open db handle for m_spent_keys"); + lmdb_db_open(txn, LMDB_SPENT_KEYS, MDB_INTEGERKEY | MDB_CREATE | MDB_DUPSORT | MDB_DUPFIXED, m_spent_keys, "Failed to open db handle for m_spent_keys"); lmdb_db_open(txn, LMDB_HF_STARTING_HEIGHTS, MDB_CREATE, m_hf_starting_heights, "Failed to open db handle for m_hf_starting_heights"); lmdb_db_open(txn, LMDB_HF_VERSIONS, MDB_INTEGERKEY | MDB_CREATE, m_hf_versions, "Failed to open db handle for m_hf_versions"); @@ -1109,6 +1121,7 @@ void BlockchainLMDB::open(const std::string& filename, const int mdb_flags) mdb_set_dupsort(txn, m_spent_keys, compare_hash32); mdb_set_dupsort(txn, m_block_heights, compare_hash32); mdb_set_dupsort(txn, m_tx_indices, compare_hash32); + mdb_set_dupsort(txn, m_output_keys, compare_uint64); mdb_set_compare(txn, m_hf_starting_heights, compare_uint8); mdb_set_compare(txn, m_properties, compare_string); @@ -1138,9 +1151,8 @@ void BlockchainLMDB::open(const std::string& filename, const int mdb_flags) // terminate. if(m_height > 0) { - MDB_val_copy k(0); MDB_val v; - auto get_result = mdb_get(txn, m_output_keys, &k, &v); + auto get_result = mdb_get(txn, m_output_keys, (MDB_val *)&zerokval, &v); if(get_result != MDB_SUCCESS) { txn.abort(); @@ -1149,7 +1161,7 @@ void BlockchainLMDB::open(const std::string& filename, const int mdb_flags) } // LOG_PRINT_L0("Output keys size: " << v.mv_size); - if(v.mv_size != sizeof(output_data_t)) + if(v.mv_size != sizeof(outkey)) compatible = false; } @@ -1898,14 +1910,15 @@ output_data_t BlockchainLMDB::get_output_key(const uint64_t &global_index) const const mdb_txn_cursors *m_cursors = m_write_txn ? &m_wcursors : &m_tinfo->m_ti_rcursors; RCURSOR(output_keys); - MDB_val_copy k(global_index); - MDB_val v; - auto get_result = mdb_cursor_get(m_cur_output_keys, &k, &v, MDB_SET); + outkey ok = {global_index}; + MDB_val v = {sizeof(ok), (void *)&ok}; + auto get_result = mdb_cursor_get(m_cur_output_keys, (MDB_val *)&zerokval, &v, MDB_GET_BOTH); if (get_result == MDB_NOTFOUND) throw1(OUTPUT_DNE("Attempting to get output pubkey by global index, but key does not exist")); else if (get_result) throw0(DB_ERROR("Error attempting to retrieve an output pubkey from the db")); - output_data_t ret = *(const output_data_t *) v.mv_data; + outkey *okp = (outkey *)v.mv_data; + output_data_t ret = okp->data; TXN_POSTFIX_RDONLY(); return ret; } @@ -2684,16 +2697,17 @@ void BlockchainLMDB::get_output_key(const uint64_t &amount, const std::vector k(index); - MDB_val v; + outkey ok = {index}; + MDB_val v = {sizeof(ok), (void *)&ok}; - auto get_result = mdb_cursor_get(m_cur_output_keys, &k, &v, MDB_SET); + auto get_result = mdb_cursor_get(m_cur_output_keys, (MDB_val *)&zerokval, &v, MDB_GET_BOTH); if (get_result == MDB_NOTFOUND) throw1(OUTPUT_DNE("Attempting to get output pubkey by global index, but key does not exist")); else if (get_result) throw0(DB_ERROR("Error attempting to retrieve an output pubkey from the db")); - output_data_t data = *(const output_data_t *) v.mv_data; + outkey *okp = (outkey *)v.mv_data; + output_data_t data = okp->data; outputs.push_back(data); } From 6225716f3c727aedc8365aeca0231a720599b4c1 Mon Sep 17 00:00:00 2001 From: Howard Chu Date: Sun, 27 Mar 2016 22:43:16 +0100 Subject: [PATCH 16/23] More outputs consolidation Also bumped DB VERSION to 1 Another significant speedup and space savings: Get rid of global_output_indices, remove indirection from output to keys This is the change warptangent described on irc but never got to finish. --- external/db_drivers/liblmdb/mdb.c | 2 +- src/blockchain_db/lmdb/db_lmdb.cpp | 736 ++++++++++------------------- src/blockchain_db/lmdb/db_lmdb.h | 26 +- 3 files changed, 268 insertions(+), 496 deletions(-) diff --git a/external/db_drivers/liblmdb/mdb.c b/external/db_drivers/liblmdb/mdb.c index a366b4024..acab07b82 100644 --- a/external/db_drivers/liblmdb/mdb.c +++ b/external/db_drivers/liblmdb/mdb.c @@ -6771,8 +6771,8 @@ set1: if (op == MDB_GET_BOTH || rc > 0) return MDB_NOTFOUND; rc = 0; - *data = olddata; } + *data = olddata; } else { if (mc->mc_xcursor) diff --git a/src/blockchain_db/lmdb/db_lmdb.cpp b/src/blockchain_db/lmdb/db_lmdb.cpp index cd35286c5..604fde0aa 100644 --- a/src/blockchain_db/lmdb/db_lmdb.cpp +++ b/src/blockchain_db/lmdb/db_lmdb.cpp @@ -46,7 +46,7 @@ using epee::string_tools::pod_to_hex; // Increase when the DB changes in a non backward compatible way, and there // is no automatic conversion, so that a full resync is needed. -#define VERSION 0 +#define VERSION 1 namespace { @@ -65,6 +65,8 @@ inline void throw1(const T &e) throw e; } +#define MDB_val_set(var, val) MDB_val var = {sizeof(val), (void *)&val} + template struct MDB_val_copy: public MDB_val { @@ -152,9 +154,7 @@ const char* const LMDB_TX_INDICES = "tx_indices"; const char* const LMDB_TX_OUTPUTS = "tx_outputs"; const char* const LMDB_OUTPUT_TXS = "output_txs"; -const char* const LMDB_OUTPUT_INDICES = "output_indices"; const char* const LMDB_OUTPUT_AMOUNTS = "output_amounts"; -const char* const LMDB_OUTPUT_KEYS = "output_keys"; const char* const LMDB_SPENT_KEYS = "spent_keys"; const char* const LMDB_HF_STARTING_HEIGHTS = "hf_starting_heights"; @@ -192,10 +192,12 @@ inline void lmdb_db_open(MDB_txn* txn, const char* name, int flags, MDB_dbi& dbi int result = mdb_cursor_open(m_txn, m_ ## name, (MDB_cursor **)&m_cur_ ## name); \ if (result) \ throw0(DB_ERROR(lmdb_error("Failed to open cursor: ", result).c_str())); \ - if (!m_write_txn) \ + if (m_cursors != &m_wcursors) \ m_tinfo->m_ti_rflags.m_rf_ ## name = true; \ - } else if (!m_write_txn && !m_tinfo->m_ti_rflags.m_rf_ ## name) { \ - mdb_cursor_renew(m_txn, m_cur_ ## name); \ + } else if (m_cursors != &m_wcursors && !m_tinfo->m_ti_rflags.m_rf_ ## name) { \ + int result = mdb_cursor_renew(m_txn, m_cur_ ## name); \ + if (result) \ + throw0(DB_ERROR(lmdb_error("Failed to renew cursor: ", result).c_str())); \ m_tinfo->m_ti_rflags.m_rf_ ## name = true; \ } @@ -223,10 +225,16 @@ typedef struct txindex { } txindex; typedef struct outkey { - uint64_t num_outputs; + uint64_t amount_index; + uint64_t tx_index; output_data_t data; } outkey; +typedef struct outtx { + crypto::hash tx_hash; + uint64_t local_index; +} outtx; + std::atomic mdb_txn_safe::num_active_txns{0}; std::atomic_flag mdb_txn_safe::creation_gate = ATOMIC_FLAG_INIT; @@ -241,17 +249,26 @@ mdb_threadinfo::~mdb_threadinfo() mdb_txn_abort(m_ti_rtxn); } -mdb_txn_safe::mdb_txn_safe() : m_txn(NULL) +mdb_txn_safe::mdb_txn_safe(const bool check) : m_txn(NULL), m_tinfo(NULL), m_check(check) { - while (creation_gate.test_and_set()); - num_active_txns++; - creation_gate.clear(); + if (check) + { + while (creation_gate.test_and_set()); + num_active_txns++; + creation_gate.clear(); + } } mdb_txn_safe::~mdb_txn_safe() { + if (!m_check) + return; LOG_PRINT_L3("mdb_txn_safe: destructor"); - if (m_txn != nullptr) + if (m_tinfo != nullptr) + { + mdb_txn_reset(m_tinfo->m_ti_rtxn); + memset(&m_tinfo->m_ti_rflags, 0, sizeof(m_tinfo->m_ti_rflags)); + } else if (m_txn != nullptr) { if (m_batch_txn) // this is a batch txn and should have been handled before this point for safety { @@ -550,11 +567,12 @@ void BlockchainLMDB::add_block(const block& blk, const size_t& block_size, const { blk_height ph = {blk.prev_id, 0}; MDB_val parent_key = {sizeof(ph), (void *)&ph}; - if (mdb_cursor_get(m_cur_block_heights, (MDB_val *)&zerokval, &parent_key, MDB_GET_BOTH)) + int result = mdb_cursor_get(m_cur_block_heights, (MDB_val *)&zerokval, &parent_key, MDB_GET_BOTH); + if (result) { LOG_PRINT_L3("m_height: " << m_height); LOG_PRINT_L3("parent_key: " << blk.prev_id); - throw0(DB_ERROR("Failed to get top block hash to check for new block's parent")); + throw0(DB_ERROR(lmdb_error("Failed to get top block hash to check for new block's parent: ", result).c_str())); } blk_height *prev = (blk_height *)parent_key.mv_data; if (prev->height != m_height - 1) @@ -595,6 +613,8 @@ void BlockchainLMDB::add_block(const block& blk, const size_t& block_size, const void BlockchainLMDB::remove_block() { + int result; + LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); @@ -605,22 +625,22 @@ void BlockchainLMDB::remove_block() CURSOR(block_info) MDB_val_copy k(m_height - 1); MDB_val h; - if (mdb_cursor_get(m_cur_block_info, &k, &h, MDB_SET)) - throw1(BLOCK_DNE("Attempting to remove block that's not in the db")); + if ((result = mdb_cursor_get(m_cur_block_info, &k, &h, MDB_SET))) + throw1(BLOCK_DNE(lmdb_error("Attempting to remove block that's not in the db: ", result).c_str())); // must use h now; deleting from m_block_info will invalidate it mdb_block_info *bi = (mdb_block_info *)h.mv_data; blk_height bh = {bi->bi_hash, 0}; h.mv_data = (void *)&bh; h.mv_size = sizeof(bh); - if (mdb_del(*m_write_txn, m_block_heights, (MDB_val *)&zerokval, &h)) - throw1(DB_ERROR("Failed to add removal of block height by hash to db transaction")); + if ((result = mdb_del(*m_write_txn, m_block_heights, (MDB_val *)&zerokval, &h))) + throw1(DB_ERROR(lmdb_error("Failed to add removal of block height by hash to db transaction: ", result).c_str())); - if (mdb_del(*m_write_txn, m_blocks, &k, NULL)) - throw1(DB_ERROR("Failed to add removal of block to db transaction")); + if ((result = mdb_del(*m_write_txn, m_blocks, &k, NULL))) + throw1(DB_ERROR(lmdb_error("Failed to add removal of block to db transaction: ", result).c_str())); - if (mdb_cursor_del(m_cur_block_info, 0)) - throw1(DB_ERROR("Failed to add removal of block info to db transaction")); + if ((result = mdb_cursor_del(m_cur_block_info, 0))) + throw1(DB_ERROR(lmdb_error("Failed to add removal of block info to db transaction: ", result).c_str())); } uint64_t BlockchainLMDB::add_transaction_data(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash& tx_hash) @@ -670,6 +690,8 @@ uint64_t BlockchainLMDB::add_transaction_data(const crypto::hash& blk_hash, cons // passing it in to functions like this void BlockchainLMDB::remove_transaction_data(const crypto::hash& tx_hash, const transaction& tx) { + int result; + LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); @@ -678,7 +700,6 @@ void BlockchainLMDB::remove_transaction_data(const crypto::hash& tx_hash, const txindex ti = {tx_hash}; MDB_val val_h = {sizeof(ti), (void *)&ti}; - MDB_val v; if (mdb_cursor_get(m_cur_tx_indices, (MDB_val *)&zerokval, &val_h, MDB_GET_BOTH)) throw1(TX_DNE("Attempting to remove transaction that isn't in the db")); @@ -686,12 +707,12 @@ void BlockchainLMDB::remove_transaction_data(const crypto::hash& tx_hash, const uint64_t tx_index = tip->data.tx_index; MDB_val_copy val_tx_index(tx_index); - if (mdb_del(*m_write_txn, m_txs, &val_tx_index, NULL)) - throw1(DB_ERROR("Failed to add removal of tx to db transaction")); + if ((result = mdb_del(*m_write_txn, m_txs, &val_tx_index, NULL))) + throw1(DB_ERROR(lmdb_error("Failed to add removal of tx to db transaction: ", result).c_str())); remove_tx_outputs(tx_index, tx); - int result = mdb_del(*m_write_txn, m_tx_outputs, &val_tx_index, NULL); + result = mdb_del(*m_write_txn, m_tx_outputs, &val_tx_index, NULL); if (result == MDB_NOTFOUND) LOG_PRINT_L1("tx has no outputs to remove: " << tx_hash); else if (result) @@ -708,6 +729,7 @@ void BlockchainLMDB::remove_transaction_data(const crypto::hash& tx_hash, const m_num_txs--; } +// global_output_index is no longer used for locating outputs void BlockchainLMDB::add_output(const crypto::hash& tx_hash, const tx_out& tx_output, const uint64_t& local_index, @@ -722,55 +744,53 @@ void BlockchainLMDB::add_output(const crypto::hash& tx_hash, int result = 0; CURSOR(output_txs) - CURSOR(output_indices) CURSOR(output_amounts) - CURSOR(output_keys) - MDB_val_copy k(m_num_outputs); - MDB_val_copy v(tx_hash); + if (tx_output.target.type() != typeid(txout_to_key)) + throw0(DB_ERROR("Wrong output type: expected txout_to_key")); - result = mdb_cursor_put(m_cur_output_txs, &k, &v, MDB_APPEND); + MDB_val_set(k, m_num_outputs); + + outtx ot = {tx_hash, local_index}; + MDB_val_set(vot, ot); + + result = mdb_cursor_put(m_cur_output_txs, &k, &vot, MDB_APPEND); if (result) throw0(DB_ERROR(lmdb_error("Failed to add output tx hash to db transaction: ", result).c_str())); - MDB_val_copy val_local_index(local_index); - result = mdb_cursor_put(m_cur_output_indices, &k, &val_local_index, MDB_APPEND); - if (result) - throw0(DB_ERROR(lmdb_error("Failed to add tx output index to db transaction: ", result).c_str())); - + outkey ok; + MDB_val data; MDB_val_copy val_amount(tx_output.amount); - result = mdb_cursor_put(m_cur_output_amounts, &val_amount, &k, 0); - if (result) - throw0(DB_ERROR(lmdb_error("Failed to add output amount to db transaction: ", result).c_str())); - - size_t num_elems = 0; - result = mdb_cursor_count(m_cur_output_amounts, &num_elems); - if (result) - throw0(DB_ERROR(std::string("Failed to get number of outputs for amount: ").append(mdb_strerror(result)).c_str())); - - amount_output_index = num_elems - 1; - global_output_index = m_num_outputs; - - if (tx_output.target.type() == typeid(txout_to_key)) - { - outkey ok; - ok.num_outputs = m_num_outputs; - ok.data.pubkey = boost::get < txout_to_key > (tx_output.target).key; - ok.data.unlock_time = unlock_time; - ok.data.height = m_height; - - MDB_val data = {sizeof(ok), (void *)&ok}; - if (mdb_cursor_put(m_cur_output_keys, (MDB_val *)&zerokval, &data, MDB_APPENDDUP)) - throw0(DB_ERROR("Failed to add output pubkey to db transaction")); - } + result = mdb_cursor_get(m_cur_output_amounts, &val_amount, &data, MDB_SET); + if (!result) + { + mdb_size_t num_elems = 0; + result = mdb_cursor_count(m_cur_output_amounts, &num_elems); + if (result) + throw0(DB_ERROR(std::string("Failed to get number of outputs for amount: ").append(mdb_strerror(result)).c_str())); + ok.amount_index = num_elems; + } + else if (result != MDB_NOTFOUND) + throw0(DB_ERROR(lmdb_error("Failed to get output amount in db transaction: ", result).c_str())); else - { - throw0(DB_ERROR("Wrong output type: expected txout_to_key")); - } + ok.amount_index = 0; + ok.tx_index = m_num_outputs; + ok.data.pubkey = boost::get < txout_to_key > (tx_output.target).key; + ok.data.unlock_time = unlock_time; + ok.data.height = m_height; + data.mv_data = &ok; + data.mv_size = sizeof(ok); + + if ((result = mdb_cursor_put(m_cur_output_amounts, &val_amount, &data, MDB_APPENDDUP))) + throw0(DB_ERROR(lmdb_error("Failed to add output pubkey to db transaction: ", result).c_str())); + + amount_output_index = ok.amount_index; + global_output_index = m_num_outputs; m_num_outputs++; } +// global_output_indices is now ignored void BlockchainLMDB::add_amount_and_global_output_indices(const uint64_t tx_index, const std::vector& amount_output_indices, const std::vector& global_output_indices) @@ -783,17 +803,11 @@ void BlockchainLMDB::add_amount_and_global_output_indices(const uint64_t tx_inde int result = 0; int num_outputs = amount_output_indices.size(); - std::unique_ptr paired_indices(new uint64_t[2*num_outputs]); - for (int i = 0; i < num_outputs; ++i) - { - paired_indices[2*i] = amount_output_indices[i]; - paired_indices[2*i+1] = global_output_indices[i]; - } MDB_val_copy k_tx_index(tx_index); MDB_val v; - v.mv_data = (void*)paired_indices.get(); - v.mv_size = sizeof(uint64_t) * 2 * num_outputs; + v.mv_data = (void *)amount_output_indices.data(); + v.mv_size = sizeof(uint64_t) * num_outputs; // LOG_PRINT_L1("tx_outputs[tx_hash] size: " << v.mv_size); result = mdb_cursor_put(m_cur_tx_outputs, &k_tx_index, &v, MDB_APPEND); @@ -805,22 +819,22 @@ void BlockchainLMDB::remove_tx_outputs(const uint64_t tx_index, const transactio { LOG_PRINT_L3("BlockchainLMDB::" << __func__); - // only need global_output_indices + // only need amount_output_indices std::vector amount_output_indices, global_output_indices; get_amount_and_global_output_indices(tx_index, amount_output_indices, global_output_indices); - if (global_output_indices.empty()) + if (amount_output_indices.empty()) { if (tx.vout.empty()) - LOG_PRINT_L2("tx has no outputs, so no global output indices"); + LOG_PRINT_L2("tx has no outputs, so no output indices"); else - throw0(DB_ERROR("tx has outputs, but no global output indices found")); + throw0(DB_ERROR("tx has outputs, but no output indices found")); } for (uint64_t i = tx.vout.size(); i > 0; --i) { const tx_out tx_output = tx.vout[i-1]; - remove_output(global_output_indices[i-1], tx_output.amount); + remove_output(amount_output_indices[i-1], tx_output.amount); } } @@ -838,91 +852,41 @@ void BlockchainLMDB::remove_output(const uint64_t& out_index, const uint64_t amo MDB_val_copy k(out_index); - auto result = mdb_del(*m_write_txn, m_output_indices, &k, NULL); - if (result == MDB_NOTFOUND) - { - LOG_PRINT_L0("Unexpected: global output index not found in m_output_indices"); - } - else if (result) - { - throw1(DB_ERROR("Error adding removal of output tx index to db transaction")); - } - - result = mdb_del(*m_write_txn, m_output_txs, &k, NULL); - // if (result != 0 && result != MDB_NOTFOUND) - // throw1(DB_ERROR("Error adding removal of output tx hash to db transaction")); + auto result = mdb_del(*m_write_txn, m_output_txs, &k, NULL); if (result == MDB_NOTFOUND) { LOG_PRINT_L0("Unexpected: global output index not found in m_output_txs"); } else if (result) { - throw1(DB_ERROR("Error adding removal of output tx hash to db transaction")); + throw1(DB_ERROR(lmdb_error("Error adding removal of output tx to db transaction", result).c_str())); } - result = mdb_del(*m_write_txn, m_output_keys, (MDB_val *)&zerokval, &k); - if (result == MDB_NOTFOUND) - { - LOG_PRINT_L0("Unexpected: global output index not found in m_output_keys"); - } - else if (result) - throw1(DB_ERROR("Error adding removal of output pubkey to db transaction")); - remove_amount_output_index(amount, out_index); m_num_outputs--; } -void BlockchainLMDB::remove_amount_output_index(const uint64_t amount, const uint64_t global_output_index) +void BlockchainLMDB::remove_amount_output_index(const uint64_t amount, const uint64_t output_index) { LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); mdb_txn_cursors *m_cursors = &m_wcursors; CURSOR(output_amounts); - MDB_val_copy k(amount); - MDB_val v; + MDB_val_set(k, amount); + MDB_val_set(v, output_index); - auto result = mdb_cursor_get(m_cur_output_amounts, &k, &v, MDB_SET); + auto result = mdb_cursor_get(m_cur_output_amounts, &k, &v, MDB_GET_BOTH); if (result == MDB_NOTFOUND) throw1(OUTPUT_DNE("Attempting to get an output index by amount and amount index, but amount not found")); else if (result) - throw0(DB_ERROR("DB error attempting to get an output")); + throw0(DB_ERROR(lmdb_error("DB error attempting to get an output", result).c_str())); - mdb_size_t num_elems = 0; - mdb_cursor_count(m_cur_output_amounts, &num_elems); - - mdb_cursor_get(m_cur_output_amounts, &k, &v, MDB_LAST_DUP); - - uint64_t amount_output_index = 0; - uint64_t goi = 0; - bool found_index = false; - for (uint64_t i = num_elems; i > 0; --i) - { - mdb_cursor_get(m_cur_output_amounts, &k, &v, MDB_GET_CURRENT); - goi = *(const uint64_t *)v.mv_data; - if (goi == global_output_index) - { - amount_output_index = i-1; - found_index = true; - break; - } - if (i > 1) - mdb_cursor_get(m_cur_output_amounts, &k, &v, MDB_PREV_DUP); - } - if (found_index) - { - // found the amount output index - // now delete it - result = mdb_cursor_del(m_cur_output_amounts, 0); - if (result) - throw0(DB_ERROR(std::string("Error deleting amount output index ").append(boost::lexical_cast(amount_output_index)).c_str())); - } - else - { - // not found - throw1(OUTPUT_DNE("Failed to find amount output index")); - } + // now delete it + result = mdb_cursor_del(m_cur_output_amounts, 0); + if (result) + throw0(DB_ERROR(lmdb_error(std::string("Error deleting amount for output index ").append(boost::lexical_cast(output_index).append(": ")).c_str(), result).c_str())); } void BlockchainLMDB::add_spent_key(const crypto::key_image& k_image) @@ -950,7 +914,7 @@ void BlockchainLMDB::remove_spent_key(const crypto::key_image& k_image) MDB_val k = {sizeof(k_image), (void *)&k_image}; auto result = mdb_del(*m_write_txn, m_spent_keys, (MDB_val *)&zerokval, &k); if (result != 0 && result != MDB_NOTFOUND) - throw1(DB_ERROR("Error adding removal of key image to db transaction")); + throw1(DB_ERROR(lmdb_error("Error adding removal of key image to db transaction", result).c_str())); } blobdata BlockchainLMDB::output_to_blob(const tx_out& output) const @@ -976,19 +940,6 @@ tx_out BlockchainLMDB::output_from_blob(const blobdata& blob) const return o; } -uint64_t BlockchainLMDB::get_output_global_index(const uint64_t& amount, const uint64_t& index) -{ - LOG_PRINT_L3("BlockchainLMDB::" << __func__); - std::vector offsets; - std::vector global_indices; - offsets.push_back(index); - get_output_global_indices(amount, offsets, global_indices); - if (!global_indices.size()) - throw1(OUTPUT_DNE("Attempting to get an output index by amount and amount index, but amount not found")); - - return global_indices[0]; -} - void BlockchainLMDB::check_open() const { LOG_PRINT_L3("BlockchainLMDB::" << __func__); @@ -1028,6 +979,8 @@ BlockchainLMDB::BlockchainLMDB(bool batch_transactions) void BlockchainLMDB::open(const std::string& filename, const int mdb_flags) { + int result; + LOG_PRINT_L3("BlockchainLMDB::" << __func__); if (m_open) @@ -1057,10 +1010,10 @@ void BlockchainLMDB::open(const std::string& filename, const int mdb_flags) m_folder = filename; // set up lmdb environment - if (mdb_env_create(&m_env)) - throw0(DB_ERROR("Failed to create lmdb environment")); - if (mdb_env_set_maxdbs(m_env, 20)) - throw0(DB_ERROR("Failed to set max number of dbs")); + if ((result = mdb_env_create(&m_env))) + throw0(DB_ERROR(lmdb_error("Failed to create lmdb environment: ", result).c_str())); + if ((result = mdb_env_set_maxdbs(m_env, 20))) + throw0(DB_ERROR(lmdb_error("Failed to set max number of dbs: ", result).c_str())); size_t mapsize = DEFAULT_MAPSIZE; @@ -1107,9 +1060,7 @@ void BlockchainLMDB::open(const std::string& filename, const int mdb_flags) lmdb_db_open(txn, LMDB_TX_OUTPUTS, MDB_INTEGERKEY | MDB_CREATE, m_tx_outputs, "Failed to open db handle for m_tx_outputs"); lmdb_db_open(txn, LMDB_OUTPUT_TXS, MDB_INTEGERKEY | MDB_CREATE, m_output_txs, "Failed to open db handle for m_output_txs"); - lmdb_db_open(txn, LMDB_OUTPUT_INDICES, MDB_INTEGERKEY | MDB_CREATE, m_output_indices, "Failed to open db handle for m_output_indices"); - lmdb_db_open(txn, LMDB_OUTPUT_AMOUNTS, MDB_INTEGERKEY | MDB_INTEGERDUP | MDB_DUPSORT | MDB_DUPFIXED | MDB_CREATE, m_output_amounts, "Failed to open db handle for m_output_amounts"); - lmdb_db_open(txn, LMDB_OUTPUT_KEYS, MDB_INTEGERKEY | MDB_CREATE | MDB_DUPSORT | MDB_DUPFIXED, m_output_keys, "Failed to open db handle for m_output_keys"); + lmdb_db_open(txn, LMDB_OUTPUT_AMOUNTS, MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED | MDB_CREATE, m_output_amounts, "Failed to open db handle for m_output_amounts"); lmdb_db_open(txn, LMDB_SPENT_KEYS, MDB_INTEGERKEY | MDB_CREATE | MDB_DUPSORT | MDB_DUPFIXED, m_spent_keys, "Failed to open db handle for m_spent_keys"); @@ -1121,50 +1072,30 @@ void BlockchainLMDB::open(const std::string& filename, const int mdb_flags) mdb_set_dupsort(txn, m_spent_keys, compare_hash32); mdb_set_dupsort(txn, m_block_heights, compare_hash32); mdb_set_dupsort(txn, m_tx_indices, compare_hash32); - mdb_set_dupsort(txn, m_output_keys, compare_uint64); + mdb_set_dupsort(txn, m_output_amounts, compare_uint64); mdb_set_compare(txn, m_hf_starting_heights, compare_uint8); mdb_set_compare(txn, m_properties, compare_string); // get and keep current height MDB_stat db_stats; - if (mdb_stat(txn, m_blocks, &db_stats)) - throw0(DB_ERROR("Failed to query m_blocks")); + if ((result = mdb_stat(txn, m_blocks, &db_stats))) + throw0(DB_ERROR(lmdb_error("Failed to query m_blocks: ", result).c_str())); LOG_PRINT_L2("Setting m_height to: " << db_stats.ms_entries); m_height = db_stats.ms_entries; // get and keep current number of txs - if (mdb_stat(txn, m_tx_indices, &db_stats)) - throw0(DB_ERROR("Failed to query m_tx_indices")); + if ((result = mdb_stat(txn, m_tx_indices, &db_stats))) + throw0(DB_ERROR(lmdb_error("Failed to query m_tx_indices: ", result).c_str())); m_num_txs = db_stats.ms_entries; // get and keep current number of outputs - if (mdb_stat(txn, m_output_indices, &db_stats)) - throw0(DB_ERROR("Failed to query m_output_indices")); + if ((result = mdb_stat(txn, m_output_txs, &db_stats))) + throw0(DB_ERROR(lmdb_error("Failed to query m_output_txs: ", result).c_str())); m_num_outputs = db_stats.ms_entries; bool compatible = true; - // ND: This "new" version of the lmdb database is incompatible with - // the previous version. Ensure that the output_keys database is - // sizeof(output_data_t) in length. Otherwise, inform user and - // terminate. - if(m_height > 0) - { - MDB_val v; - auto get_result = mdb_get(txn, m_output_keys, (MDB_val *)&zerokval, &v); - if(get_result != MDB_SUCCESS) - { - txn.abort(); - m_open = false; - return; - } - - // LOG_PRINT_L0("Output keys size: " << v.mv_size); - if(v.mv_size != sizeof(outkey)) - compatible = false; - } - MDB_val_copy k("version"); MDB_val v; auto get_result = mdb_get(txn, m_properties, &k, &v); @@ -1260,17 +1191,15 @@ void BlockchainLMDB::reset() check_open(); mdb_txn_safe txn; - if (mdb_txn_begin(m_env, NULL, 0, txn)) - throw0(DB_ERROR("Failed to create a transaction for the db")); + if (auto result = mdb_txn_begin(m_env, NULL, 0, txn)) + throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str())); mdb_drop(txn, m_blocks, 0); mdb_drop(txn, m_block_info, 0); mdb_drop(txn, m_block_heights, 0); mdb_drop(txn, m_txs, 0); mdb_drop(txn, m_tx_outputs, 0); mdb_drop(txn, m_output_txs, 0); - mdb_drop(txn, m_output_indices, 0); mdb_drop(txn, m_output_amounts, 0); - mdb_drop(txn, m_output_keys, 0); mdb_drop(txn, m_spent_keys, 0); mdb_drop(txn, m_hf_starting_heights, 0); mdb_drop(txn, m_hf_versions, 0); @@ -1332,10 +1261,12 @@ void BlockchainLMDB::unlock() } \ #define TXN_PREFIX_RDONLY() \ - bool my_rtxn = block_rtxn_start(); \ - MDB_txn *m_txn = m_write_txn ? m_write_txn->m_txn : m_tinfo->m_ti_rtxn -#define TXN_POSTFIX_RDONLY() \ - if (my_rtxn) block_rtxn_stop() + MDB_txn *m_txn; \ + mdb_txn_cursors *m_cursors; \ + bool my_rtxn = block_rtxn_start(&m_txn, &m_cursors); \ + mdb_txn_safe auto_txn(my_rtxn); \ + if (my_rtxn) auto_txn.m_tinfo = m_tinfo.get() +#define TXN_POSTFIX_RDONLY() #define TXN_POSTFIX_SUCCESS() \ do { \ @@ -1375,23 +1306,23 @@ bool BlockchainLMDB::block_exists(const crypto::hash& h) const check_open(); TXN_PREFIX_RDONLY(); - const mdb_txn_cursors *m_cursors = m_write_txn ? &m_wcursors : &m_tinfo->m_ti_rcursors; RCURSOR(block_heights); + bool ret = false; blk_height bh = {h, 0}; MDB_val key = {sizeof(bh), (void *)&bh}; auto get_result = mdb_cursor_get(m_cur_block_heights, (MDB_val *)&zerokval, &key, MDB_GET_BOTH); if (get_result == MDB_NOTFOUND) { - TXN_POSTFIX_RDONLY(); LOG_PRINT_L3("Block with hash " << epee::string_tools::pod_to_hex(h) << " not found in db"); - return false; } else if (get_result) - throw0(DB_ERROR("DB error attempting to fetch block index from hash")); + throw0(DB_ERROR(lmdb_error("DB error attempting to fetch block index from hash", get_result).c_str())); + else + ret = true; TXN_POSTFIX_RDONLY(); - return true; + return ret; } block BlockchainLMDB::get_block(const crypto::hash& h) const @@ -1408,7 +1339,6 @@ uint64_t BlockchainLMDB::get_block_height(const crypto::hash& h) const check_open(); TXN_PREFIX_RDONLY(); - const mdb_txn_cursors *m_cursors = m_write_txn ? &m_wcursors : &m_tinfo->m_ti_rcursors; RCURSOR(block_heights); blk_height bh = {h, 0}; @@ -1440,7 +1370,6 @@ block BlockchainLMDB::get_block_from_height(const uint64_t& height) const check_open(); TXN_PREFIX_RDONLY(); - const mdb_txn_cursors *m_cursors = m_write_txn ? &m_wcursors : &m_tinfo->m_ti_rcursors; RCURSOR(blocks); MDB_val_copy key(height); @@ -1471,7 +1400,6 @@ uint64_t BlockchainLMDB::get_block_timestamp(const uint64_t& height) const check_open(); TXN_PREFIX_RDONLY(); - const mdb_txn_cursors *m_cursors = m_write_txn ? &m_wcursors : &m_tinfo->m_ti_rcursors; RCURSOR(block_info); MDB_val_copy key(height); @@ -1510,7 +1438,6 @@ size_t BlockchainLMDB::get_block_size(const uint64_t& height) const check_open(); TXN_PREFIX_RDONLY(); - const mdb_txn_cursors *m_cursors = m_write_txn ? &m_wcursors : &m_tinfo->m_ti_rcursors; RCURSOR(block_info); MDB_val_copy key(height); @@ -1535,7 +1462,6 @@ difficulty_type BlockchainLMDB::get_block_cumulative_difficulty(const uint64_t& check_open(); TXN_PREFIX_RDONLY(); - const mdb_txn_cursors *m_cursors = m_write_txn ? &m_wcursors : &m_tinfo->m_ti_rcursors; RCURSOR(block_info); MDB_val_copy key(height); @@ -1577,7 +1503,6 @@ uint64_t BlockchainLMDB::get_block_already_generated_coins(const uint64_t& heigh check_open(); TXN_PREFIX_RDONLY(); - const mdb_txn_cursors *m_cursors = m_write_txn ? &m_wcursors : &m_tinfo->m_ti_rcursors; RCURSOR(block_info); MDB_val_copy key(height); @@ -1602,7 +1527,6 @@ crypto::hash BlockchainLMDB::get_block_hash_from_height(const uint64_t& height) check_open(); TXN_PREFIX_RDONLY(); - const mdb_txn_cursors *m_cursors = m_write_txn ? &m_wcursors : &m_tinfo->m_ti_rcursors; RCURSOR(block_info); MDB_val_copy key(height); @@ -1689,7 +1613,6 @@ bool BlockchainLMDB::tx_exists(const crypto::hash& h) const check_open(); TXN_PREFIX_RDONLY(); - const mdb_txn_cursors *m_cursors = m_write_txn ? &m_wcursors : &m_tinfo->m_ti_rcursors; RCURSOR(tx_indices); RCURSOR(txs); @@ -1730,7 +1653,6 @@ bool BlockchainLMDB::tx_exists(const crypto::hash& h, uint64_t& tx_index) const check_open(); TXN_PREFIX_RDONLY(); - const mdb_txn_cursors *m_cursors = m_write_txn ? &m_wcursors : &m_tinfo->m_ti_rcursors; RCURSOR(tx_indices); txindex ti = {h}; @@ -1747,14 +1669,17 @@ bool BlockchainLMDB::tx_exists(const crypto::hash& h, uint64_t& tx_index) const TXN_POSTFIX_RDONLY(); + bool ret = false; if (get_result == MDB_NOTFOUND) { LOG_PRINT_L1("transaction with hash " << epee::string_tools::pod_to_hex(h) << " not found in db"); - return false; } else if (get_result) - throw0(DB_ERROR("DB error attempting to fetch transaction from hash")); - return true; + throw0(DB_ERROR(lmdb_error("DB error attempting to fetch transaction from hash", get_result).c_str())); + else + ret = true; + + return ret; } uint64_t BlockchainLMDB::get_tx_unlock_time(const crypto::hash& h) const @@ -1763,7 +1688,6 @@ uint64_t BlockchainLMDB::get_tx_unlock_time(const crypto::hash& h) const check_open(); TXN_PREFIX_RDONLY(); - const mdb_txn_cursors *m_cursors = m_write_txn ? &m_wcursors : &m_tinfo->m_ti_rcursors; RCURSOR(tx_indices); txindex ti = {h}; @@ -1786,7 +1710,6 @@ transaction BlockchainLMDB::get_tx(const crypto::hash& h) const check_open(); TXN_PREFIX_RDONLY(); - const mdb_txn_cursors *m_cursors = m_write_txn ? &m_wcursors : &m_tinfo->m_ti_rcursors; RCURSOR(tx_indices); RCURSOR(txs); @@ -1804,7 +1727,7 @@ transaction BlockchainLMDB::get_tx(const crypto::hash& h) const if (get_result == MDB_NOTFOUND) throw1(TX_DNE(std::string("tx with hash ").append(epee::string_tools::pod_to_hex(h)).append(" not found in db").c_str())); else if (get_result) - throw0(DB_ERROR("DB error attempting to fetch tx from hash")); + throw0(DB_ERROR(lmdb_error("DB error attempting to fetch tx from hash", get_result).c_str())); blobdata bd; bd.assign(reinterpret_cast(result.mv_data), result.mv_size); @@ -1854,7 +1777,6 @@ uint64_t BlockchainLMDB::get_tx_block_height(const crypto::hash& h) const check_open(); TXN_PREFIX_RDONLY(); - const mdb_txn_cursors *m_cursors = m_write_txn ? &m_wcursors : &m_tinfo->m_ti_rcursors; RCURSOR(tx_indices); txindex ti = {h}; @@ -1865,7 +1787,7 @@ uint64_t BlockchainLMDB::get_tx_block_height(const crypto::hash& h) const throw1(TX_DNE(std::string("tx_data_t with hash ").append(epee::string_tools::pod_to_hex(h)).append(" not found in db").c_str())); } else if (get_result) - throw0(DB_ERROR("DB error attempting to fetch tx height from hash")); + throw0(DB_ERROR(lmdb_error("DB error attempting to fetch tx height from hash", get_result).c_str())); txindex *tip = (txindex *)v.mv_data; uint64_t ret = tip->data.height; @@ -1879,42 +1801,45 @@ uint64_t BlockchainLMDB::get_num_outputs(const uint64_t& amount) const check_open(); TXN_PREFIX_RDONLY(); - const mdb_txn_cursors *m_cursors = m_write_txn ? &m_wcursors : &m_tinfo->m_ti_rcursors; RCURSOR(output_amounts); MDB_val_copy k(amount); MDB_val v; - auto result = mdb_cursor_get(m_cur_output_amounts, &k, &v, MDB_SET); - if (result == MDB_NOTFOUND) - { - TXN_POSTFIX_RDONLY(); - return 0; - } - else if (result) - throw0(DB_ERROR("DB error attempting to get number of outputs of an amount")); - mdb_size_t num_elems = 0; - mdb_cursor_count(m_cur_output_amounts, &num_elems); + auto result = mdb_cursor_get(m_cur_output_amounts, &k, &v, MDB_SET); + if (result == MDB_SUCCESS) + { + mdb_cursor_count(m_cur_output_amounts, &num_elems); + } + else if (result != MDB_NOTFOUND) + throw0(DB_ERROR("DB error attempting to get number of outputs of an amount")); TXN_POSTFIX_RDONLY(); return num_elems; } +// TODO: probably remove this function output_data_t BlockchainLMDB::get_output_key(const uint64_t &global_index) const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__ << " (unused version - does nothing)"); + outkey ok = {0}; + return ok.data; +} + +output_data_t BlockchainLMDB::get_output_key(const uint64_t& amount, const uint64_t& index) { LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); TXN_PREFIX_RDONLY(); - const mdb_txn_cursors *m_cursors = m_write_txn ? &m_wcursors : &m_tinfo->m_ti_rcursors; - RCURSOR(output_keys); + RCURSOR(output_amounts); - outkey ok = {global_index}; - MDB_val v = {sizeof(ok), (void *)&ok}; - auto get_result = mdb_cursor_get(m_cur_output_keys, (MDB_val *)&zerokval, &v, MDB_GET_BOTH); + MDB_val_set(k, amount); + MDB_val_set(v, index); + auto get_result = mdb_cursor_get(m_cur_output_amounts, &k, &v, MDB_GET_BOTH); if (get_result == MDB_NOTFOUND) - throw1(OUTPUT_DNE("Attempting to get output pubkey by global index, but key does not exist")); + throw1(OUTPUT_DNE("Attempting to get output pubkey by index, but key does not exist")); else if (get_result) throw0(DB_ERROR("Error attempting to retrieve an output pubkey from the db")); outkey *okp = (outkey *)v.mv_data; @@ -1923,24 +1848,13 @@ output_data_t BlockchainLMDB::get_output_key(const uint64_t &global_index) const return ret; } -output_data_t BlockchainLMDB::get_output_key(const uint64_t& amount, const uint64_t& index) -{ - LOG_PRINT_L3("BlockchainLMDB::" << __func__); - check_open(); - - uint64_t glob_index = get_output_global_index(amount, index); - return get_output_key(glob_index); -} - tx_out_index BlockchainLMDB::get_output_tx_and_index_from_global(const uint64_t& index) const { LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); TXN_PREFIX_RDONLY(); - const mdb_txn_cursors *m_cursors = m_write_txn ? &m_wcursors : &m_tinfo->m_ti_rcursors; RCURSOR(output_txs); - RCURSOR(output_indices); MDB_val_copy k(index); MDB_val v; @@ -1951,15 +1865,9 @@ tx_out_index BlockchainLMDB::get_output_tx_and_index_from_global(const uint64_t& else if (get_result) throw0(DB_ERROR("DB error attempting to fetch output tx hash")); - crypto::hash tx_hash = *(const crypto::hash*)v.mv_data; + outtx *ot = (outtx *)v.mv_data; + tx_out_index ret = tx_out_index(ot->tx_hash, ot->local_index); - get_result = mdb_cursor_get(m_cur_output_indices, &k, &v, MDB_SET); - if (get_result == MDB_NOTFOUND) - throw1(OUTPUT_DNE("output with given index not in db")); - else if (get_result) - throw0(DB_ERROR("DB error attempting to fetch output tx index")); - - tx_out_index ret = tx_out_index(tx_hash, *(const uint64_t *)v.mv_data); TXN_POSTFIX_RDONLY(); return ret; } @@ -1977,6 +1885,7 @@ tx_out_index BlockchainLMDB::get_output_tx_and_index(const uint64_t& amount, con return indices[0]; } +// we don't have global_output_indices void BlockchainLMDB::get_amount_and_global_output_indices(const uint64_t tx_index, std::vector& amount_output_indices, std::vector& global_output_indices) const @@ -1992,7 +1901,6 @@ void BlockchainLMDB::get_amount_and_global_output_indices(const uint64_t tx_inde // txn, regardless of batch mode. Otherwise, remove_tx_outputs() would now // create a new read-only txn here, which is incorrect. TXN_PREFIX_RDONLY(); - const mdb_txn_cursors *m_cursors = m_write_txn ? &m_wcursors : &m_tinfo->m_ti_rcursors; RCURSOR(tx_outputs); int result = 0; @@ -2001,25 +1909,21 @@ void BlockchainLMDB::get_amount_and_global_output_indices(const uint64_t tx_inde result = mdb_cursor_get(m_cur_tx_outputs, &k_tx_index, &v, MDB_SET); if (result == MDB_NOTFOUND) - LOG_PRINT_L0("WARNING: Unexpected: tx has no amount and global indices stored in " + LOG_PRINT_L0("WARNING: Unexpected: tx has no amount indices stored in " "tx_outputs, but it should have an empty entry even if it's a tx without " "outputs"); else if (result) - throw0(DB_ERROR("DB error attempting to get data for tx_outputs[tx_index]")); + throw0(DB_ERROR(lmdb_error("DB error attempting to get data for tx_outputs[tx_index]", result).c_str())); - uint64_t* paired_indices = (uint64_t*)v.mv_data; - int num_elems = v.mv_size / sizeof(uint64_t); - if (num_elems % 2 != 0) - throw0(DB_ERROR("tx_outputs[tx_index] does not have an even numer of indices")); - int num_outputs = num_elems / 2; + uint64_t* indices = (uint64_t*)v.mv_data; + int num_outputs = v.mv_size / sizeof(uint64_t); for (int i = 0; i < num_outputs; ++i) { // LOG_PRINT_L0("amount output index[" << 2*i << "]" << ": " << paired_indices[2*i] << " global output index: " << paired_indices[2*i+1]); - amount_output_indices.push_back(paired_indices[2*i]); - global_output_indices.push_back(paired_indices[2*i+1]); + amount_output_indices.push_back(indices[i]); } - paired_indices = nullptr; + indices = nullptr; TXN_POSTFIX_RDONLY(); } @@ -2045,11 +1949,11 @@ bool BlockchainLMDB::has_key_image(const crypto::key_image& img) const bool ret; TXN_PREFIX_RDONLY(); - const mdb_txn_cursors *m_cursors = m_write_txn ? &m_wcursors : &m_tinfo->m_ti_rcursors; RCURSOR(spent_keys); MDB_val k = {sizeof(img), (void *)&img}; ret = (mdb_cursor_get(m_cur_spent_keys, (MDB_val *)&zerokval, &k, MDB_GET_BOTH) == 0); + TXN_POSTFIX_RDONLY(); return ret; } @@ -2060,7 +1964,6 @@ bool BlockchainLMDB::for_all_key_images(std::functionm_ti_rcursors; RCURSOR(spent_keys); MDB_val k; @@ -2093,7 +1996,6 @@ bool BlockchainLMDB::for_all_blocks(std::functionm_ti_rcursors; RCURSOR(blocks); MDB_val k; @@ -2135,7 +2037,6 @@ bool BlockchainLMDB::for_all_transactions(std::functionm_ti_rcursors; RCURSOR(txs); MDB_val k; @@ -2174,7 +2075,6 @@ bool BlockchainLMDB::for_all_outputs(std::functionm_ti_rcursors; RCURSOR(output_amounts); MDB_val k; @@ -2191,8 +2091,8 @@ bool BlockchainLMDB::for_all_outputs(std::functiontx_index); if (!f(amount, toi.first, toi.second)) { ret = false; break; @@ -2218,6 +2118,7 @@ void BlockchainLMDB::batch_start(uint64_t batch_num_blocks) throw0(DB_ERROR("batch transaction attempted, but m_write_txn already in use")); check_open(); + m_writer = boost::this_thread::get_id(); check_and_resize_for_batch(batch_num_blocks); m_write_batch_txn = new mdb_txn_safe(); @@ -2260,6 +2161,7 @@ void BlockchainLMDB::batch_commit() m_write_txn = nullptr; delete m_write_batch_txn; + m_write_batch_txn = nullptr; memset(&m_wcursors, 0, sizeof(m_wcursors)); } @@ -2299,8 +2201,9 @@ void BlockchainLMDB::batch_abort() m_write_txn = nullptr; // explicitly call in case mdb_env_close() (BlockchainLMDB::close()) called before BlockchainLMDB destructor called. m_write_batch_txn->abort(); - m_batch_active = false; + delete m_write_batch_txn; m_write_batch_txn = nullptr; + m_batch_active = false; memset(&m_wcursors, 0, sizeof(m_wcursors)); LOG_PRINT_L3("batch transaction: aborted"); } @@ -2317,10 +2220,14 @@ void BlockchainLMDB::set_batch_transactions(bool batch_transactions) } // return true if we started the txn, false if already started -bool BlockchainLMDB::block_rtxn_start() const +bool BlockchainLMDB::block_rtxn_start(MDB_txn **mtxn, mdb_txn_cursors **mcur) const { - if (m_write_txn) - return false; + bool ret = false; + if (m_write_txn && m_writer == boost::this_thread::get_id()) { + *mtxn = m_write_txn->m_txn; + *mcur = (mdb_txn_cursors *)&m_wcursors; + return ret; + } if (!m_tinfo.get()) { m_tinfo.reset(new mdb_threadinfo); @@ -2328,17 +2235,21 @@ bool BlockchainLMDB::block_rtxn_start() const memset(&m_tinfo->m_ti_rflags, 0, sizeof(m_tinfo->m_ti_rflags)); if (auto mdb_res = mdb_txn_begin(m_env, NULL, MDB_RDONLY, &m_tinfo->m_ti_rtxn)) throw0(DB_ERROR_TXN_START(lmdb_error("Failed to create a read transaction for the db: ", mdb_res).c_str())); + ret = true; } else if (!m_tinfo->m_ti_rflags.m_rf_txn) { if (auto mdb_res = mdb_txn_renew(m_tinfo->m_ti_rtxn)) throw0(DB_ERROR_TXN_START(lmdb_error("Failed to renew a read transaction for the db: ", mdb_res).c_str())); - } else - { - return false; + ret = true; } - m_tinfo->m_ti_rflags.m_rf_txn = true; - LOG_PRINT_L3("BlockchainLMDB::" << __func__); - return true; + if (ret) + m_tinfo->m_ti_rflags.m_rf_txn = true; + *mtxn = m_tinfo->m_ti_rtxn; + *mcur = &m_tinfo->m_ti_rcursors; + + if (ret) + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + return ret; } void BlockchainLMDB::block_rtxn_stop() const @@ -2353,7 +2264,7 @@ void BlockchainLMDB::block_txn_start(bool readonly) if (readonly) { bool didit = false; - if (m_write_txn) + if (m_write_txn && m_writer == boost::this_thread::get_id()) return; if (!m_tinfo.get()) { @@ -2389,6 +2300,7 @@ void BlockchainLMDB::block_txn_start(bool readonly) throw0(DB_ERROR_TXN_START((std::string("Attempted to start new write txn when write txn already exists in ")+__FUNCTION__).c_str())); if (! m_batch_active) { + m_writer = boost::this_thread::get_id(); m_write_txn = new mdb_txn_safe(); if (auto mdb_res = mdb_txn_begin(m_env, NULL, 0, *m_write_txn)) { @@ -2403,9 +2315,9 @@ void BlockchainLMDB::block_txn_start(bool readonly) void BlockchainLMDB::block_txn_stop() { LOG_PRINT_L3("BlockchainLMDB::" << __func__); - if (! m_batch_active) + if (m_write_txn && m_writer == boost::this_thread::get_id()) { - if (m_write_txn) + if (! m_batch_active) { TIME_MEASURE_START(time1); m_write_txn->commit(); @@ -2416,38 +2328,38 @@ void BlockchainLMDB::block_txn_stop() m_write_txn = nullptr; memset(&m_wcursors, 0, sizeof(m_wcursors)); } - else if (m_tinfo->m_ti_rtxn) - { - mdb_txn_reset(m_tinfo->m_ti_rtxn); - memset(&m_tinfo->m_ti_rflags, 0, sizeof(m_tinfo->m_ti_rflags)); - } + } + else if (m_tinfo->m_ti_rtxn) + { + mdb_txn_reset(m_tinfo->m_ti_rtxn); + memset(&m_tinfo->m_ti_rflags, 0, sizeof(m_tinfo->m_ti_rflags)); } } void BlockchainLMDB::block_txn_abort() { LOG_PRINT_L3("BlockchainLMDB::" << __func__); - if (! m_batch_active) + if (m_write_txn && m_writer == boost::this_thread::get_id()) { - if (m_write_txn != nullptr) + if (! m_batch_active) { delete m_write_txn; m_write_txn = nullptr; memset(&m_wcursors, 0, sizeof(m_wcursors)); } - else if (m_tinfo->m_ti_rtxn) - { - mdb_txn_reset(m_tinfo->m_ti_rtxn); - memset(&m_tinfo->m_ti_rflags, 0, sizeof(m_tinfo->m_ti_rflags)); - } - else - { - // This would probably mean an earlier exception was caught, but then we - // proceeded further than we should have. - throw0(DB_ERROR((std::string("BlockchainLMDB::") + __func__ + - std::string(": block-level DB transaction abort called when write txn doesn't exist") - ).c_str())); - } + } + else if (m_tinfo->m_ti_rtxn) + { + mdb_txn_reset(m_tinfo->m_ti_rtxn); + memset(&m_tinfo->m_ti_rflags, 0, sizeof(m_tinfo->m_ti_rflags)); + } + else + { + // This would probably mean an earlier exception was caught, but then we + // proceeded further than we should have. + throw0(DB_ERROR((std::string("BlockchainLMDB::") + __func__ + + std::string(": block-level DB transaction abort called when write txn doesn't exist") + ).c_str())); } } @@ -2521,9 +2433,7 @@ void BlockchainLMDB::get_output_tx_and_index_from_global(const std::vectorm_ti_rcursors; RCURSOR(output_txs); - RCURSOR(output_indices); for (const uint64_t &index : global_indices) { @@ -2536,149 +2446,14 @@ void BlockchainLMDB::get_output_tx_and_index_from_global(const std::vectortx_hash, ot->local_index); tx_out_indices.push_back(result); } TXN_POSTFIX_RDONLY(); } -void BlockchainLMDB::get_output_global_indices(const uint64_t& amount, const std::vector &offsets, - std::vector &global_indices) -{ - LOG_PRINT_L3("BlockchainLMDB::" << __func__); - TIME_MEASURE_START(txx); - check_open(); - global_indices.clear(); - - uint64_t max = 0; - for (const uint64_t &index : offsets) - { - if (index > max) - max = index; - } - - TXN_PREFIX_RDONLY(); - const mdb_txn_cursors *m_cursors = m_write_txn ? &m_wcursors : &m_tinfo->m_ti_rcursors; - RCURSOR(output_amounts); - - MDB_val_copy k(amount); - MDB_val v; - auto result = mdb_cursor_get(m_cur_output_amounts, &k, &v, MDB_SET); - if (result == MDB_NOTFOUND) - throw1(OUTPUT_DNE("Attempting to get an output index by amount and amount index, but amount not found")); - else if (result) - throw0(DB_ERROR("DB error attempting to get an output")); - - mdb_size_t num_elems = 0; - mdb_cursor_count(m_cur_output_amounts, &num_elems); - if (max <= 1 && num_elems <= max) - throw1(OUTPUT_DNE("Attempting to get an output index by amount and amount index, but output not found")); - - uint64_t t_dbmul = 0; - uint64_t t_dbscan = 0; - if (max <= 1) - { - for (const uint64_t& index : offsets) - { - mdb_cursor_get(m_cur_output_amounts, &k, &v, MDB_FIRST_DUP); - for (uint64_t i = 0; i < index; ++i) - { - mdb_cursor_get(m_cur_output_amounts, &k, &v, MDB_NEXT_DUP); - } - - mdb_cursor_get(m_cur_output_amounts, &k, &v, MDB_GET_CURRENT); - uint64_t glob_index = *(const uint64_t*) v.mv_data; - LOG_PRINT_L3("Amount: " << amount << " M0->v: " << glob_index); - global_indices.push_back(glob_index); - } - } - else - { - uint32_t curcount = 0; - uint32_t blockstart = 0; - for (const uint64_t& index : offsets) - { - if (index >= num_elems) - { - LOG_PRINT_L1("Index: " << index << " Elems: " << num_elems << " partial results found for get_output_tx_and_index"); - break; - } - if (!curcount && index > num_elems/2) - { - mdb_cursor_get(m_cur_output_amounts, &k, &v, MDB_LAST_DUP); - mdb_cursor_get(m_cur_output_amounts, &k, &v, MDB_PREV); /* kludge to unset C_EOF */ - mdb_cursor_get(m_cur_output_amounts, &k, &v, MDB_NEXT); - mdb_cursor_get(m_cur_output_amounts, &k, &v, MDB_GET_MULTIPLE); - - curcount = num_elems; - while(1) - { - TIME_MEASURE_START(db1); - int count = v.mv_size / sizeof(uint64_t); - curcount -= count; - if (curcount > index) - { - mdb_cursor_get(m_cur_output_amounts, &k, &v, MDB_PREV_MULTIPLE); - } else - { - blockstart = curcount; - curcount += count; - break; - } - TIME_MEASURE_FINISH(db1); - t_dbmul += db1; - } - - } else - { - while (index >= curcount) - { - TIME_MEASURE_START(db1); - if (mdb_cursor_get(m_cur_output_amounts, &k, &v, curcount == 0 ? MDB_GET_MULTIPLE : MDB_NEXT_MULTIPLE) != 0) - { - // allow partial results - result = false; - break; - } - - int count = v.mv_size / sizeof(uint64_t); - - blockstart = curcount; - curcount += count; - TIME_MEASURE_FINISH(db1); - t_dbmul += db1; - } - } - - LOG_PRINT_L3("Records returned: " << curcount << " Index: " << index); - TIME_MEASURE_START(db2); - uint64_t actual_index = index - blockstart; - uint64_t glob_index = ((const uint64_t*) v.mv_data)[actual_index]; - - LOG_PRINT_L3("Amount: " << amount << " M1->v: " << glob_index); - global_indices.push_back(glob_index); - - TIME_MEASURE_FINISH(db2); - t_dbscan += db2; - - } - } - - TXN_POSTFIX_RDONLY(); - - TIME_MEASURE_FINISH(txx); - LOG_PRINT_L3("txx: " << txx << " db1: " << t_dbmul << " db2: " << t_dbscan); -} - void BlockchainLMDB::get_output_key(const uint64_t &amount, const std::vector &offsets, std::vector &outputs) { LOG_PRINT_L3("BlockchainLMDB::" << __func__); @@ -2687,31 +2462,25 @@ void BlockchainLMDB::get_output_key(const uint64_t &amount, const std::vectorm_ti_rcursors; - std::vector global_indices; - get_output_global_indices(amount, offsets, global_indices); - if (global_indices.size() > 0) + RCURSOR(output_amounts); + + MDB_val_set(k, amount); + for (const uint64_t &index : offsets) { - RCURSOR(output_keys); + MDB_val_set(v, index); - for (const uint64_t &index : global_indices) - { - outkey ok = {index}; - MDB_val v = {sizeof(ok), (void *)&ok}; - - auto get_result = mdb_cursor_get(m_cur_output_keys, (MDB_val *)&zerokval, &v, MDB_GET_BOTH); - if (get_result == MDB_NOTFOUND) - throw1(OUTPUT_DNE("Attempting to get output pubkey by global index, but key does not exist")); - else if (get_result) - throw0(DB_ERROR("Error attempting to retrieve an output pubkey from the db")); - - outkey *okp = (outkey *)v.mv_data; - output_data_t data = okp->data; - outputs.push_back(data); - } + auto get_result = mdb_cursor_get(m_cur_output_amounts, &k, &v, MDB_GET_BOTH); + if (get_result == MDB_NOTFOUND) + throw1(OUTPUT_DNE("Attempting to get output pubkey by global index, but key does not exist")); + else if (get_result) + throw0(DB_ERROR(lmdb_error("Error attempting to retrieve an output pubkey from the db", get_result).c_str())); + outkey *okp = (outkey *)v.mv_data; + output_data_t data = okp->data; + outputs.push_back(data); } + TXN_POSTFIX_RDONLY(); TIME_MEASURE_FINISH(db3); @@ -2724,13 +2493,30 @@ void BlockchainLMDB::get_output_tx_and_index(const uint64_t& amount, const std:: check_open(); indices.clear(); - std::vector global_indices; - get_output_global_indices(amount, offsets, global_indices); + std::vector tx_indices; + TXN_PREFIX_RDONLY(); + + RCURSOR(output_amounts); + + MDB_val_set(k, amount); + for (const uint64_t &index : offsets) + { + MDB_val_set(v, index); + + auto get_result = mdb_cursor_get(m_cur_output_amounts, &k, &v, MDB_GET_BOTH); + if (get_result == MDB_NOTFOUND) + throw1(OUTPUT_DNE("Attempting to get output by index, but key does not exist")); + else if (get_result) + throw0(DB_ERROR(lmdb_error("Error attempting to retrieve an output from the db", get_result).c_str())); + + outkey *okp = (outkey *)v.mv_data; + tx_indices.push_back(okp->tx_index); + } TIME_MEASURE_START(db3); - if(global_indices.size() > 0) + if(tx_indices.size() > 0) { - get_output_tx_and_index_from_global(global_indices, indices); + get_output_tx_and_index_from_global(tx_indices, indices); } TIME_MEASURE_FINISH(db3); LOG_PRINT_L3("db3: " << db3); @@ -2800,18 +2586,21 @@ uint64_t BlockchainLMDB::get_hard_fork_starting_height(uint8_t version) const MDB_val_copy val_key(version); MDB_val val_ret; + uint64_t ret = 0; auto result = mdb_get(m_txn, m_hf_starting_heights, &val_key, &val_ret); - if (result == MDB_NOTFOUND) - return std::numeric_limits::max(); - if (result) - throw0(DB_ERROR("Error attempting to retrieve a hard fork starting height from the db")); - - uint64_t ret; + if (result == MDB_SUCCESS) + { #ifdef MISALIGNED_OK - ret = *(const uint64_t*)val_ret.mv_data; + ret = *(const uint64_t*)val_ret.mv_data; #else - memcpy(&ret, val_ret.mv_data, sizeof(uint64_t)); + memcpy(&ret, val_ret.mv_data, sizeof(uint64_t)); #endif + } else if (result == MDB_NOTFOUND) + { + ret = std::numeric_limits::max(); + } else if (result) + throw0(DB_ERROR(lmdb_error("Error attempting to retrieve a hard fork starting height from the db", result).c_str())); + TXN_POSTFIX_RDONLY(); return ret; } @@ -2841,7 +2630,6 @@ uint8_t BlockchainLMDB::get_hard_fork_version(uint64_t height) const check_open(); TXN_PREFIX_RDONLY(); - const mdb_txn_cursors *m_cursors = m_write_txn ? &m_wcursors : &m_tinfo->m_ti_rcursors; RCURSOR(hf_versions); MDB_val_copy val_key(height); diff --git a/src/blockchain_db/lmdb/db_lmdb.h b/src/blockchain_db/lmdb/db_lmdb.h index 0eda3bc4d..a298ee68f 100644 --- a/src/blockchain_db/lmdb/db_lmdb.h +++ b/src/blockchain_db/lmdb/db_lmdb.h @@ -46,9 +46,7 @@ typedef struct mdb_txn_cursors MDB_cursor *m_txc_block_info; MDB_cursor *m_txc_output_txs; - MDB_cursor *m_txc_output_indices; MDB_cursor *m_txc_output_amounts; - MDB_cursor *m_txc_output_keys; MDB_cursor *m_txc_txs; MDB_cursor *m_txc_tx_indices; @@ -63,9 +61,7 @@ typedef struct mdb_txn_cursors #define m_cur_block_heights m_cursors->m_txc_block_heights #define m_cur_block_info m_cursors->m_txc_block_info #define m_cur_output_txs m_cursors->m_txc_output_txs -#define m_cur_output_indices m_cursors->m_txc_output_indices #define m_cur_output_amounts m_cursors->m_txc_output_amounts -#define m_cur_output_keys m_cursors->m_txc_output_keys #define m_cur_txs m_cursors->m_txc_txs #define m_cur_tx_indices m_cursors->m_txc_tx_indices #define m_cur_tx_outputs m_cursors->m_txc_tx_outputs @@ -79,9 +75,7 @@ typedef struct mdb_rflags bool m_rf_block_heights; bool m_rf_block_info; bool m_rf_output_txs; - bool m_rf_output_indices; bool m_rf_output_amounts; - bool m_rf_output_keys; bool m_rf_txs; bool m_rf_tx_indices; bool m_rf_tx_outputs; @@ -100,7 +94,7 @@ typedef struct mdb_threadinfo struct mdb_txn_safe { - mdb_txn_safe(); + mdb_txn_safe(const bool check=true); ~mdb_txn_safe(); void commit(std::string message = ""); @@ -127,8 +121,10 @@ struct mdb_txn_safe static void wait_no_active_txns(); static void allow_new_txns(); + mdb_threadinfo* m_tinfo; MDB_txn* m_txn; bool m_batch_txn = false; + bool m_check; static std::atomic num_active_txns; // could use a mutex here, but this should be sufficient. @@ -230,7 +226,6 @@ public: virtual tx_out_index get_output_tx_and_index(const uint64_t& amount, const uint64_t& index); virtual void get_output_tx_and_index(const uint64_t& amount, const std::vector &offsets, std::vector &indices); - virtual void get_output_global_indices(const uint64_t& amount, const std::vector &offsets, std::vector &indices); virtual void get_amount_and_global_output_indices(const uint64_t tx_index, std::vector& amount_output_indices, @@ -261,7 +256,7 @@ public: virtual void block_txn_start(bool readonly); virtual void block_txn_stop(); virtual void block_txn_abort(); - virtual bool block_rtxn_start() const; + virtual bool block_rtxn_start(MDB_txn **mtxn, mdb_txn_cursors **mcur) const; virtual void block_rtxn_stop() const; virtual void pop_block(block& blk, std::vector& txs); @@ -337,16 +332,6 @@ private: */ tx_out output_from_blob(const blobdata& blob) const; - /** - * @brief get the global index of the index-th output of the given amount - * - * @param amount the output amount - * @param index the index into the set of outputs of that amount - * - * @return the global index of the desired output - */ - uint64_t get_output_global_index(const uint64_t& amount, const uint64_t& index); - void check_open() const; virtual bool is_read_only() const; @@ -365,9 +350,7 @@ private: MDB_dbi m_tx_outputs; MDB_dbi m_output_txs; - MDB_dbi m_output_indices; MDB_dbi m_output_amounts; - MDB_dbi m_output_keys; MDB_dbi m_spent_keys; @@ -384,6 +367,7 @@ private: std::string m_folder; mdb_txn_safe* m_write_txn; // may point to either a short-lived txn or a batch txn mdb_txn_safe* m_write_batch_txn; // persist batch txn outside of BlockchainLMDB + boost::thread::id m_writer; bool m_batch_transactions; // support for batch transactions bool m_batch_active; // whether batch transaction is in progress From 118dd69dd553a9b214d7d14e4d15f98b46164c06 Mon Sep 17 00:00:00 2001 From: Howard Chu Date: Mon, 28 Mar 2016 18:26:37 +0100 Subject: [PATCH 17/23] Use DUPFIXED for block_info and output_txs Saves another ~150MB or so on the full blockchain --- src/blockchain_db/lmdb/db_lmdb.cpp | 101 +++++++++++++---------------- src/blockchain_db/lmdb/db_lmdb.h | 1 - 2 files changed, 46 insertions(+), 56 deletions(-) diff --git a/src/blockchain_db/lmdb/db_lmdb.cpp b/src/blockchain_db/lmdb/db_lmdb.cpp index 604fde0aa..1ccd477f3 100644 --- a/src/blockchain_db/lmdb/db_lmdb.cpp +++ b/src/blockchain_db/lmdb/db_lmdb.cpp @@ -231,6 +231,7 @@ typedef struct outkey { } outkey; typedef struct outtx { + uint64_t txnum; crypto::hash tx_hash; uint64_t local_index; } outtx; @@ -592,6 +593,7 @@ void BlockchainLMDB::add_block(const block& blk, const size_t& block_size, const throw0(DB_ERROR(lmdb_error("Failed to add block blob to db transaction: ", result).c_str())); mdb_block_info bi; + bi.bi_height = m_height; bi.bi_timestamp = blk.timestamp; bi.bi_coins = coins_generated; bi.bi_size = block_size; @@ -599,7 +601,7 @@ void BlockchainLMDB::add_block(const block& blk, const size_t& block_size, const bi.bi_hash = blk_hash; MDB_val val = {sizeof(bi), (void *)&bi}; - result = mdb_cursor_put(m_cur_block_info, &key, &val, MDB_APPEND); + result = mdb_cursor_put(m_cur_block_info, (MDB_val *)&zerokval, &val, MDB_APPENDDUP); if (result) throw0(DB_ERROR(lmdb_error("Failed to add block info to db transaction: ", result).c_str())); @@ -624,8 +626,8 @@ void BlockchainLMDB::remove_block() mdb_txn_cursors *m_cursors = &m_wcursors; CURSOR(block_info) MDB_val_copy k(m_height - 1); - MDB_val h; - if ((result = mdb_cursor_get(m_cur_block_info, &k, &h, MDB_SET))) + MDB_val h = k; + if ((result = mdb_cursor_get(m_cur_block_info, (MDB_val *)&zerokval, &h, MDB_GET_BOTH))) throw1(BLOCK_DNE(lmdb_error("Attempting to remove block that's not in the db: ", result).c_str())); // must use h now; deleting from m_block_info will invalidate it @@ -749,12 +751,10 @@ void BlockchainLMDB::add_output(const crypto::hash& tx_hash, if (tx_output.target.type() != typeid(txout_to_key)) throw0(DB_ERROR("Wrong output type: expected txout_to_key")); - MDB_val_set(k, m_num_outputs); - - outtx ot = {tx_hash, local_index}; + outtx ot = {m_num_outputs, tx_hash, local_index}; MDB_val_set(vot, ot); - result = mdb_cursor_put(m_cur_output_txs, &k, &vot, MDB_APPEND); + result = mdb_cursor_put(m_cur_output_txs, (MDB_val *)&zerokval, &vot, MDB_APPENDDUP); if (result) throw0(DB_ERROR(lmdb_error("Failed to add output tx hash to db transaction: ", result).c_str())); @@ -849,10 +849,22 @@ void BlockchainLMDB::remove_output(const uint64_t& out_index, const uint64_t amo { LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); + mdb_txn_cursors *m_cursors = &m_wcursors; + CURSOR(output_amounts); + CURSOR(output_txs); - MDB_val_copy k(out_index); + MDB_val_set(k, amount); + MDB_val_set(v, out_index); - auto result = mdb_del(*m_write_txn, m_output_txs, &k, NULL); + auto result = mdb_cursor_get(m_cur_output_amounts, &k, &v, MDB_GET_BOTH); + if (result == MDB_NOTFOUND) + throw1(OUTPUT_DNE("Attempting to get an output index by amount and amount index, but amount not found")); + else if (result) + throw0(DB_ERROR(lmdb_error("DB error attempting to get an output", result).c_str())); + + outkey *ok = (outkey *)v.mv_data; + MDB_val_set(otxk, ok->tx_index); + result = mdb_cursor_get(m_cur_output_txs, (MDB_val *)&zerokval, &otxk, MDB_GET_BOTH); if (result == MDB_NOTFOUND) { LOG_PRINT_L0("Unexpected: global output index not found in m_output_txs"); @@ -861,32 +873,16 @@ void BlockchainLMDB::remove_output(const uint64_t& out_index, const uint64_t amo { throw1(DB_ERROR(lmdb_error("Error adding removal of output tx to db transaction", result).c_str())); } + result = mdb_cursor_del(m_cur_output_txs, 0); + if (result) + throw0(DB_ERROR(lmdb_error(std::string("Error deleting output index ").append(boost::lexical_cast(out_index).append(": ")).c_str(), result).c_str())); - remove_amount_output_index(amount, out_index); - - m_num_outputs--; -} - -void BlockchainLMDB::remove_amount_output_index(const uint64_t amount, const uint64_t output_index) -{ - LOG_PRINT_L3("BlockchainLMDB::" << __func__); - check_open(); - mdb_txn_cursors *m_cursors = &m_wcursors; - CURSOR(output_amounts); - - MDB_val_set(k, amount); - MDB_val_set(v, output_index); - - auto result = mdb_cursor_get(m_cur_output_amounts, &k, &v, MDB_GET_BOTH); - if (result == MDB_NOTFOUND) - throw1(OUTPUT_DNE("Attempting to get an output index by amount and amount index, but amount not found")); - else if (result) - throw0(DB_ERROR(lmdb_error("DB error attempting to get an output", result).c_str())); - - // now delete it + // now delete the amount result = mdb_cursor_del(m_cur_output_amounts, 0); if (result) - throw0(DB_ERROR(lmdb_error(std::string("Error deleting amount for output index ").append(boost::lexical_cast(output_index).append(": ")).c_str(), result).c_str())); + throw0(DB_ERROR(lmdb_error(std::string("Error deleting amount for output index ").append(boost::lexical_cast(out_index).append(": ")).c_str(), result).c_str())); + + m_num_outputs--; } void BlockchainLMDB::add_spent_key(const crypto::key_image& k_image) @@ -1052,14 +1048,14 @@ void BlockchainLMDB::open(const std::string& filename, const int mdb_flags) // uses macros to avoid having to change things too many places lmdb_db_open(txn, LMDB_BLOCKS, MDB_INTEGERKEY | MDB_CREATE, m_blocks, "Failed to open db handle for m_blocks"); - lmdb_db_open(txn, LMDB_BLOCK_INFO, MDB_INTEGERKEY | MDB_CREATE, m_block_info, "Failed to open db handle for m_block_info"); + lmdb_db_open(txn, LMDB_BLOCK_INFO, MDB_INTEGERKEY | MDB_CREATE | MDB_DUPSORT | MDB_DUPFIXED, m_block_info, "Failed to open db handle for m_block_info"); lmdb_db_open(txn, LMDB_BLOCK_HEIGHTS, MDB_INTEGERKEY | MDB_CREATE | MDB_DUPSORT | MDB_DUPFIXED, m_block_heights, "Failed to open db handle for m_block_heights"); lmdb_db_open(txn, LMDB_TXS, MDB_INTEGERKEY | MDB_CREATE, m_txs, "Failed to open db handle for m_txs"); lmdb_db_open(txn, LMDB_TX_INDICES, MDB_INTEGERKEY | MDB_CREATE | MDB_DUPSORT | MDB_DUPFIXED, m_tx_indices, "Failed to open db handle for m_tx_indices"); lmdb_db_open(txn, LMDB_TX_OUTPUTS, MDB_INTEGERKEY | MDB_CREATE, m_tx_outputs, "Failed to open db handle for m_tx_outputs"); - lmdb_db_open(txn, LMDB_OUTPUT_TXS, MDB_INTEGERKEY | MDB_CREATE, m_output_txs, "Failed to open db handle for m_output_txs"); + lmdb_db_open(txn, LMDB_OUTPUT_TXS, MDB_INTEGERKEY | MDB_CREATE | MDB_DUPSORT | MDB_DUPFIXED, m_output_txs, "Failed to open db handle for m_output_txs"); lmdb_db_open(txn, LMDB_OUTPUT_AMOUNTS, MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED | MDB_CREATE, m_output_amounts, "Failed to open db handle for m_output_amounts"); lmdb_db_open(txn, LMDB_SPENT_KEYS, MDB_INTEGERKEY | MDB_CREATE | MDB_DUPSORT | MDB_DUPFIXED, m_spent_keys, "Failed to open db handle for m_spent_keys"); @@ -1073,6 +1069,8 @@ void BlockchainLMDB::open(const std::string& filename, const int mdb_flags) mdb_set_dupsort(txn, m_block_heights, compare_hash32); mdb_set_dupsort(txn, m_tx_indices, compare_hash32); mdb_set_dupsort(txn, m_output_amounts, compare_uint64); + mdb_set_dupsort(txn, m_output_txs, compare_uint64); + mdb_set_dupsort(txn, m_block_info, compare_uint64); mdb_set_compare(txn, m_hf_starting_heights, compare_uint8); mdb_set_compare(txn, m_properties, compare_string); @@ -1402,9 +1400,8 @@ uint64_t BlockchainLMDB::get_block_timestamp(const uint64_t& height) const TXN_PREFIX_RDONLY(); RCURSOR(block_info); - MDB_val_copy key(height); - MDB_val result; - auto get_result = mdb_cursor_get(m_cur_block_info, &key, &result, MDB_SET); + MDB_val_set(result, height); + auto get_result = mdb_cursor_get(m_cur_block_info, (MDB_val *)&zerokval, &result, MDB_GET_BOTH); if (get_result == MDB_NOTFOUND) { throw0(BLOCK_DNE(std::string("Attempt to get timestamp from height ").append(boost::lexical_cast(height)).append(" failed -- timestamp not in db").c_str())); @@ -1440,9 +1437,8 @@ size_t BlockchainLMDB::get_block_size(const uint64_t& height) const TXN_PREFIX_RDONLY(); RCURSOR(block_info); - MDB_val_copy key(height); - MDB_val result; - auto get_result = mdb_cursor_get(m_cur_block_info, &key, &result, MDB_SET); + MDB_val_set(result, height); + auto get_result = mdb_cursor_get(m_cur_block_info, (MDB_val *)&zerokval, &result, MDB_GET_BOTH); if (get_result == MDB_NOTFOUND) { throw0(BLOCK_DNE(std::string("Attempt to get block size from height ").append(boost::lexical_cast(height)).append(" failed -- block size not in db").c_str())); @@ -1464,9 +1460,8 @@ difficulty_type BlockchainLMDB::get_block_cumulative_difficulty(const uint64_t& TXN_PREFIX_RDONLY(); RCURSOR(block_info); - MDB_val_copy key(height); - MDB_val result; - auto get_result = mdb_cursor_get(m_cur_block_info, &key, &result, MDB_SET); + MDB_val_set(result, height); + auto get_result = mdb_cursor_get(m_cur_block_info, (MDB_val *)&zerokval, &result, MDB_GET_BOTH); if (get_result == MDB_NOTFOUND) { throw0(BLOCK_DNE(std::string("Attempt to get cumulative difficulty from height ").append(boost::lexical_cast(height)).append(" failed -- difficulty not in db").c_str())); @@ -1505,9 +1500,8 @@ uint64_t BlockchainLMDB::get_block_already_generated_coins(const uint64_t& heigh TXN_PREFIX_RDONLY(); RCURSOR(block_info); - MDB_val_copy key(height); - MDB_val result; - auto get_result = mdb_cursor_get(m_cur_block_info, &key, &result, MDB_SET); + MDB_val_set(result, height); + auto get_result = mdb_cursor_get(m_cur_block_info, (MDB_val *)&zerokval, &result, MDB_GET_BOTH); if (get_result == MDB_NOTFOUND) { throw0(BLOCK_DNE(std::string("Attempt to get generated coins from height ").append(boost::lexical_cast(height)).append(" failed -- block size not in db").c_str())); @@ -1529,9 +1523,8 @@ crypto::hash BlockchainLMDB::get_block_hash_from_height(const uint64_t& height) TXN_PREFIX_RDONLY(); RCURSOR(block_info); - MDB_val_copy key(height); - MDB_val result; - auto get_result = mdb_cursor_get(m_cur_block_info, &key, &result, MDB_SET); + MDB_val_set(result, height); + auto get_result = mdb_cursor_get(m_cur_block_info, (MDB_val *)&zerokval, &result, MDB_GET_BOTH); if (get_result == MDB_NOTFOUND) { throw0(BLOCK_DNE(std::string("Attempt to get hash from height ").append(boost::lexical_cast(height)).append(" failed -- hash not in db").c_str())); @@ -1856,10 +1849,9 @@ tx_out_index BlockchainLMDB::get_output_tx_and_index_from_global(const uint64_t& TXN_PREFIX_RDONLY(); RCURSOR(output_txs); - MDB_val_copy k(index); - MDB_val v; + MDB_val_set(v, index); - auto get_result = mdb_cursor_get(m_cur_output_txs, &k, &v, MDB_SET); + auto get_result = mdb_cursor_get(m_cur_output_txs, (MDB_val *)&zerokval, &v, MDB_GET_BOTH); if (get_result == MDB_NOTFOUND) throw1(OUTPUT_DNE("output with given index not in db")); else if (get_result) @@ -2437,10 +2429,9 @@ void BlockchainLMDB::get_output_tx_and_index_from_global(const std::vector k(index); - MDB_val v; + MDB_val_set(v, index); - auto get_result = mdb_cursor_get(m_cur_output_txs, &k, &v, MDB_SET); + auto get_result = mdb_cursor_get(m_cur_output_txs, (MDB_val *)&zerokval, &v, MDB_GET_BOTH); if (get_result == MDB_NOTFOUND) throw1(OUTPUT_DNE("output with given index not in db")); else if (get_result) diff --git a/src/blockchain_db/lmdb/db_lmdb.h b/src/blockchain_db/lmdb/db_lmdb.h index a298ee68f..2ac8b94d8 100644 --- a/src/blockchain_db/lmdb/db_lmdb.h +++ b/src/blockchain_db/lmdb/db_lmdb.h @@ -300,7 +300,6 @@ private: void remove_tx_outputs(const uint64_t tx_index, const transaction& tx); void remove_output(const uint64_t& out_index, const uint64_t amount); - void remove_amount_output_index(const uint64_t amount, const uint64_t global_output_index); virtual void add_spent_key(const crypto::key_image& k_image); From b2f1c588055c2b7bed7fa5ef9ec42ad796f2a834 Mon Sep 17 00:00:00 2001 From: Howard Chu Date: Thu, 31 Mar 2016 20:55:16 +0100 Subject: [PATCH 18/23] Use cursors in some remove functions Helps when they're called repeatedly in one txn --- src/blockchain_db/lmdb/db_lmdb.cpp | 40 ++++++++++++++++++++++++------ 1 file changed, 33 insertions(+), 7 deletions(-) diff --git a/src/blockchain_db/lmdb/db_lmdb.cpp b/src/blockchain_db/lmdb/db_lmdb.cpp index 1ccd477f3..b9e07427b 100644 --- a/src/blockchain_db/lmdb/db_lmdb.cpp +++ b/src/blockchain_db/lmdb/db_lmdb.cpp @@ -625,6 +625,8 @@ void BlockchainLMDB::remove_block() mdb_txn_cursors *m_cursors = &m_wcursors; CURSOR(block_info) + CURSOR(block_heights) + CURSOR(blocks) MDB_val_copy k(m_height - 1); MDB_val h = k; if ((result = mdb_cursor_get(m_cur_block_info, (MDB_val *)&zerokval, &h, MDB_GET_BOTH))) @@ -635,10 +637,14 @@ void BlockchainLMDB::remove_block() blk_height bh = {bi->bi_hash, 0}; h.mv_data = (void *)&bh; h.mv_size = sizeof(bh); - if ((result = mdb_del(*m_write_txn, m_block_heights, (MDB_val *)&zerokval, &h))) + if ((result = mdb_cursor_get(m_cur_block_heights, (MDB_val *)&zerokval, &h, MDB_GET_BOTH))) + throw1(DB_ERROR(lmdb_error("Failed to locate block height by hash for removal: ", result).c_str())); + if ((result = mdb_cursor_del(m_cur_block_heights, 0))) throw1(DB_ERROR(lmdb_error("Failed to add removal of block height by hash to db transaction: ", result).c_str())); - if ((result = mdb_del(*m_write_txn, m_blocks, &k, NULL))) + if ((result = mdb_cursor_get(m_cur_blocks, &k, NULL, MDB_SET))) + throw1(DB_ERROR(lmdb_error("Failed to locate block for removal: ", result).c_str())); + if ((result = mdb_cursor_del(m_cur_blocks, 0))) throw1(DB_ERROR(lmdb_error("Failed to add removal of block to db transaction: ", result).c_str())); if ((result = mdb_cursor_del(m_cur_block_info, 0))) @@ -699,6 +705,8 @@ void BlockchainLMDB::remove_transaction_data(const crypto::hash& tx_hash, const mdb_txn_cursors *m_cursors = &m_wcursors; CURSOR(tx_indices) + CURSOR(txs) + CURSOR(tx_outputs) txindex ti = {tx_hash}; MDB_val val_h = {sizeof(ti), (void *)&ti}; @@ -709,16 +717,25 @@ void BlockchainLMDB::remove_transaction_data(const crypto::hash& tx_hash, const uint64_t tx_index = tip->data.tx_index; MDB_val_copy val_tx_index(tx_index); - if ((result = mdb_del(*m_write_txn, m_txs, &val_tx_index, NULL))) + if ((result = mdb_cursor_get(m_cur_txs, &val_tx_index, NULL, MDB_SET))) + throw1(DB_ERROR(lmdb_error("Failed to locate tx for removal: ", result).c_str())); + result = mdb_cursor_del(m_cur_txs, 0); + if (result) throw1(DB_ERROR(lmdb_error("Failed to add removal of tx to db transaction: ", result).c_str())); remove_tx_outputs(tx_index, tx); - result = mdb_del(*m_write_txn, m_tx_outputs, &val_tx_index, NULL); + result = mdb_cursor_get(m_cur_tx_outputs, &val_tx_index, NULL, MDB_SET); if (result == MDB_NOTFOUND) LOG_PRINT_L1("tx has no outputs to remove: " << tx_hash); else if (result) - throw1(DB_ERROR(lmdb_error("Failed to add removal of tx outputs to db transaction: ", result).c_str())); + throw1(DB_ERROR(lmdb_error("Failed to locate tx outputs for removal: ", result).c_str())); + if (!result) + { + result = mdb_cursor_del(m_cur_tx_outputs, 0); + if (result) + throw1(DB_ERROR(lmdb_error("Failed to add removal of tx outputs to db transaction: ", result).c_str())); + } // Though other things could change, so long as earlier functions (like // remove_tx_outputs) need to do the lookup of tx hash -> tx index, don't @@ -906,11 +923,20 @@ void BlockchainLMDB::remove_spent_key(const crypto::key_image& k_image) { LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); + mdb_txn_cursors *m_cursors = &m_wcursors; + + CURSOR(spent_keys) MDB_val k = {sizeof(k_image), (void *)&k_image}; - auto result = mdb_del(*m_write_txn, m_spent_keys, (MDB_val *)&zerokval, &k); + auto result = mdb_cursor_get(m_cur_spent_keys, (MDB_val *)&zerokval, &k, MDB_GET_BOTH); if (result != 0 && result != MDB_NOTFOUND) - throw1(DB_ERROR(lmdb_error("Error adding removal of key image to db transaction", result).c_str())); + throw1(DB_ERROR(lmdb_error("Error finding spent key to remove", result).c_str())); + if (!result) + { + result = mdb_cursor_del(m_cur_spent_keys, 0); + if (result) + throw1(DB_ERROR(lmdb_error("Error adding removal of key image to db transaction", result).c_str())); + } } blobdata BlockchainLMDB::output_to_blob(const tx_out& output) const From 591e421875988ea63313a7da22062f62f06b30ab Mon Sep 17 00:00:00 2001 From: Howard Chu Date: Mon, 4 Apr 2016 02:10:58 +0100 Subject: [PATCH 19/23] Cleanup and clarify Try to rationalize the variable names, document usage. --- src/blockchain_db/blockchain_db.cpp | 10 +- src/blockchain_db/blockchain_db.h | 58 +++++---- src/blockchain_db/lmdb/db_lmdb.cpp | 188 +++++++++++++--------------- src/blockchain_db/lmdb/db_lmdb.h | 19 +-- 4 files changed, 129 insertions(+), 146 deletions(-) diff --git a/src/blockchain_db/blockchain_db.cpp b/src/blockchain_db/blockchain_db.cpp index 8cde4f138..68f635d18 100644 --- a/src/blockchain_db/blockchain_db.cpp +++ b/src/blockchain_db/blockchain_db.cpp @@ -82,21 +82,17 @@ void BlockchainDB::add_transaction(const crypto::hash& blk_hash, const transacti } } - uint64_t tx_index = add_transaction_data(blk_hash, tx, tx_hash); + uint64_t tx_id = add_transaction_data(blk_hash, tx, tx_hash); std::vector amount_output_indices; - std::vector global_output_indices; // iterate tx.vout using indices instead of C++11 foreach syntax because // we need the index for (uint64_t i = 0; i < tx.vout.size(); ++i) { - uint64_t amount_output_index, global_output_index; - add_output(tx_hash, tx.vout[i], i, tx.unlock_time, amount_output_index, global_output_index); - amount_output_indices.push_back(amount_output_index); - global_output_indices.push_back(global_output_index); + amount_output_indices.push_back(add_output(tx_hash, tx.vout[i], i, tx.unlock_time)); } - add_amount_and_global_output_indices(tx_index, amount_output_indices, global_output_indices); + add_tx_amount_output_indices(tx_id, amount_output_indices); } uint64_t BlockchainDB::add_block( const block& blk diff --git a/src/blockchain_db/blockchain_db.h b/src/blockchain_db/blockchain_db.h index 665cccd40..99b520d8b 100644 --- a/src/blockchain_db/blockchain_db.h +++ b/src/blockchain_db/blockchain_db.h @@ -63,6 +63,26 @@ * of this interface call to private members of the interface to be added * later that will then populate as needed. * + * Indices and Identifiers: + * The word "index" is used ambiguously throughout this code. It is + * particularly confusing when talking about the output or transaction + * tables since their indexing can refer to themselves or each other. + * I have attempted to clarify these usages here: + * + * Blocks, transactions, and outputs are all identified by a hash. + * For storage efficiency, a 64-bit integer ID is used instead of the hash + * inside the DB. Tables exist to map between hash and ID. A block ID is + * also referred to as its "height". Transactions and outputs generally are + * not referred to by ID outside of this module, but the tx ID is returned + * by tx_exists() and used by get_tx_amount_output_indices(). Like their + * corresponding hashes, IDs are globally unique. + * + * The remaining uses of the word "index" refer to local offsets, and are + * not globally unique. An "amount output index" N refers to the Nth output + * of a specific amount. An "output local index" N refers to the Nth output + * of a specific tx. + * + * * General: * open() * is_open() @@ -99,7 +119,7 @@ * void pop_block(block&, tx_list&) * * Transactions: - * bool tx_exists(hash) + * bool tx_exists(hash, tx_id) * uint64_t get_tx_unlock_time(hash) * tx get_tx(hash) * uint64_t get_tx_count() @@ -111,7 +131,7 @@ * pub_key get_output_key(amount, index) * hash,index get_output_tx_and_index_from_global(index) * hash,index get_output_tx_and_index(amount, index) - * vec get_tx_output_indices(tx_hash) + * vec get_tx_amount_output_indices(tx_id) * * * Spent Output Key Images: @@ -151,9 +171,9 @@ struct output_data_t #pragma pack(push, 1) struct tx_data_t { - uint64_t tx_index; + uint64_t tx_id; uint64_t unlock_time; - uint64_t height; + uint64_t block_id; }; #pragma pack(pop) @@ -299,25 +319,22 @@ private: // tells the subclass to remove data about the top block virtual void remove_block() = 0; - // tells the subclass to store the transaction and its metadata + // tells the subclass to store the transaction and its metadata, returns tx ID virtual uint64_t add_transaction_data(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash& tx_hash) = 0; // tells the subclass to remove data about a transaction virtual void remove_transaction_data(const crypto::hash& tx_hash, const transaction& tx) = 0; - // tells the subclass to store an output - virtual void add_output(const crypto::hash& tx_hash, + // tells the subclass to store an output, returns amount output index + virtual uint64_t add_output(const crypto::hash& tx_hash, const tx_out& tx_output, const uint64_t& local_index, - const uint64_t unlock_time, - uint64_t& amount_output_index, - uint64_t& global_output_index + const uint64_t unlock_time ) = 0; - // tells the subclass to store indices for a tx's outputs, both amount output indices and global output indices - virtual void add_amount_and_global_output_indices(const uint64_t tx_index, - const std::vector& amount_output_indices, - const std::vector& global_output_indices + // tells the subclass to store amount output indices for a tx's outputs + virtual void add_tx_amount_output_indices(const uint64_t tx_id, + const std::vector& amount_output_indices ) = 0; // tells the subclass to remove an output @@ -482,7 +499,7 @@ public: // return true if a transaction with hash exists virtual bool tx_exists(const crypto::hash& h) const = 0; - virtual bool tx_exists(const crypto::hash& h, uint64_t& tx_index) const = 0; + virtual bool tx_exists(const crypto::hash& h, uint64_t& tx_id) const = 0; // return unlock time of tx with hash virtual uint64_t get_tx_unlock_time(const crypto::hash& h) const = 0; @@ -523,16 +540,9 @@ public: virtual bool can_thread_bulk_indices() const = 0; - // return two vectors of indices: vector of amount output indices and global - // output indices, corresponding to each output in the transaction with hash - // - virtual void get_amount_and_global_output_indices(const uint64_t tx_index, - std::vector& amount_output_indices, - std::vector& global_output_indices) const = 0; - // return a vector of indices corresponding to the amount output index for - // each output in the transaction with hash - virtual std::vector get_tx_amount_output_indices(const uint64_t tx_index) const = 0; + // each output in the transaction with ID + virtual std::vector get_tx_amount_output_indices(const uint64_t tx_id) const = 0; // returns true if key image is present in spent key images storage virtual bool has_key_image(const crypto::key_image& img) const = 0; diff --git a/src/blockchain_db/lmdb/db_lmdb.cpp b/src/blockchain_db/lmdb/db_lmdb.cpp index b9e07427b..a9645a95e 100644 --- a/src/blockchain_db/lmdb/db_lmdb.cpp +++ b/src/blockchain_db/lmdb/db_lmdb.cpp @@ -145,9 +145,34 @@ int compare_string(const MDB_val *a, const MDB_val *b) return strcmp(va, vb); } +/* DB schema: + * + * Table Key Data + * ----- --- ---- + * blocks block ID block blob + * block_heights block hash block height + * block_info block ID {block metadata} + * + * txs txn ID txn blob + * tx_indices txn hash {txn ID, metadata} + * tx_outputs txn ID [txn amount output indices] + * + * output_txs output ID {txn hash, local index} + * output_amounts amount [{amount output index, metadata}...] + * + * spent_keys output hash - + * + * Note: where the data items are of uniform size, DUPFIXED tables have + * been used to save space. In most of these cases, a dummy "zerokval" + * key is used when accessing the table; the Key listed above will be + * attached as a prefix on the Data to serve as the DUPSORT key. + * (DUPFIXED saves 8 bytes per record.) + * + * The output_amounts table doesn't use a dummy key, but uses DUPSORT. + */ const char* const LMDB_BLOCKS = "blocks"; -const char* const LMDB_BLOCK_INFO = "block_info"; const char* const LMDB_BLOCK_HEIGHTS = "block_heights"; +const char* const LMDB_BLOCK_INFO = "block_info"; const char* const LMDB_TXS = "txs"; const char* const LMDB_TX_INDICES = "tx_indices"; @@ -215,8 +240,8 @@ typedef struct mdb_block_info } mdb_block_info; typedef struct blk_height { - crypto::hash key; - uint64_t height; + crypto::hash bh_hash; + uint64_t bh_height; } blk_height; typedef struct txindex { @@ -226,12 +251,12 @@ typedef struct txindex { typedef struct outkey { uint64_t amount_index; - uint64_t tx_index; + uint64_t output_id; output_data_t data; } outkey; typedef struct outtx { - uint64_t txnum; + uint64_t output_id; crypto::hash tx_hash; uint64_t local_index; } outtx; @@ -560,14 +585,13 @@ void BlockchainLMDB::add_block(const block& blk, const size_t& block_size, const CURSOR(block_heights) blk_height bh = {blk_hash, m_height}; - MDB_val val_h = {sizeof(bh), (void *)&bh}; + MDB_val_set(val_h, bh); if (mdb_cursor_get(m_cur_block_heights, (MDB_val *)&zerokval, &val_h, MDB_GET_BOTH) == 0) throw1(BLOCK_EXISTS("Attempting to add block that's already in the db")); if (m_height > 0) { - blk_height ph = {blk.prev_id, 0}; - MDB_val parent_key = {sizeof(ph), (void *)&ph}; + MDB_val_set(parent_key, blk.prev_id); int result = mdb_cursor_get(m_cur_block_heights, (MDB_val *)&zerokval, &parent_key, MDB_GET_BOTH); if (result) { @@ -576,13 +600,13 @@ void BlockchainLMDB::add_block(const block& blk, const size_t& block_size, const throw0(DB_ERROR(lmdb_error("Failed to get top block hash to check for new block's parent: ", result).c_str())); } blk_height *prev = (blk_height *)parent_key.mv_data; - if (prev->height != m_height - 1) + if (prev->bh_height != m_height - 1) throw0(BLOCK_PARENT_DNE("Top block is not new block's parent")); } int result = 0; - MDB_val_copy key(m_height); + MDB_val_set(key, m_height); CURSOR(blocks) CURSOR(block_info) @@ -600,7 +624,7 @@ void BlockchainLMDB::add_block(const block& blk, const size_t& block_size, const bi.bi_diff = cumulative_difficulty; bi.bi_hash = blk_hash; - MDB_val val = {sizeof(bi), (void *)&bi}; + MDB_val_set(val, bi); result = mdb_cursor_put(m_cur_block_info, (MDB_val *)&zerokval, &val, MDB_APPENDDUP); if (result) throw0(DB_ERROR(lmdb_error("Failed to add block info to db transaction: ", result).c_str())); @@ -657,26 +681,27 @@ uint64_t BlockchainLMDB::add_transaction_data(const crypto::hash& blk_hash, cons check_open(); mdb_txn_cursors *m_cursors = &m_wcursors; - int result = 0; - uint64_t tx_index = m_num_txs; + int result; + uint64_t tx_id = m_num_txs; CURSOR(txs) CURSOR(tx_indices) - txindex ti = {tx_hash}; - MDB_val_copy val_tx_index(tx_index); - MDB_val val_h = {sizeof(ti), (void *)&ti}; + MDB_val_set(val_tx_id, tx_id); + MDB_val_set(val_h, tx_hash); result = mdb_cursor_get(m_cur_tx_indices, (MDB_val *)&zerokval, &val_h, MDB_GET_BOTH); if (result == 0) { txindex *tip = (txindex *)val_h.mv_data; - throw1(TX_EXISTS(std::string("Attempting to add transaction that's already in the db (tx index ").append(boost::lexical_cast(tip->data.tx_index)).append(")").c_str())); + throw1(TX_EXISTS(std::string("Attempting to add transaction that's already in the db (tx id ").append(boost::lexical_cast(tip->data.tx_id)).append(")").c_str())); } else if (result != MDB_NOTFOUND) { throw1(DB_ERROR(lmdb_error(std::string("Error checking if tx index exists for tx hash ") + epee::string_tools::pod_to_hex(tx_hash) + ": ", result).c_str())); } - ti.data.tx_index = tx_index; + txindex ti; + ti.key = tx_hash; + ti.data.tx_id = tx_id; ti.data.unlock_time = tx.unlock_time; - ti.data.height = m_height; + ti.data.block_id = m_height; // we don't need blk_hash since we know m_height val_h.mv_size = sizeof(ti); val_h.mv_data = (void *)&ti; @@ -686,12 +711,12 @@ uint64_t BlockchainLMDB::add_transaction_data(const crypto::hash& blk_hash, cons throw0(DB_ERROR(lmdb_error("Failed to add tx data to db transaction: ", result).c_str())); MDB_val_copy blob(tx_to_blob(tx)); - result = mdb_cursor_put(m_cur_txs, &val_tx_index, &blob, MDB_APPEND); + result = mdb_cursor_put(m_cur_txs, &val_tx_id, &blob, MDB_APPEND); if (result) throw0(DB_ERROR(lmdb_error("Failed to add tx blob to db transaction: ", result).c_str())); m_num_txs++; - return tx_index; + return tx_id; } // TODO: compare pros and cons of looking up the tx hash's tx index once and @@ -708,24 +733,22 @@ void BlockchainLMDB::remove_transaction_data(const crypto::hash& tx_hash, const CURSOR(txs) CURSOR(tx_outputs) - txindex ti = {tx_hash}; - MDB_val val_h = {sizeof(ti), (void *)&ti}; + MDB_val_set(val_h, tx_hash); if (mdb_cursor_get(m_cur_tx_indices, (MDB_val *)&zerokval, &val_h, MDB_GET_BOTH)) throw1(TX_DNE("Attempting to remove transaction that isn't in the db")); txindex *tip = (txindex *)val_h.mv_data; - uint64_t tx_index = tip->data.tx_index; - MDB_val_copy val_tx_index(tx_index); + MDB_val_set(val_tx_id, tip->data.tx_id); - if ((result = mdb_cursor_get(m_cur_txs, &val_tx_index, NULL, MDB_SET))) + if ((result = mdb_cursor_get(m_cur_txs, &val_tx_id, NULL, MDB_SET))) throw1(DB_ERROR(lmdb_error("Failed to locate tx for removal: ", result).c_str())); result = mdb_cursor_del(m_cur_txs, 0); if (result) throw1(DB_ERROR(lmdb_error("Failed to add removal of tx to db transaction: ", result).c_str())); - remove_tx_outputs(tx_index, tx); + remove_tx_outputs(tip->data.tx_id, tx); - result = mdb_cursor_get(m_cur_tx_outputs, &val_tx_index, NULL, MDB_SET); + result = mdb_cursor_get(m_cur_tx_outputs, &val_tx_id, NULL, MDB_SET); if (result == MDB_NOTFOUND) LOG_PRINT_L1("tx has no outputs to remove: " << tx_hash); else if (result) @@ -737,24 +760,17 @@ void BlockchainLMDB::remove_transaction_data(const crypto::hash& tx_hash, const throw1(DB_ERROR(lmdb_error("Failed to add removal of tx outputs to db transaction: ", result).c_str())); } - // Though other things could change, so long as earlier functions (like - // remove_tx_outputs) need to do the lookup of tx hash -> tx index, don't - // delete the tx_indices entry until the end. - if (mdb_cursor_get(m_cur_tx_indices, (MDB_val *)&zerokval, &val_h, MDB_GET_BOTH)) - throw1(TX_DNE("Attempting to remove transaction that isn't in the db")); + // Don't delete the tx_indices entry until the end, after we're done with val_tx_id if (mdb_cursor_del(m_cur_tx_indices, 0)) throw1(DB_ERROR("Failed to add removal of tx index to db transaction")); m_num_txs--; } -// global_output_index is no longer used for locating outputs -void BlockchainLMDB::add_output(const crypto::hash& tx_hash, +uint64_t BlockchainLMDB::add_output(const crypto::hash& tx_hash, const tx_out& tx_output, const uint64_t& local_index, - const uint64_t unlock_time, - uint64_t& amount_output_index, - uint64_t& global_output_index) + const uint64_t unlock_time) { LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); @@ -791,7 +807,7 @@ void BlockchainLMDB::add_output(const crypto::hash& tx_hash, throw0(DB_ERROR(lmdb_error("Failed to get output amount in db transaction: ", result).c_str())); else ok.amount_index = 0; - ok.tx_index = m_num_outputs; + ok.output_id = m_num_outputs; ok.data.pubkey = boost::get < txout_to_key > (tx_output.target).key; ok.data.unlock_time = unlock_time; ok.data.height = m_height; @@ -801,16 +817,12 @@ void BlockchainLMDB::add_output(const crypto::hash& tx_hash, if ((result = mdb_cursor_put(m_cur_output_amounts, &val_amount, &data, MDB_APPENDDUP))) throw0(DB_ERROR(lmdb_error("Failed to add output pubkey to db transaction: ", result).c_str())); - amount_output_index = ok.amount_index; - global_output_index = m_num_outputs; - m_num_outputs++; + return ok.amount_index; } -// global_output_indices is now ignored -void BlockchainLMDB::add_amount_and_global_output_indices(const uint64_t tx_index, - const std::vector& amount_output_indices, - const std::vector& global_output_indices) +void BlockchainLMDB::add_tx_amount_output_indices(const uint64_t tx_id, + const std::vector& amount_output_indices) { LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); @@ -821,24 +833,22 @@ void BlockchainLMDB::add_amount_and_global_output_indices(const uint64_t tx_inde int num_outputs = amount_output_indices.size(); - MDB_val_copy k_tx_index(tx_index); + MDB_val_set(k_tx_id, tx_id); MDB_val v; v.mv_data = (void *)amount_output_indices.data(); v.mv_size = sizeof(uint64_t) * num_outputs; // LOG_PRINT_L1("tx_outputs[tx_hash] size: " << v.mv_size); - result = mdb_cursor_put(m_cur_tx_outputs, &k_tx_index, &v, MDB_APPEND); + result = mdb_cursor_put(m_cur_tx_outputs, &k_tx_id, &v, MDB_APPEND); if (result) throw0(DB_ERROR(std::string("Failed to add to db transaction: ").append(mdb_strerror(result)).c_str())); } -void BlockchainLMDB::remove_tx_outputs(const uint64_t tx_index, const transaction& tx) +void BlockchainLMDB::remove_tx_outputs(const uint64_t tx_id, const transaction& tx) { LOG_PRINT_L3("BlockchainLMDB::" << __func__); - // only need amount_output_indices - std::vector amount_output_indices, global_output_indices; - get_amount_and_global_output_indices(tx_index, amount_output_indices, global_output_indices); + std::vector amount_output_indices = get_tx_amount_output_indices(tx_id); if (amount_output_indices.empty()) { @@ -880,7 +890,7 @@ void BlockchainLMDB::remove_output(const uint64_t& out_index, const uint64_t amo throw0(DB_ERROR(lmdb_error("DB error attempting to get an output", result).c_str())); outkey *ok = (outkey *)v.mv_data; - MDB_val_set(otxk, ok->tx_index); + MDB_val_set(otxk, ok->output_id); result = mdb_cursor_get(m_cur_output_txs, (MDB_val *)&zerokval, &otxk, MDB_GET_BOTH); if (result == MDB_NOTFOUND) { @@ -1333,8 +1343,7 @@ bool BlockchainLMDB::block_exists(const crypto::hash& h) const RCURSOR(block_heights); bool ret = false; - blk_height bh = {h, 0}; - MDB_val key = {sizeof(bh), (void *)&bh}; + MDB_val_set(key, h); auto get_result = mdb_cursor_get(m_cur_block_heights, (MDB_val *)&zerokval, &key, MDB_GET_BOTH); if (get_result == MDB_NOTFOUND) { @@ -1365,8 +1374,7 @@ uint64_t BlockchainLMDB::get_block_height(const crypto::hash& h) const TXN_PREFIX_RDONLY(); RCURSOR(block_heights); - blk_height bh = {h, 0}; - MDB_val key = {sizeof(bh), (void *)&bh}; + MDB_val_set(key, h); auto get_result = mdb_cursor_get(m_cur_block_heights, (MDB_val *)&zerokval, &key, MDB_GET_BOTH); if (get_result == MDB_NOTFOUND) throw1(BLOCK_DNE("Attempted to retrieve non-existent block height")); @@ -1374,7 +1382,7 @@ uint64_t BlockchainLMDB::get_block_height(const crypto::hash& h) const throw0(DB_ERROR("Error attempting to retrieve a block height from the db")); blk_height *bhp = (blk_height *)key.mv_data; - uint64_t ret = bhp->height; + uint64_t ret = bhp->bh_height; TXN_POSTFIX_RDONLY(); return ret; } @@ -1635,7 +1643,7 @@ bool BlockchainLMDB::tx_exists(const crypto::hash& h) const RCURSOR(tx_indices); RCURSOR(txs); - MDB_val_copy key(h); + MDB_val_set(key, h); bool tx_found = false; TIME_MEASURE_START(time1); @@ -1666,7 +1674,7 @@ bool BlockchainLMDB::tx_exists(const crypto::hash& h) const return true; } -bool BlockchainLMDB::tx_exists(const crypto::hash& h, uint64_t& tx_index) const +bool BlockchainLMDB::tx_exists(const crypto::hash& h, uint64_t& tx_id) const { LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); @@ -1674,8 +1682,7 @@ bool BlockchainLMDB::tx_exists(const crypto::hash& h, uint64_t& tx_index) const TXN_PREFIX_RDONLY(); RCURSOR(tx_indices); - txindex ti = {h}; - MDB_val v = {sizeof(ti), (void *)&ti}; + MDB_val_set(v, h); TIME_MEASURE_START(time1); auto get_result = mdb_cursor_get(m_cur_tx_indices, (MDB_val *)&zerokval, &v, MDB_GET_BOTH); @@ -1683,7 +1690,7 @@ bool BlockchainLMDB::tx_exists(const crypto::hash& h, uint64_t& tx_index) const time_tx_exists += time1; if (!get_result) { txindex *tip = (txindex *)v.mv_data; - tx_index = tip->data.tx_index; + tx_id = tip->data.tx_id; } TXN_POSTFIX_RDONLY(); @@ -1709,8 +1716,7 @@ uint64_t BlockchainLMDB::get_tx_unlock_time(const crypto::hash& h) const TXN_PREFIX_RDONLY(); RCURSOR(tx_indices); - txindex ti = {h}; - MDB_val v = {sizeof(ti), (void *)&ti}; + MDB_val_set(v, h); auto get_result = mdb_cursor_get(m_cur_tx_indices, (MDB_val *)&zerokval, &v, MDB_GET_BOTH); if (get_result == MDB_NOTFOUND) throw1(TX_DNE(lmdb_error(std::string("tx data with hash ") + epee::string_tools::pod_to_hex(h) + " not found in db: ", get_result).c_str())); @@ -1732,16 +1738,14 @@ transaction BlockchainLMDB::get_tx(const crypto::hash& h) const RCURSOR(tx_indices); RCURSOR(txs); - txindex ti = {h}; - MDB_val v = {sizeof(ti), (void *)&ti}; + MDB_val_set(v, h); MDB_val result; auto get_result = mdb_cursor_get(m_cur_tx_indices, (MDB_val *)&zerokval, &v, MDB_GET_BOTH); if (get_result == 0) { txindex *tip = (txindex *)v.mv_data; - uint64_t tx_index = tip->data.tx_index; - MDB_val_copy val_tx_index(tx_index); - get_result = mdb_cursor_get(m_cur_txs, &val_tx_index, &result, MDB_SET); + MDB_val_set(val_tx_id, tip->data.tx_id); + get_result = mdb_cursor_get(m_cur_txs, &val_tx_id, &result, MDB_SET); } if (get_result == MDB_NOTFOUND) throw1(TX_DNE(std::string("tx with hash ").append(epee::string_tools::pod_to_hex(h)).append(" not found in db").c_str())); @@ -1798,8 +1802,7 @@ uint64_t BlockchainLMDB::get_tx_block_height(const crypto::hash& h) const TXN_PREFIX_RDONLY(); RCURSOR(tx_indices); - txindex ti = {h}; - MDB_val v = {sizeof(ti), (void *)&ti}; + MDB_val_set(v, h); auto get_result = mdb_cursor_get(m_cur_tx_indices, (MDB_val *)&zerokval, &v, MDB_GET_BOTH); if (get_result == MDB_NOTFOUND) { @@ -1809,7 +1812,7 @@ uint64_t BlockchainLMDB::get_tx_block_height(const crypto::hash& h) const throw0(DB_ERROR(lmdb_error("DB error attempting to fetch tx height from hash", get_result).c_str())); txindex *tip = (txindex *)v.mv_data; - uint64_t ret = tip->data.height; + uint64_t ret = tip->data.block_id; TXN_POSTFIX_RDONLY(); return ret; } @@ -1867,7 +1870,7 @@ output_data_t BlockchainLMDB::get_output_key(const uint64_t& amount, const uint6 return ret; } -tx_out_index BlockchainLMDB::get_output_tx_and_index_from_global(const uint64_t& index) const +tx_out_index BlockchainLMDB::get_output_tx_and_index_from_global(const uint64_t& output_id) const { LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); @@ -1875,7 +1878,7 @@ tx_out_index BlockchainLMDB::get_output_tx_and_index_from_global(const uint64_t& TXN_PREFIX_RDONLY(); RCURSOR(output_txs); - MDB_val_set(v, index); + MDB_val_set(v, output_id); auto get_result = mdb_cursor_get(m_cur_output_txs, (MDB_val *)&zerokval, &v, MDB_GET_BOTH); if (get_result == MDB_NOTFOUND) @@ -1903,29 +1906,21 @@ tx_out_index BlockchainLMDB::get_output_tx_and_index(const uint64_t& amount, con return indices[0]; } -// we don't have global_output_indices -void BlockchainLMDB::get_amount_and_global_output_indices(const uint64_t tx_index, - std::vector& amount_output_indices, - std::vector& global_output_indices) const +std::vector BlockchainLMDB::get_tx_amount_output_indices(const uint64_t tx_id) const { LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); - // If a new txn is created, it only needs to read. - // - // This must existence of m_write_txn too (not only m_batch_active), as - // that's what remove_tx_outputs() expected to use instead of creating a new - // txn, regardless of batch mode. Otherwise, remove_tx_outputs() would now - // create a new read-only txn here, which is incorrect. TXN_PREFIX_RDONLY(); RCURSOR(tx_outputs); int result = 0; - MDB_val_copy k_tx_index(tx_index); + MDB_val_set(k_tx_id, tx_id); MDB_val v; + std::vector amount_output_indices; - result = mdb_cursor_get(m_cur_tx_outputs, &k_tx_index, &v, MDB_SET); + result = mdb_cursor_get(m_cur_tx_outputs, &k_tx_id, &v, MDB_SET); if (result == MDB_NOTFOUND) LOG_PRINT_L0("WARNING: Unexpected: tx has no amount indices stored in " "tx_outputs, but it should have an empty entry even if it's a tx without " @@ -1944,21 +1939,10 @@ void BlockchainLMDB::get_amount_and_global_output_indices(const uint64_t tx_inde indices = nullptr; TXN_POSTFIX_RDONLY(); -} - -std::vector BlockchainLMDB::get_tx_amount_output_indices(const uint64_t tx_index) const -{ - LOG_PRINT_L3("BlockchainLMDB::" << __func__); - - std::vector amount_output_indices, global_output_indices; - // only need amount_output_indices - get_amount_and_global_output_indices(tx_index, amount_output_indices, global_output_indices); - return amount_output_indices; } - bool BlockchainLMDB::has_key_image(const crypto::key_image& img) const { LOG_PRINT_L3("BlockchainLMDB::" << __func__); @@ -2110,7 +2094,7 @@ bool BlockchainLMDB::for_all_outputs(std::functiontx_index); + tx_out_index toi = get_output_tx_and_index_from_global(ok->output_id); if (!f(amount, toi.first, toi.second)) { ret = false; break; @@ -2453,9 +2437,9 @@ void BlockchainLMDB::get_output_tx_and_index_from_global(const std::vectortx_index); + tx_indices.push_back(okp->output_id); } TIME_MEASURE_START(db3); diff --git a/src/blockchain_db/lmdb/db_lmdb.h b/src/blockchain_db/lmdb/db_lmdb.h index 2ac8b94d8..534f5575f 100644 --- a/src/blockchain_db/lmdb/db_lmdb.h +++ b/src/blockchain_db/lmdb/db_lmdb.h @@ -227,11 +227,7 @@ public: virtual tx_out_index get_output_tx_and_index(const uint64_t& amount, const uint64_t& index); virtual void get_output_tx_and_index(const uint64_t& amount, const std::vector &offsets, std::vector &indices); - virtual void get_amount_and_global_output_indices(const uint64_t tx_index, - std::vector& amount_output_indices, - std::vector& global_output_indices) const; - - virtual std::vector get_tx_amount_output_indices(const uint64_t tx_index) const; + virtual std::vector get_tx_amount_output_indices(const uint64_t tx_id) const; virtual bool has_key_image(const crypto::key_image& img) const; @@ -282,22 +278,19 @@ private: virtual void remove_transaction_data(const crypto::hash& tx_hash, const transaction& tx); - virtual void add_output(const crypto::hash& tx_hash, + virtual uint64_t add_output(const crypto::hash& tx_hash, const tx_out& tx_output, const uint64_t& local_index, - const uint64_t unlock_time, - uint64_t& amount_output_index, - uint64_t& global_output_index + const uint64_t unlock_time ); - virtual void add_amount_and_global_output_indices(const uint64_t tx_index, - const std::vector& amount_output_indices, - const std::vector& global_output_indices + virtual void add_tx_amount_output_indices(const uint64_t tx_id, + const std::vector& amount_output_indices ); virtual void remove_output(const tx_out& tx_output); - void remove_tx_outputs(const uint64_t tx_index, const transaction& tx); + void remove_tx_outputs(const uint64_t tx_id, const transaction& tx); void remove_output(const uint64_t& out_index, const uint64_t amount); From 372acee72303b4d7ea225981c523d328fda297be Mon Sep 17 00:00:00 2001 From: Howard Chu Date: Mon, 4 Apr 2016 17:28:31 +0100 Subject: [PATCH 20/23] Cleanup drop obsolete remove_output() fix get_output_key(global), fix crash in blockchain_dump --- src/blockchain_db/blockchain_db.h | 3 -- src/blockchain_db/lmdb/db_lmdb.cpp | 82 +++++++++++++++++++++++------- src/blockchain_db/lmdb/db_lmdb.h | 4 +- tests/unit_tests/hardfork.cpp | 9 ++-- 4 files changed, 70 insertions(+), 28 deletions(-) diff --git a/src/blockchain_db/blockchain_db.h b/src/blockchain_db/blockchain_db.h index 99b520d8b..93527b484 100644 --- a/src/blockchain_db/blockchain_db.h +++ b/src/blockchain_db/blockchain_db.h @@ -337,9 +337,6 @@ private: const std::vector& amount_output_indices ) = 0; - // tells the subclass to remove an output - virtual void remove_output(const tx_out& tx_output) = 0; - // tells the subclass to store a spent key virtual void add_spent_key(const crypto::key_image& k_image) = 0; diff --git a/src/blockchain_db/lmdb/db_lmdb.cpp b/src/blockchain_db/lmdb/db_lmdb.cpp index a9645a95e..f9deca0b4 100644 --- a/src/blockchain_db/lmdb/db_lmdb.cpp +++ b/src/blockchain_db/lmdb/db_lmdb.cpp @@ -861,18 +861,11 @@ void BlockchainLMDB::remove_tx_outputs(const uint64_t tx_id, const transaction& for (uint64_t i = tx.vout.size(); i > 0; --i) { const tx_out tx_output = tx.vout[i-1]; - remove_output(amount_output_indices[i-1], tx_output.amount); + remove_output(tx_output.amount, amount_output_indices[i-1]); } } -// TODO: probably remove this function -void BlockchainLMDB::remove_output(const tx_out& tx_output) -{ - LOG_PRINT_L3("BlockchainLMDB::" << __func__ << " (unused version - does nothing)"); - return; -} - -void BlockchainLMDB::remove_output(const uint64_t& out_index, const uint64_t amount) +void BlockchainLMDB::remove_output(const uint64_t amount, const uint64_t& out_index) { LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); @@ -1841,12 +1834,54 @@ uint64_t BlockchainLMDB::get_num_outputs(const uint64_t& amount) const return num_elems; } -// TODO: probably remove this function +// This is a lot harder now that we've removed the output_keys index output_data_t BlockchainLMDB::get_output_key(const uint64_t &global_index) const { LOG_PRINT_L3("BlockchainLMDB::" << __func__ << " (unused version - does nothing)"); - outkey ok = {0}; - return ok.data; + check_open(); + TXN_PREFIX_RDONLY(); + RCURSOR(output_txs); + RCURSOR(tx_indices); + RCURSOR(txs); + + output_data_t od; + MDB_val_set(v, global_index); + auto get_result = mdb_cursor_get(m_cur_output_txs, (MDB_val *)&zerokval, &v, MDB_GET_BOTH); + if (get_result == MDB_NOTFOUND) + throw1(OUTPUT_DNE("output with given index not in db")); + else if (get_result) + throw0(DB_ERROR("DB error attempting to fetch output tx hash")); + + outtx *ot = (outtx *)v.mv_data; + + MDB_val_set(val_h, ot->tx_hash); + get_result = mdb_cursor_get(m_cur_tx_indices, (MDB_val *)&zerokval, &val_h, MDB_GET_BOTH); + if (get_result) + throw0(DB_ERROR(lmdb_error(std::string("DB error attempting to fetch transaction index from hash ") + epee::string_tools::pod_to_hex(ot->tx_hash) + ": ", get_result).c_str())); + + txindex *tip = (txindex *)val_h.mv_data; + MDB_val_set(val_tx_id, tip->data.tx_id); + MDB_val result; + get_result = mdb_cursor_get(m_cur_txs, &val_tx_id, &result, MDB_SET); + if (get_result == MDB_NOTFOUND) + throw1(TX_DNE(std::string("tx with hash ").append(epee::string_tools::pod_to_hex(ot->tx_hash)).append(" not found in db").c_str())); + else if (get_result) + throw0(DB_ERROR(lmdb_error("DB error attempting to fetch tx from hash", get_result).c_str())); + + blobdata bd; + bd.assign(reinterpret_cast(result.mv_data), result.mv_size); + + transaction tx; + if (!parse_and_validate_tx_from_blob(bd, tx)) + throw0(DB_ERROR("Failed to parse tx from blob retrieved from the db")); + + const tx_out tx_output = tx.vout[ot->local_index]; + od.unlock_time = tip->data.unlock_time; + od.height = tip->data.block_id; + od.pubkey = boost::get(tx_output.target).key; + + TXN_POSTFIX_RDONLY(); + return od; } output_data_t BlockchainLMDB::get_output_key(const uint64_t& amount, const uint64_t& index) @@ -1968,19 +2003,20 @@ bool BlockchainLMDB::for_all_key_images(std::functionkey; + k.mv_data = (void *)&ti->data.tx_id; + k.mv_size = sizeof(ti->data.tx_id); + ret = mdb_cursor_get(m_cur_txs, &k, &v, MDB_SET); + if (ret == MDB_NOTFOUND) + break; + if (ret) + throw0(DB_ERROR(lmdb_error("Failed to enumerate transactions: ", ret).c_str())); blobdata bd; bd.assign(reinterpret_cast(v.mv_data), v.mv_size); transaction tx; diff --git a/src/blockchain_db/lmdb/db_lmdb.h b/src/blockchain_db/lmdb/db_lmdb.h index 534f5575f..1f8c3bbef 100644 --- a/src/blockchain_db/lmdb/db_lmdb.h +++ b/src/blockchain_db/lmdb/db_lmdb.h @@ -288,11 +288,9 @@ private: const std::vector& amount_output_indices ); - virtual void remove_output(const tx_out& tx_output); - void remove_tx_outputs(const uint64_t tx_id, const transaction& tx); - void remove_output(const uint64_t& out_index, const uint64_t amount); + void remove_output(const uint64_t amount, const uint64_t& out_index); virtual void add_spent_key(const crypto::key_image& k_image); diff --git a/tests/unit_tests/hardfork.cpp b/tests/unit_tests/hardfork.cpp index 50e0e5ae8..1faef2082 100644 --- a/tests/unit_tests/hardfork.cpp +++ b/tests/unit_tests/hardfork.cpp @@ -78,6 +78,7 @@ public: virtual block get_top_block() const { return block(); } virtual uint64_t height() const { return blocks.size(); } virtual bool tx_exists(const crypto::hash& h) const { return false; } + virtual bool tx_exists(const crypto::hash& h, uint64_t& tx_index) const { return false; } virtual uint64_t get_tx_unlock_time(const crypto::hash& h) const { return 0; } virtual transaction get_tx(const crypto::hash& h) const { return transaction(); } virtual uint64_t get_tx_count() const { return 0; } @@ -93,13 +94,13 @@ public: virtual void get_output_key(const uint64_t &amount, const std::vector &offsets, std::vector &outputs) {} virtual bool can_thread_bulk_indices() const { return false; } virtual std::vector get_tx_output_indices(const crypto::hash& h) const { return std::vector(); } - virtual std::vector get_tx_amount_output_indices(const crypto::hash& h) const { return std::vector(); } + virtual std::vector get_tx_amount_output_indices(const uint64_t tx_index) const { return std::vector(); } virtual bool has_key_image(const crypto::key_image& img) const { return false; } virtual void remove_block() { blocks.pop_back(); } - virtual void add_transaction_data(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash& tx_hash) {} + virtual uint64_t add_transaction_data(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash& tx_hash) {return 0;} virtual void remove_transaction_data(const crypto::hash& tx_hash, const transaction& tx) {} - virtual void add_output(const crypto::hash& tx_hash, const tx_out& tx_output, const uint64_t& local_index, const uint64_t unlock_time) {} - virtual void remove_output(const tx_out& tx_output) {} + virtual uint64_t add_output(const crypto::hash& tx_hash, const tx_out& tx_output, const uint64_t& local_index, const uint64_t unlock_time) {return 0;} + virtual void add_tx_amount_output_indices(const uint64_t tx_index, const std::vector& amount_output_indices) {} virtual void add_spent_key(const crypto::key_image& k_image) {} virtual void remove_spent_key(const crypto::key_image& k_image) {} From c14f9efd52084cee21ff9072c3a903eb33635b1d Mon Sep 17 00:00:00 2001 From: Howard Chu Date: Wed, 6 Apr 2016 18:05:20 +0100 Subject: [PATCH 21/23] Migration Migrate from DB version 0 to version 1 on startup --- src/blockchain_db/lmdb/db_lmdb.cpp | 1048 +++++++++++++++++++++++++++- src/blockchain_db/lmdb/db_lmdb.h | 6 + 2 files changed, 1049 insertions(+), 5 deletions(-) diff --git a/src/blockchain_db/lmdb/db_lmdb.cpp b/src/blockchain_db/lmdb/db_lmdb.cpp index 4320b12bf..3ab2b7154 100644 --- a/src/blockchain_db/lmdb/db_lmdb.cpp +++ b/src/blockchain_db/lmdb/db_lmdb.cpp @@ -160,7 +160,7 @@ int compare_string(const MDB_val *a, const MDB_val *b) * output_txs output ID {txn hash, local index} * output_amounts amount [{amount output index, metadata}...] * - * spent_keys output hash - + * spent_keys input hash - * * Note: where the data items are of uniform size, DUPFIXED tables have * been used to save space. In most of these cases, a dummy "zerokval" @@ -1114,8 +1114,8 @@ void BlockchainLMDB::open(const std::string& filename, const int mdb_flags) m_height = db_stats.ms_entries; // get and keep current number of txs - if ((result = mdb_stat(txn, m_tx_indices, &db_stats))) - throw0(DB_ERROR(lmdb_error("Failed to query m_tx_indices: ", result).c_str())); + if ((result = mdb_stat(txn, m_txs, &db_stats))) + throw0(DB_ERROR(lmdb_error("Failed to query m_txs: ", result).c_str())); m_num_txs = db_stats.ms_entries; // get and keep current number of outputs @@ -1138,13 +1138,21 @@ void BlockchainLMDB::open(const std::string& filename, const int mdb_flags) #if VERSION > 0 else if (*(const uint32_t*)v.mv_data < VERSION) { - compatible = false; + // Note that there was a schema change within version 0 as well. + // See commit e5d2680094ee15889934fe28901e4e133cda56f2 2015/07/10 + // We don't handle the old format previous to that commit. + txn.commit(); + migrate(*(const uint32_t *)v.mv_data); + m_open = true; + return; } #endif } else { - // if not found, but we're on version 0, it's fine. If the DB's empty, it's fine too. + // if not found, and the DB is non-empty, this is probably + // an "old" version 0, which we don't handle. If the DB is + // empty it's fine. if (VERSION > 0 && m_height > 0) compatible = false; } @@ -2769,4 +2777,1034 @@ void BlockchainLMDB::fixup() BlockchainDB::fixup(); } +static int compare_amtidx(const MDB_val *a, const MDB_val *b) +{ + const uint64_t *pa = (const uint64_t *)a->mv_data; + const uint64_t *pb = (const uint64_t *)b->mv_data; + return (pa[0] < pb[1]) ? -1 : pa[0] > pb[1]; +} + +#define RENAME_DB(name) \ + k.mv_data = (void *)name; \ + k.mv_size = sizeof(name)-1; \ + result = mdb_cursor_open(txn, 1, &c_cur); \ + if (result) \ + throw0(DB_ERROR(lmdb_error("Failed to open a cursor for " name ": ", result).c_str())); \ + result = mdb_cursor_get(c_cur, &k, NULL, MDB_SET_KEY); \ + if (result) \ + throw0(DB_ERROR(lmdb_error("Failed to get DB record for " name ": ", result).c_str())); \ + ptr = (char *)k.mv_data; \ + ptr[sizeof(name)-2] = 's' + +#define LOGIF(y) if (y <= epee::log_space::log_singletone::get_log_detalisation_level()) + +void BlockchainLMDB::migrate_0_1() +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + uint64_t i, z; + int result; + mdb_txn_safe txn(false); + MDB_val k, v; + char *ptr; + + LOG_PRINT_YELLOW("Migrating blockchain from DB version 0 to 1 - this may take a while:", LOG_LEVEL_0); + LOG_PRINT_L0("updating blocks, hf_versions, outputs, txs, and spent_keys tables..."); + + LOG_PRINT_L0("Total number of blocks: " << m_height); + LOG_PRINT_L1("block migration will update block_heights, block_info, and hf_versions..."); + + do { + LOG_PRINT_L1("migrating block_heights:"); + MDB_dbi o_heights; + + unsigned int flags; + result = mdb_txn_begin(m_env, NULL, 0, txn); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str())); + result = mdb_dbi_flags(txn, m_block_heights, &flags); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to retrieve block_heights flags: ", result).c_str())); + /* if the flags are what we expect, this table has already been migrated */ + if ((flags & (MDB_INTEGERKEY|MDB_DUPSORT|MDB_DUPFIXED)) == (MDB_INTEGERKEY|MDB_DUPSORT|MDB_DUPFIXED)) { + txn.abort(); + LOG_PRINT_L1(" block_heights already migrated"); + break; + } + + /* the block_heights table name is the same but the old version and new version + * have incompatible DB flags. Create a new table with the right flags. We want + * the name to be similar to the old name so that it will occupy the same location + * in the DB. + */ + o_heights = m_block_heights; + lmdb_db_open(txn, "block_heightr", MDB_INTEGERKEY | MDB_CREATE | MDB_DUPSORT | MDB_DUPFIXED, m_block_heights, "Failed to open db handle for block_heightr"); + mdb_set_dupsort(txn, m_block_heights, compare_hash32); + + MDB_cursor *c_old, *c_cur; + blk_height bh; + MDB_val_set(nv, bh); + + /* old table was k(hash), v(height). + * new table is DUPFIXED, k(zeroval), v{hash, height}. + */ + i = 0; + z = m_height; + while(1) { + if (!(i % 2000)) { + if (i) { + LOGIF(1) { + std::cout << i << " / " << z << " \r" << std::flush; + } + txn.commit(); + result = mdb_txn_begin(m_env, NULL, 0, txn); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str())); + } + result = mdb_cursor_open(txn, m_block_heights, &c_cur); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to open a cursor for block_heightr: ", result).c_str())); + result = mdb_cursor_open(txn, o_heights, &c_old); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to open a cursor for block_heights: ", result).c_str())); + if (!i) { + MDB_stat ms; + mdb_stat(txn, m_block_heights, &ms); + i = ms.ms_entries; + } + } + result = mdb_cursor_get(c_old, &k, &v, MDB_NEXT); + if (result == MDB_NOTFOUND) { + txn.commit(); + break; + } + else if (result) + throw0(DB_ERROR(lmdb_error("Failed to get a record from block_heights: ", result).c_str())); + bh.bh_hash = *(crypto::hash *)k.mv_data; + bh.bh_height = *(uint64_t *)v.mv_data; + result = mdb_cursor_put(c_cur, (MDB_val *)&zerokval, &nv, MDB_APPENDDUP); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to put a record into block_heightr: ", result).c_str())); + /* we delete the old records immediately, so the overall DB and mapsize should not grow. + * This is a little slower than just letting mdb_drop() delete it all at the end, but + * it saves a significant amount of disk space. + */ + result = mdb_cursor_del(c_old, 0); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to delete a record from block_heights: ", result).c_str())); + i++; + } + + result = mdb_txn_begin(m_env, NULL, 0, txn); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str())); + /* Delete the old table */ + result = mdb_drop(txn, o_heights, 1); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to delete old block_heights table: ", result).c_str())); + + RENAME_DB("block_heightr"); + + /* close and reopen to get old dbi slot back */ + mdb_dbi_close(m_env, m_block_heights); + lmdb_db_open(txn, "block_heights", MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, m_block_heights, "Failed to open db handle for block_heights"); + mdb_set_dupsort(txn, m_block_heights, compare_hash32); + txn.commit(); + + } while(0); + + /* old tables are k(height), v(value). + * new table is DUPFIXED, k(zeroval), v{height, values...}. + */ + do { + LOG_PRINT_L1("migrating block info:"); + + MDB_dbi coins; + result = mdb_txn_begin(m_env, NULL, 0, txn); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str())); + result = mdb_dbi_open(txn, "block_coins", 0, &coins); + if (result == MDB_NOTFOUND) { + txn.abort(); + LOG_PRINT_L1(" block_info already migrated"); + break; + } + MDB_dbi diffs, hashes, sizes, timestamps; + mdb_block_info bi; + MDB_val_set(nv, bi); + + lmdb_db_open(txn, "block_diffs", 0, diffs, "Failed to open db handle for block_diffs"); + lmdb_db_open(txn, "block_hashes", 0, hashes, "Failed to open db handle for block_hashes"); + lmdb_db_open(txn, "block_sizes", 0, sizes, "Failed to open db handle for block_sizes"); + lmdb_db_open(txn, "block_timestamps", 0, timestamps, "Failed to open db handle for block_timestamps"); + MDB_cursor *c_cur, *c_coins, *c_diffs, *c_hashes, *c_sizes, *c_timestamps; + i = 0; + z = m_height; + while(1) { + MDB_val k, v; + if (!(i % 2000)) { + if (i) { + LOGIF(1) { + std::cout << i << " / " << z << " \r" << std::flush; + } + txn.commit(); + result = mdb_txn_begin(m_env, NULL, 0, txn); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str())); + } + result = mdb_cursor_open(txn, m_block_info, &c_cur); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to open a cursor for block_info: ", result).c_str())); + result = mdb_cursor_open(txn, coins, &c_coins); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to open a cursor for block_coins: ", result).c_str())); + result = mdb_cursor_open(txn, diffs, &c_diffs); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to open a cursor for block_diffs: ", result).c_str())); + result = mdb_cursor_open(txn, hashes, &c_hashes); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to open a cursor for block_hashes: ", result).c_str())); + result = mdb_cursor_open(txn, sizes, &c_sizes); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to open a cursor for block_coins: ", result).c_str())); + result = mdb_cursor_open(txn, timestamps, &c_timestamps); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to open a cursor for block_timestamps: ", result).c_str())); + if (!i) { + MDB_stat ms; + mdb_stat(txn, m_block_info, &ms); + i = ms.ms_entries; + } + } + result = mdb_cursor_get(c_coins, &k, &v, MDB_NEXT); + if (result == MDB_NOTFOUND) { + break; + } else if (result) + throw0(DB_ERROR(lmdb_error("Failed to get a record from block_coins: ", result).c_str())); + bi.bi_height = *(uint64_t *)k.mv_data; + bi.bi_coins = *(uint64_t *)v.mv_data; + result = mdb_cursor_get(c_diffs, &k, &v, MDB_NEXT); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to get a record from block_diffs: ", result).c_str())); + bi.bi_diff = *(uint64_t *)v.mv_data; + result = mdb_cursor_get(c_hashes, &k, &v, MDB_NEXT); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to get a record from block_hashes: ", result).c_str())); + bi.bi_hash = *(crypto::hash *)v.mv_data; + result = mdb_cursor_get(c_sizes, &k, &v, MDB_NEXT); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to get a record from block_sizes: ", result).c_str())); + if (v.mv_size == sizeof(uint32_t)) + bi.bi_size = *(uint32_t *)v.mv_data; + else + bi.bi_size = *(uint64_t *)v.mv_data; // this is a 32/64 compat bug in version 0 + result = mdb_cursor_get(c_timestamps, &k, &v, MDB_NEXT); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to get a record from block_timestamps: ", result).c_str())); + bi.bi_timestamp = *(uint64_t *)v.mv_data; + result = mdb_cursor_put(c_cur, (MDB_val *)&zerokval, &nv, MDB_APPENDDUP); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to put a record into block_info: ", result).c_str())); + result = mdb_cursor_del(c_coins, 0); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to delete a record from block_coins: ", result).c_str())); + result = mdb_cursor_del(c_diffs, 0); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to delete a record from block_diffs: ", result).c_str())); + result = mdb_cursor_del(c_hashes, 0); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to delete a record from block_hashes: ", result).c_str())); + result = mdb_cursor_del(c_sizes, 0); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to delete a record from block_sizes: ", result).c_str())); + result = mdb_cursor_del(c_timestamps, 0); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to delete a record from block_timestamps: ", result).c_str())); + i++; + } + mdb_cursor_close(c_timestamps); + mdb_cursor_close(c_sizes); + mdb_cursor_close(c_hashes); + mdb_cursor_close(c_diffs); + mdb_cursor_close(c_coins); + result = mdb_drop(txn, timestamps, 1); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to delete block_timestamps from the db: ", result).c_str())); + result = mdb_drop(txn, sizes, 1); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to delete block_sizes from the db: ", result).c_str())); + result = mdb_drop(txn, hashes, 1); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to delete block_hashes from the db: ", result).c_str())); + result = mdb_drop(txn, diffs, 1); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to delete block_diffs from the db: ", result).c_str())); + result = mdb_drop(txn, coins, 1); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to delete block_coins from the db: ", result).c_str())); + txn.commit(); + } while(0); + + do { + LOG_PRINT_L1("migrating hf_versions:"); + MDB_dbi o_hfv; + + unsigned int flags; + result = mdb_txn_begin(m_env, NULL, 0, txn); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str())); + result = mdb_dbi_flags(txn, m_hf_versions, &flags); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to retrieve hf_versions flags: ", result).c_str())); + /* if the flags are what we expect, this table has already been migrated */ + if (flags & MDB_INTEGERKEY) { + txn.abort(); + LOG_PRINT_L1(" hf_versions already migrated"); + break; + } + + /* the hf_versions table name is the same but the old version and new version + * have incompatible DB flags. Create a new table with the right flags. + */ + o_hfv = m_hf_versions; + lmdb_db_open(txn, "hf_versionr", MDB_INTEGERKEY | MDB_CREATE, m_hf_versions, "Failed to open db handle for hf_versionr"); + + MDB_cursor *c_old, *c_cur; + i = 0; + z = m_height; + + while(1) { + if (!(i % 2000)) { + if (i) { + LOGIF(1) { + std::cout << i << " / " << z << " \r" << std::flush; + } + txn.commit(); + result = mdb_txn_begin(m_env, NULL, 0, txn); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str())); + } + result = mdb_cursor_open(txn, m_hf_versions, &c_cur); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to open a cursor for spent_keyr: ", result).c_str())); + result = mdb_cursor_open(txn, o_hfv, &c_old); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to open a cursor for spent_keys: ", result).c_str())); + if (!i) { + MDB_stat ms; + mdb_stat(txn, m_hf_versions, &ms); + i = ms.ms_entries; + } + } + result = mdb_cursor_get(c_old, &k, &v, MDB_NEXT); + if (result == MDB_NOTFOUND) { + txn.commit(); + break; + } + else if (result) + throw0(DB_ERROR(lmdb_error("Failed to get a record from hf_versions: ", result).c_str())); + result = mdb_cursor_put(c_cur, &k, &v, MDB_APPEND); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to put a record into hf_versionr: ", result).c_str())); + result = mdb_cursor_del(c_old, 0); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to delete a record from hf_versions: ", result).c_str())); + i++; + } + + result = mdb_txn_begin(m_env, NULL, 0, txn); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str())); + /* Delete the old table */ + result = mdb_drop(txn, o_hfv, 1); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to delete old hf_versions table: ", result).c_str())); + RENAME_DB("hf_versionr"); + mdb_dbi_close(m_env, m_hf_versions); + lmdb_db_open(txn, "hf_versions", MDB_INTEGERKEY, m_hf_versions, "Failed to open db handle for hf_versions"); + + txn.commit(); + } while(0); + + LOG_PRINT_L0("Total number of outputs: " << m_num_outputs); + LOG_PRINT_L1("outputs migration will update output_amounts and output_txs..."); + + do { + LOG_PRINT_L1("migrating output_amounts:"); + + MDB_dbi keys; + result = mdb_txn_begin(m_env, NULL, 0, txn); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str())); + result = mdb_dbi_open(txn, "output_keys", 0, &keys); + if (result == MDB_NOTFOUND) { + txn.abort(); + LOG_PRINT_L1(" output_amounts already migrated"); + break; + } + MDB_dbi o_amts; + outkey ok; + MDB_val_set(nv, ok); + + /* the output_amounts table name is the same but the old version and new version + * have incompatible DB flags. Create a new table with the right flags. + */ + o_amts = m_output_amounts; + lmdb_db_open(txn, "output_amountr", MDB_INTEGERKEY | MDB_CREATE | MDB_DUPSORT | MDB_DUPFIXED, m_output_amounts, "Failed to open db handle for output_amountr"); + mdb_set_dupsort(txn, m_output_amounts, compare_uint64); + + MDB_cursor *c_oamts, *c_keys, *c_cur; + i = 0; + z = m_num_outputs; + uint64_t j; + mdb_size_t num_elems; + MDB_val v2; + + while(1) { + if (!(i % 1000)) { + if (i) { + LOGIF(1) { + std::cout << i << " / " << z << " \r" << std::flush; + } + txn.commit(); + result = mdb_txn_begin(m_env, NULL, 0, txn); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str())); + } + result = mdb_cursor_open(txn, m_output_amounts, &c_cur); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to open a cursor for output_amountr: ", result).c_str())); + result = mdb_cursor_open(txn, o_amts, &c_oamts); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to open a cursor for output_amounts: ", result).c_str())); + result = mdb_cursor_open(txn, keys, &c_keys); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to open a cursor for output_keys: ", result).c_str())); + if (!i) { + MDB_stat ms; + mdb_stat(txn, m_output_amounts, &ms); + i = ms.ms_entries; + } + } + result = mdb_cursor_get(c_oamts, &k, &v, MDB_FIRST); + if (result == MDB_NOTFOUND) { + txn.commit(); + break; + } else if (result) + throw0(DB_ERROR(lmdb_error("Failed to get a record from output_amounts: ", result).c_str())); + result = mdb_cursor_count(c_oamts, &num_elems); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to get number of outputs for amount: ", result).c_str())); + for (j=0; j(v.mv_data), v.mv_size); + if (!parse_and_validate_block_from_blob(bd, b)) + throw0(DB_ERROR("Failed to parse block from blob retrieved from the db")); + + ti.key = get_transaction_hash(b.miner_tx); + ti.data.tx_id = num_txs++; + ti.data.block_id = *(uint64_t *)k.mv_data; + ti.data.unlock_time = b.miner_tx.unlock_time; + i++; + + result = mdb_cursor_put(c_cur, (MDB_val *)&zerokval, &iv, 0); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to put a record into tx_indices: ", result).c_str())); + + result = mdb_cursor_del(c_tx_unlocks, 0); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to delete a record from tx_unlocks: ", result).c_str())); + result = mdb_cursor_del(c_tx_heights, 0); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to delete a record from tx_heights: ", result).c_str())); + + for (j=0; jkey; + k.mv_size = sizeof(txp->key); + ik.mv_data = (void *)&txp->data.tx_id; + ik.mv_size = sizeof(txp->data.tx_id); + + result = mdb_cursor_get(c_txs, &k, &v, MDB_FIRST); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to get a record from txs: ", result).c_str())); + result = mdb_cursor_put(c_cur, &ik, &v, 0); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to put a record into txr: ", result).c_str())); + + result = mdb_cursor_get(c_tx_outputs, &k, &iv, MDB_SET); + if (!result) { + blobdata bd; + bd.assign(reinterpret_cast(v.mv_data), v.mv_size); + + transaction tx; + if (!parse_and_validate_tx_from_blob(bd, tx)) + throw0(DB_ERROR("Failed to parse tx from blob retrieved from the db")); + + /* turn global output indices into amount output indices */ + std::vector indices; + int j; + for (j=0;;j++) { + MDB_val_set(vk, tx.vout[j].amount); + result = mdb_cursor_get(c_oamts, &vk, &iv, MDB_GET_BOTH); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to get a record from output_amounts: ", result).c_str())); + + outkey *ok = (outkey *)v.mv_data; + indices.push_back(ok->amount_index); + result = mdb_cursor_get(c_tx_outputs, &k, &iv, MDB_NEXT_DUP); + if (result == MDB_NOTFOUND) + break; + else if (result) + throw0(DB_ERROR(lmdb_error("Failed to get a record from tx_outputs: ", result).c_str())); + } + v.mv_data = (void *)indices.data(); + v.mv_size = sizeof(uint64_t) * indices.size(); + result = mdb_cursor_put(c_curtxo, &ik, &v, 0); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to put a record into tx_outputr: ", result).c_str())); + result = mdb_cursor_del(c_tx_outputs, MDB_NODUPDATA); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to delete a record from tx_outputs: ", result).c_str())); + } else if (result == MDB_NOTFOUND) { + /* get_tx_amount_output_indices expects a record here, even if it's empty */ + v.mv_size = 0; + v.mv_data = NULL; + result = mdb_cursor_put(c_curtxo, &ik, &v, 0); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to put a record into tx_outputr: ", result).c_str())); + } else + throw0(DB_ERROR(lmdb_error("Failed to get a record from tx_outputs: ", result).c_str())); + + result = mdb_cursor_del(c_txs, 0); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to delete a record from txs: ", result).c_str())); + i++; + } + result = mdb_txn_begin(m_env, NULL, 0, txn); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str())); + result = mdb_drop(txn, o_txs, 1); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to delete txs from the db: ", result).c_str())); + result = mdb_drop(txn, o_tx_outputs, 1); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to delete tx_outputs from the db: ", result).c_str())); + + RENAME_DB("txr"); + RENAME_DB("tx_outputr"); + + mdb_dbi_close(m_env, m_txs); + mdb_dbi_close(m_env, m_tx_outputs); + + lmdb_db_open(txn, "txs", MDB_INTEGERKEY, m_txs, "Failed to open db handle for txs"); + lmdb_db_open(txn, "tx_outputs", MDB_INTEGERKEY, m_tx_outputs, "Failed to open db handle for tx_outputs"); + mdb_set_dupsort(txn, m_output_amounts, compare_uint64); + + txn.commit(); + } while(0); + + do { + LOG_PRINT_L1("migrating spent_keys:"); + MDB_dbi o_keys; + + unsigned int flags; + result = mdb_txn_begin(m_env, NULL, 0, txn); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str())); + result = mdb_dbi_flags(txn, m_spent_keys, &flags); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to retrieve spent_keys flags: ", result).c_str())); + /* if the flags are what we expect, this table has already been migrated */ + if ((flags & (MDB_INTEGERKEY|MDB_DUPSORT|MDB_DUPFIXED)) == (MDB_INTEGERKEY|MDB_DUPSORT|MDB_DUPFIXED)) { + txn.abort(); + LOG_PRINT_L1(" spent_keys already migrated"); + break; + } + + /* the spent_keys table name is the same but the old version and new version + * have incompatible DB flags. Create a new table with the right flags. + */ + o_keys = m_spent_keys; + lmdb_db_open(txn, "spent_keyr", MDB_INTEGERKEY | MDB_CREATE | MDB_DUPSORT | MDB_DUPFIXED, m_spent_keys, "Failed to open db handle for spent_keyr"); + mdb_set_dupsort(txn, m_spent_keys, compare_hash32); + + MDB_cursor *c_old, *c_cur; + i = 0; + MDB_stat ms; + mdb_stat(txn, o_keys, &ms); + z = ms.ms_entries; + mdb_stat(txn, m_spent_keys, &ms); + z += ms.ms_entries; + + while(1) { + if (!(i % 2000)) { + if (i) { + LOGIF(1) { + std::cout << i << " / " << z << " \r" << std::flush; + } + txn.commit(); + result = mdb_txn_begin(m_env, NULL, 0, txn); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str())); + } + result = mdb_cursor_open(txn, m_spent_keys, &c_cur); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to open a cursor for spent_keyr: ", result).c_str())); + result = mdb_cursor_open(txn, o_keys, &c_old); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to open a cursor for spent_keys: ", result).c_str())); + if (!i) + i = ms.ms_entries; + } + result = mdb_cursor_get(c_old, &k, NULL, MDB_NEXT); + if (result == MDB_NOTFOUND) { + txn.commit(); + break; + } + else if (result) + throw0(DB_ERROR(lmdb_error("Failed to get a record from spent_keys: ", result).c_str())); + result = mdb_cursor_put(c_cur, (MDB_val *)&zerokval, &k, MDB_APPENDDUP); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to put a record into spent_keyr: ", result).c_str())); + result = mdb_cursor_del(c_old, 0); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to delete a record from spent_keys: ", result).c_str())); + i++; + } + + result = mdb_txn_begin(m_env, NULL, 0, txn); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str())); + /* Delete the old table */ + result = mdb_drop(txn, o_keys, 1); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to delete old spent_keys table: ", result).c_str())); + RENAME_DB("spent_keyr"); + mdb_dbi_close(m_env, m_spent_keys); + lmdb_db_open(txn, "spent_keys", MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, m_spent_keys, "Failed to open db handle for spent_keys"); + mdb_set_dupsort(txn, m_spent_keys, compare_hash32); + + txn.commit(); + } while(0); + + uint32_t version = 1; + v.mv_data = (void *)&version; + v.mv_size = sizeof(version); + MDB_val_copy vk("version"); + result = mdb_txn_begin(m_env, NULL, 0, txn); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str())); + result = mdb_put(txn, m_properties, &vk, &v, 0); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to update version for the db: ", result).c_str())); + txn.commit(); +} + +void BlockchainLMDB::migrate(const uint32_t oldversion) +{ + switch(oldversion) { + case 0: + migrate_0_1(); /* FALLTHRU */ + default: + ; + } +} + } // namespace cryptonote diff --git a/src/blockchain_db/lmdb/db_lmdb.h b/src/blockchain_db/lmdb/db_lmdb.h index 8021a652c..c7121bf63 100644 --- a/src/blockchain_db/lmdb/db_lmdb.h +++ b/src/blockchain_db/lmdb/db_lmdb.h @@ -339,6 +339,12 @@ private: // fix up anything that may be wrong due to past bugs virtual void fixup(); + // migrate from older DB version to current + void migrate(const uint32_t oldversion); + + // migrate from DB version 0 to 1 + void migrate_0_1(); + MDB_env* m_env; MDB_dbi m_blocks; From 66b1e13aa750777a80815c0c9dd7f535b7434c19 Mon Sep 17 00:00:00 2001 From: Howard Chu Date: Sat, 9 Apr 2016 20:44:05 +0100 Subject: [PATCH 22/23] mdb_drop optimization If we know there are no sub-DBs and no overflow pages, skip leaf scan. --- external/db_drivers/liblmdb/mdb.c | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/external/db_drivers/liblmdb/mdb.c b/external/db_drivers/liblmdb/mdb.c index acab07b82..2d8e458f7 100644 --- a/external/db_drivers/liblmdb/mdb.c +++ b/external/db_drivers/liblmdb/mdb.c @@ -10501,8 +10501,11 @@ mdb_drop0(MDB_cursor *mc, int subs) /* DUPSORT sub-DBs have no ovpages/DBs. Omit scanning leaves. * This also avoids any P_LEAF2 pages, which have no nodes. + * Also if the DB doesn't have sub-DBs and has no overflow + * pages, omit scanning leaves. */ - if (mc->mc_flags & C_SUB) + if ((mc->mc_flags & C_SUB) || + (!subs && !mc->mc_db->md_overflow_pages)) mdb_cursor_pop(mc); mdb_cursor_copy(mc, &mx); @@ -10529,6 +10532,9 @@ mdb_drop0(MDB_cursor *mc, int subs) pg, omp->mp_pages); if (rc) goto done; + mc->mc_db->md_overflow_pages -= omp->mp_pages; + if (!mc->mc_db->md_overflow_pages && !subs) + break; } else if (subs && (ni->mn_flags & F_SUBDATA)) { mdb_xcursor_init1(mc, ni); rc = mdb_drop0(&mc->mc_xcursor->mx_cursor, 0); @@ -10536,6 +10542,8 @@ mdb_drop0(MDB_cursor *mc, int subs) goto done; } } + if (!subs && !mc->mc_db->md_overflow_pages) + goto pop; } else { if ((rc = mdb_midl_need(&txn->mt_free_pgs, n)) != 0) goto done; @@ -10557,6 +10565,7 @@ mdb_drop0(MDB_cursor *mc, int subs) /* no more siblings, go back to beginning * of previous level. */ +pop: mdb_cursor_pop(mc); mc->mc_ki[0] = 0; for (i=1; imc_snum; i++) { From 2b0fa05f0d0d3c2aae90225d786d9ba7fcd6a2ff Mon Sep 17 00:00:00 2001 From: Howard Chu Date: Sun, 10 Apr 2016 17:25:13 +0100 Subject: [PATCH 23/23] Another take on migration Delete old indices and recreate them, rather than updating them Maybe not quite as slow as before. --- src/blockchain_db/blockchain_db.h | 24 +- src/blockchain_db/lmdb/db_lmdb.cpp | 682 +++++------------------------ 2 files changed, 115 insertions(+), 591 deletions(-) diff --git a/src/blockchain_db/blockchain_db.h b/src/blockchain_db/blockchain_db.h index 7edef51ea..1445dd13c 100644 --- a/src/blockchain_db/blockchain_db.h +++ b/src/blockchain_db/blockchain_db.h @@ -462,18 +462,6 @@ private: */ void pop_block(); - /** - * @brief helper function for add_transactions, to add each individual transaction - * - * This function is called by add_transactions() for each transaction to be - * added. - * - * @param blk_hash hash of the block which has the transaction - * @param tx the transaction to add - * @param tx_hash_ptr the hash of the transaction, if already calculated - */ - void add_transaction(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash* tx_hash_ptr = NULL); - // helper function to remove transaction from blockchain /** * @brief helper function to remove transaction from the blockchain @@ -492,6 +480,18 @@ private: protected: + /** + * @brief helper function for add_transactions, to add each individual transaction + * + * This function is called by add_transactions() for each transaction to be + * added. + * + * @param blk_hash hash of the block which has the transaction + * @param tx the transaction to add + * @param tx_hash_ptr the hash of the transaction, if already calculated + */ + void add_transaction(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash* tx_hash_ptr = NULL); + mutable uint64_t time_tx_exists = 0; //!< a performance metric uint64_t time_commit1 = 0; //!< a performance metric bool m_auto_remove_logs = true; //!< whether or not to automatically remove old logs diff --git a/src/blockchain_db/lmdb/db_lmdb.cpp b/src/blockchain_db/lmdb/db_lmdb.cpp index 3ab2b7154..8c51c09b1 100644 --- a/src/blockchain_db/lmdb/db_lmdb.cpp +++ b/src/blockchain_db/lmdb/db_lmdb.cpp @@ -1142,8 +1142,8 @@ void BlockchainLMDB::open(const std::string& filename, const int mdb_flags) // See commit e5d2680094ee15889934fe28901e4e133cda56f2 2015/07/10 // We don't handle the old format previous to that commit. txn.commit(); - migrate(*(const uint32_t *)v.mv_data); m_open = true; + migrate(*(const uint32_t *)v.mv_data); return; } #endif @@ -2777,13 +2777,6 @@ void BlockchainLMDB::fixup() BlockchainDB::fixup(); } -static int compare_amtidx(const MDB_val *a, const MDB_val *b) -{ - const uint64_t *pa = (const uint64_t *)a->mv_data; - const uint64_t *pb = (const uint64_t *)b->mv_data; - return (pa[0] < pb[1]) ? -1 : pa[0] > pb[1]; -} - #define RENAME_DB(name) \ k.mv_data = (void *)name; \ k.mv_size = sizeof(name)-1; \ @@ -3125,395 +3118,63 @@ void BlockchainLMDB::migrate_0_1() txn.commit(); } while(0); - LOG_PRINT_L0("Total number of outputs: " << m_num_outputs); - LOG_PRINT_L1("outputs migration will update output_amounts and output_txs..."); - do { - LOG_PRINT_L1("migrating output_amounts:"); + LOG_PRINT_L1("deleting old indices:"); - MDB_dbi keys; + /* Delete all other tables, we're just going to recreate them */ + MDB_dbi dbi; result = mdb_txn_begin(m_env, NULL, 0, txn); if (result) throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str())); - result = mdb_dbi_open(txn, "output_keys", 0, &keys); + + result = mdb_dbi_open(txn, "tx_unlocks", 0, &dbi); if (result == MDB_NOTFOUND) { - txn.abort(); - LOG_PRINT_L1(" output_amounts already migrated"); - break; - } - MDB_dbi o_amts; - outkey ok; - MDB_val_set(nv, ok); - - /* the output_amounts table name is the same but the old version and new version - * have incompatible DB flags. Create a new table with the right flags. - */ - o_amts = m_output_amounts; - lmdb_db_open(txn, "output_amountr", MDB_INTEGERKEY | MDB_CREATE | MDB_DUPSORT | MDB_DUPFIXED, m_output_amounts, "Failed to open db handle for output_amountr"); - mdb_set_dupsort(txn, m_output_amounts, compare_uint64); - - MDB_cursor *c_oamts, *c_keys, *c_cur; - i = 0; - z = m_num_outputs; - uint64_t j; - mdb_size_t num_elems; - MDB_val v2; - - while(1) { - if (!(i % 1000)) { - if (i) { - LOGIF(1) { - std::cout << i << " / " << z << " \r" << std::flush; - } - txn.commit(); - result = mdb_txn_begin(m_env, NULL, 0, txn); - if (result) - throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str())); - } - result = mdb_cursor_open(txn, m_output_amounts, &c_cur); - if (result) - throw0(DB_ERROR(lmdb_error("Failed to open a cursor for output_amountr: ", result).c_str())); - result = mdb_cursor_open(txn, o_amts, &c_oamts); - if (result) - throw0(DB_ERROR(lmdb_error("Failed to open a cursor for output_amounts: ", result).c_str())); - result = mdb_cursor_open(txn, keys, &c_keys); - if (result) - throw0(DB_ERROR(lmdb_error("Failed to open a cursor for output_keys: ", result).c_str())); - if (!i) { - MDB_stat ms; - mdb_stat(txn, m_output_amounts, &ms); - i = ms.ms_entries; - } - } - result = mdb_cursor_get(c_oamts, &k, &v, MDB_FIRST); - if (result == MDB_NOTFOUND) { - txn.commit(); + txn.abort(); + LOG_PRINT_L1(" old indices already deleted"); break; - } else if (result) - throw0(DB_ERROR(lmdb_error("Failed to get a record from output_amounts: ", result).c_str())); - result = mdb_cursor_count(c_oamts, &num_elems); - if (result) - throw0(DB_ERROR(lmdb_error("Failed to get number of outputs for amount: ", result).c_str())); - for (j=0; j(v.mv_data), v.mv_size); - if (!parse_and_validate_block_from_blob(bd, b)) - throw0(DB_ERROR("Failed to parse block from blob retrieved from the db")); - - ti.key = get_transaction_hash(b.miner_tx); - ti.data.tx_id = num_txs++; - ti.data.block_id = *(uint64_t *)k.mv_data; - ti.data.unlock_time = b.miner_tx.unlock_time; - i++; - - result = mdb_cursor_put(c_cur, (MDB_val *)&zerokval, &iv, 0); - if (result) - throw0(DB_ERROR(lmdb_error("Failed to put a record into tx_indices: ", result).c_str())); - - result = mdb_cursor_del(c_tx_unlocks, 0); - if (result) - throw0(DB_ERROR(lmdb_error("Failed to delete a record from tx_unlocks: ", result).c_str())); - result = mdb_cursor_del(c_tx_heights, 0); - if (result) - throw0(DB_ERROR(lmdb_error("Failed to delete a record from tx_heights: ", result).c_str())); - - for (j=0; jm_txn; + m_height = 0; while(1) { if (!(i % 1000)) { @@ -3560,122 +3217,80 @@ void BlockchainLMDB::migrate_0_1() LOGIF(1) { std::cout << i << " / " << z << " \r" << std::flush; } + MDB_val_set(pk, "txblk"); + MDB_val_set(pv, m_height); + result = mdb_cursor_put(c_props, &pk, &pv, 0); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to update txblk property: ", result).c_str())); txn.commit(); result = mdb_txn_begin(m_env, NULL, 0, txn); if (result) throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str())); + m_write_txn->m_txn = txn.m_txn; + m_write_batch_txn->m_txn = txn.m_txn; + memset(&m_wcursors, 0, sizeof(m_wcursors)); } - result = mdb_cursor_open(txn, m_txs, &c_cur); + result = mdb_cursor_open(txn, m_blocks, &c_blocks); if (result) - throw0(DB_ERROR(lmdb_error("Failed to open a cursor for txr: ", result).c_str())); + throw0(DB_ERROR(lmdb_error("Failed to open a cursor for blocks: ", result).c_str())); + result = mdb_cursor_open(txn, m_properties, &c_props); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to open a cursor for properties: ", result).c_str())); result = mdb_cursor_open(txn, o_txs, &c_txs); if (result) throw0(DB_ERROR(lmdb_error("Failed to open a cursor for txs: ", result).c_str())); - result = mdb_cursor_open(txn, o_tx_outputs, &c_tx_outputs); - if (result) - throw0(DB_ERROR(lmdb_error("Failed to open a cursor for tx_outputs: ", result).c_str())); - result = mdb_cursor_open(txn, m_tx_outputs, &c_curtxo); - if (result) - throw0(DB_ERROR(lmdb_error("Failed to open a cursor for tx_outputr: ", result).c_str())); - result = mdb_cursor_open(txn, m_tx_indices, &c_txi); - if (result) - throw0(DB_ERROR(lmdb_error("Failed to open a cursor for tx_indices: ", result).c_str())); - result = mdb_cursor_open(txn, m_output_amounts, &c_oamts); - if (result) - throw0(DB_ERROR(lmdb_error("Failed to open a cursor for output_amounts: ", result).c_str())); if (!i) { MDB_stat ms; + mdb_stat(txn, m_output_txs, &ms); + m_num_outputs = ms.ms_entries; mdb_stat(txn, m_txs, &ms); - i = ms.ms_entries; + m_num_txs = i = ms.ms_entries; + if (i) { + m_num_txs = i; + MDB_val_set(pk, "txblk"); + result = mdb_cursor_get(c_props, &pk, &k, MDB_SET); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to get a record from properties: ", result).c_str())); + m_height = *(uint64_t *)k.mv_data; + } } if (i) { - result = mdb_cursor_get(c_txs, &k, &v, MDB_FIRST); + result = mdb_cursor_get(c_blocks, &k, &v, MDB_SET); if (result) - throw0(DB_ERROR(lmdb_error("Failed to get a record from txs: ", result).c_str())); - result = mdb_cursor_get(c_txi, (MDB_val *)&zerokval, &k, MDB_GET_BOTH); - if (result) - throw0(DB_ERROR(lmdb_error("Failed to get a record from tx_indices: ", result).c_str())); - result = mdb_cursor_get(c_txi, &k, &v, MDB_PREV); - if (result) - throw0(DB_ERROR(lmdb_error("Failed to get a record from tx_indices: ", result).c_str())); + throw0(DB_ERROR(lmdb_error("Failed to get a record from blocks: ", result).c_str())); } } - /* The new tx_indices and the old txs tables are both sorted using - * compare_hash32 so we can walk thru them sequentially and they will - * stay in lock step with each other. Unfortunately, even though the - * old tx_outputs was keyed with the tx hash, it wasn't set to use - * the compare_hash32 function, so its records are in some other random - * hash order. This costs us the majority of CPU time when walking - * thru that table. Iterating with MDB_NEXT is cheap/free, but searching - * through a multi-million record table with 32byte keys is quite costly. - */ - result = mdb_cursor_get(c_txi, &k, &v, MDB_NEXT); + result = mdb_cursor_get(c_blocks, &k, &v, MDB_NEXT); if (result == MDB_NOTFOUND) { - txn.commit(); + MDB_val_set(pk, "txblk"); + mdb_cursor_get(c_props, &pk, &v, MDB_SET); + mdb_cursor_del(c_props, 0); + batch_stop(); break; } else if (result) - throw0(DB_ERROR(lmdb_error("Failed to get a record from tx_indices: ", result).c_str())); - txp = (txindex *)v.mv_data; - k.mv_data = (void *)&txp->key; - k.mv_size = sizeof(txp->key); - ik.mv_data = (void *)&txp->data.tx_id; - ik.mv_size = sizeof(txp->data.tx_id); + throw0(DB_ERROR(lmdb_error("Failed to get a record from blocks: ", result).c_str())); - result = mdb_cursor_get(c_txs, &k, &v, MDB_FIRST); - if (result) - throw0(DB_ERROR(lmdb_error("Failed to get a record from txs: ", result).c_str())); - result = mdb_cursor_put(c_cur, &ik, &v, 0); - if (result) - throw0(DB_ERROR(lmdb_error("Failed to put a record into txr: ", result).c_str())); - - result = mdb_cursor_get(c_tx_outputs, &k, &iv, MDB_SET); - if (!result) { - blobdata bd; - bd.assign(reinterpret_cast(v.mv_data), v.mv_size); + bd.assign(reinterpret_cast(v.mv_data), v.mv_size); + if (!parse_and_validate_block_from_blob(bd, b)) + throw0(DB_ERROR("Failed to parse block from blob retrieved from the db")); + add_transaction(null_hash, b.miner_tx); + for (unsigned int j = 0; j(v.mv_data), v.mv_size); if (!parse_and_validate_tx_from_blob(bd, tx)) throw0(DB_ERROR("Failed to parse tx from blob retrieved from the db")); - - /* turn global output indices into amount output indices */ - std::vector indices; - int j; - for (j=0;;j++) { - MDB_val_set(vk, tx.vout[j].amount); - result = mdb_cursor_get(c_oamts, &vk, &iv, MDB_GET_BOTH); - if (result) - throw0(DB_ERROR(lmdb_error("Failed to get a record from output_amounts: ", result).c_str())); - - outkey *ok = (outkey *)v.mv_data; - indices.push_back(ok->amount_index); - result = mdb_cursor_get(c_tx_outputs, &k, &iv, MDB_NEXT_DUP); - if (result == MDB_NOTFOUND) - break; - else if (result) - throw0(DB_ERROR(lmdb_error("Failed to get a record from tx_outputs: ", result).c_str())); - } - v.mv_data = (void *)indices.data(); - v.mv_size = sizeof(uint64_t) * indices.size(); - result = mdb_cursor_put(c_curtxo, &ik, &v, 0); + add_transaction(null_hash, tx, &b.tx_hashes[j]); + result = mdb_cursor_del(c_txs, 0); if (result) - throw0(DB_ERROR(lmdb_error("Failed to put a record into tx_outputr: ", result).c_str())); - result = mdb_cursor_del(c_tx_outputs, MDB_NODUPDATA); - if (result) - throw0(DB_ERROR(lmdb_error("Failed to delete a record from tx_outputs: ", result).c_str())); - } else if (result == MDB_NOTFOUND) { - /* get_tx_amount_output_indices expects a record here, even if it's empty */ - v.mv_size = 0; - v.mv_data = NULL; - result = mdb_cursor_put(c_curtxo, &ik, &v, 0); - if (result) - throw0(DB_ERROR(lmdb_error("Failed to put a record into tx_outputr: ", result).c_str())); - } else - throw0(DB_ERROR(lmdb_error("Failed to get a record from tx_outputs: ", result).c_str())); - - result = mdb_cursor_del(c_txs, 0); - if (result) - throw0(DB_ERROR(lmdb_error("Failed to delete a record from txs: ", result).c_str())); + throw0(DB_ERROR(lmdb_error("Failed to get record from txs: ", result).c_str())); + } i++; + m_height = i; } result = mdb_txn_begin(m_env, NULL, 0, txn); if (result) @@ -3683,103 +3298,12 @@ void BlockchainLMDB::migrate_0_1() result = mdb_drop(txn, o_txs, 1); if (result) throw0(DB_ERROR(lmdb_error("Failed to delete txs from the db: ", result).c_str())); - result = mdb_drop(txn, o_tx_outputs, 1); - if (result) - throw0(DB_ERROR(lmdb_error("Failed to delete tx_outputs from the db: ", result).c_str())); RENAME_DB("txr"); - RENAME_DB("tx_outputr"); mdb_dbi_close(m_env, m_txs); - mdb_dbi_close(m_env, m_tx_outputs); lmdb_db_open(txn, "txs", MDB_INTEGERKEY, m_txs, "Failed to open db handle for txs"); - lmdb_db_open(txn, "tx_outputs", MDB_INTEGERKEY, m_tx_outputs, "Failed to open db handle for tx_outputs"); - mdb_set_dupsort(txn, m_output_amounts, compare_uint64); - - txn.commit(); - } while(0); - - do { - LOG_PRINT_L1("migrating spent_keys:"); - MDB_dbi o_keys; - - unsigned int flags; - result = mdb_txn_begin(m_env, NULL, 0, txn); - if (result) - throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str())); - result = mdb_dbi_flags(txn, m_spent_keys, &flags); - if (result) - throw0(DB_ERROR(lmdb_error("Failed to retrieve spent_keys flags: ", result).c_str())); - /* if the flags are what we expect, this table has already been migrated */ - if ((flags & (MDB_INTEGERKEY|MDB_DUPSORT|MDB_DUPFIXED)) == (MDB_INTEGERKEY|MDB_DUPSORT|MDB_DUPFIXED)) { - txn.abort(); - LOG_PRINT_L1(" spent_keys already migrated"); - break; - } - - /* the spent_keys table name is the same but the old version and new version - * have incompatible DB flags. Create a new table with the right flags. - */ - o_keys = m_spent_keys; - lmdb_db_open(txn, "spent_keyr", MDB_INTEGERKEY | MDB_CREATE | MDB_DUPSORT | MDB_DUPFIXED, m_spent_keys, "Failed to open db handle for spent_keyr"); - mdb_set_dupsort(txn, m_spent_keys, compare_hash32); - - MDB_cursor *c_old, *c_cur; - i = 0; - MDB_stat ms; - mdb_stat(txn, o_keys, &ms); - z = ms.ms_entries; - mdb_stat(txn, m_spent_keys, &ms); - z += ms.ms_entries; - - while(1) { - if (!(i % 2000)) { - if (i) { - LOGIF(1) { - std::cout << i << " / " << z << " \r" << std::flush; - } - txn.commit(); - result = mdb_txn_begin(m_env, NULL, 0, txn); - if (result) - throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str())); - } - result = mdb_cursor_open(txn, m_spent_keys, &c_cur); - if (result) - throw0(DB_ERROR(lmdb_error("Failed to open a cursor for spent_keyr: ", result).c_str())); - result = mdb_cursor_open(txn, o_keys, &c_old); - if (result) - throw0(DB_ERROR(lmdb_error("Failed to open a cursor for spent_keys: ", result).c_str())); - if (!i) - i = ms.ms_entries; - } - result = mdb_cursor_get(c_old, &k, NULL, MDB_NEXT); - if (result == MDB_NOTFOUND) { - txn.commit(); - break; - } - else if (result) - throw0(DB_ERROR(lmdb_error("Failed to get a record from spent_keys: ", result).c_str())); - result = mdb_cursor_put(c_cur, (MDB_val *)&zerokval, &k, MDB_APPENDDUP); - if (result) - throw0(DB_ERROR(lmdb_error("Failed to put a record into spent_keyr: ", result).c_str())); - result = mdb_cursor_del(c_old, 0); - if (result) - throw0(DB_ERROR(lmdb_error("Failed to delete a record from spent_keys: ", result).c_str())); - i++; - } - - result = mdb_txn_begin(m_env, NULL, 0, txn); - if (result) - throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str())); - /* Delete the old table */ - result = mdb_drop(txn, o_keys, 1); - if (result) - throw0(DB_ERROR(lmdb_error("Failed to delete old spent_keys table: ", result).c_str())); - RENAME_DB("spent_keyr"); - mdb_dbi_close(m_env, m_spent_keys); - lmdb_db_open(txn, "spent_keys", MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, m_spent_keys, "Failed to open db handle for spent_keys"); - mdb_set_dupsort(txn, m_spent_keys, compare_hash32); txn.commit(); } while(0);