From e167c4d9a13849ae3e799571997bea843bda0683 Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Fri, 1 Sep 2017 11:58:51 +0100 Subject: [PATCH 1/3] blockchain_import: do not error out on truncated files This will happen often when downloading from a live file --- src/blockchain_utilities/blockchain_import.cpp | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/src/blockchain_utilities/blockchain_import.cpp b/src/blockchain_utilities/blockchain_import.cpp index ded854ca4..b5917f9c0 100644 --- a/src/blockchain_utilities/blockchain_import.cpp +++ b/src/blockchain_utilities/blockchain_import.cpp @@ -326,9 +326,19 @@ int import_from_file(cryptonote::core& core, const std::string& import_file_path } import_file.read(buffer_block, chunk_size); if (! import_file) { - MFATAL("ERROR: unexpected end of file: bytes read before error: " - << import_file.gcount() << " of chunk_size " << chunk_size); - return 2; + if (import_file.eof()) + { + std::cout << refresh_string; + MINFO("End of file reached - file was truncated"); + quit = 1; + break; + } + else + { + MFATAL("ERROR: unexpected end of file: bytes read before error: " + << import_file.gcount() << " of chunk_size " << chunk_size); + return 2; + } } bytes_read += chunk_size; MDEBUG("Total bytes read: " << bytes_read); From 5b29e87f8514b537a2ba779f4566015c42130f3a Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Fri, 1 Sep 2017 11:59:24 +0100 Subject: [PATCH 2/3] blockchain_import: properly cleanup core/db on exit --- .../blockchain_import.cpp | 34 +++++++------------ 1 file changed, 12 insertions(+), 22 deletions(-) diff --git a/src/blockchain_utilities/blockchain_import.cpp b/src/blockchain_utilities/blockchain_import.cpp index b5917f9c0..9afad95dc 100644 --- a/src/blockchain_utilities/blockchain_import.cpp +++ b/src/blockchain_utilities/blockchain_import.cpp @@ -692,18 +692,12 @@ int main(int argc, char* argv[]) MINFO("bootstrap file path: " << import_file_path); MINFO("database path: " << m_config_folder); + cryptonote::cryptonote_protocol_stub pr; //TODO: stub only for this kind of test, make real validation of relayed objects + cryptonote::core core(&pr); + try { - // fake_core needed for verification to work when enabled. - // - // NOTE: don't need fake_core method of doing things when we're going to call - // BlockchainDB add_block() directly and have available the 3 block - // properties to do so. Both ways work, but fake core isn't necessary in that - // circumstance. - - cryptonote::cryptonote_protocol_stub pr; //TODO: stub only for this kind of test, make real validation of relayed objects - cryptonote::core core(&pr); core.disable_dns_checkpoints(true); if (!core.init(vm, NULL)) { @@ -731,23 +725,19 @@ int main(int argc, char* argv[]) import_from_file(core, import_file_path, block_stop); - } - catch (const DB_ERROR& e) - { - std::cout << std::string("Error loading blockchain db: ") + e.what() + " -- shutting down now" << ENDL; - return 1; - } - - // destructors called at exit: - // // ensure db closed // - transactions properly checked and handled // - disk sync if needed // - // fake_core object's destructor is called when it goes out of scope. For an - // LMDB fake_core, it calls Blockchain::deinit() on its object, which in turn - // calls delete on its BlockchainDB derived class' object, which closes its - // files. + core.deinit(); + } + catch (const DB_ERROR& e) + { + std::cout << std::string("Error loading blockchain db: ") + e.what() + " -- shutting down now" << ENDL; + core.deinit(); + return 1; + } + return 0; CATCH_ENTRY("Import error", 1); From 4e0e4e9942074609aa5c5cb44b8509a11b24bc4c Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Fri, 1 Sep 2017 12:00:25 +0100 Subject: [PATCH 3/3] blockchain_import: warn for chunks over 500000, not 100000 We have a lot of 350000 byte blocks now. --- src/blockchain_utilities/blockchain_import.cpp | 4 ++-- src/blockchain_utilities/blockchain_utilities.h | 1 + src/blockchain_utilities/bootstrap_file.cpp | 4 ++-- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/blockchain_utilities/blockchain_import.cpp b/src/blockchain_utilities/blockchain_import.cpp index 9afad95dc..4a180430f 100644 --- a/src/blockchain_utilities/blockchain_import.cpp +++ b/src/blockchain_utilities/blockchain_import.cpp @@ -316,9 +316,9 @@ int import_from_file(cryptonote::core& core, const std::string& import_file_path MWARNING("WARNING: chunk_size " << chunk_size << " > BUFFER_SIZE " << BUFFER_SIZE); throw std::runtime_error("Aborting: chunk size exceeds buffer size"); } - if (chunk_size > 100000) + if (chunk_size > CHUNK_SIZE_WARNING_THRESHOLD) { - MINFO("NOTE: chunk_size " << chunk_size << " > 100000"); + MINFO("NOTE: chunk_size " << chunk_size << " > " << CHUNK_SIZE_WARNING_THRESHOLD); } else if (chunk_size == 0) { MFATAL("ERROR: chunk_size == 0"); diff --git a/src/blockchain_utilities/blockchain_utilities.h b/src/blockchain_utilities/blockchain_utilities.h index af934bf29..6fb5e1131 100644 --- a/src/blockchain_utilities/blockchain_utilities.h +++ b/src/blockchain_utilities/blockchain_utilities.h @@ -34,6 +34,7 @@ // bounds checking is done before writing to buffer, but buffer size // should be a sensible maximum #define BUFFER_SIZE 1000000 +#define CHUNK_SIZE_WARNING_THRESHOLD 500000 #define NUM_BLOCKS_PER_CHUNK 1 #define BLOCKCHAIN_RAW "blockchain.raw" diff --git a/src/blockchain_utilities/bootstrap_file.cpp b/src/blockchain_utilities/bootstrap_file.cpp index d5bb37d93..2b1a5d6c7 100644 --- a/src/blockchain_utilities/bootstrap_file.cpp +++ b/src/blockchain_utilities/bootstrap_file.cpp @@ -436,10 +436,10 @@ uint64_t BootstrapFile::count_blocks(const std::string& import_file_path) << " height: " << h-1); throw std::runtime_error("Aborting: chunk size exceeds buffer size"); } - if (chunk_size > 100000) + if (chunk_size > CHUNK_SIZE_WARNING_THRESHOLD) { std::cout << refresh_string; - MDEBUG("NOTE: chunk_size " << chunk_size << " > 100000" << " height: " + MDEBUG("NOTE: chunk_size " << chunk_size << " > " << CHUNK_SIZE_WARNING_THRESHOLD << " << height: " << h-1); } else if (chunk_size <= 0) {