Merges #36

src/blockchain_converter/README.md

@@ -0,0 +1,54 @@
+
+For importing into the LMDB database, compile with `DATABASE=lmdb`
+
+e.g.
+
+`DATABASE=lmdb make release`
+
+This is also the default compile setting on the blockchain branch.
+
+By default, the exporter will use the original in-memory database (blockchain.bin) as its source.
+This default is to make migrating to an LMDB database easy, without having to recompile anything.
+To change the source, adjust `SOURCE_DB` in `src/blockchain_converter/blockchain_export.h` according to the comments.
+
+# Usage:
+
+See also each utility's "--help" option.
+
+## Export an existing in-memory database
+
+`$ blockchain_export`
+
+This loads the existing blockchain, for whichever database type it was compiled for, and exports it to `$MONERO_DATA_DIR/export/blockchain.raw`
+
+## Import the exported file
+
+`$ blockchain_import`
+
+This imports blocks from `$MONERO_DATA_DIR/export/blockchain.raw` into the current database.
+
+Defaults: `--batch on`, `--batch size 20000`, `--verify on`
+
+Batch size refers to number of blocks and can be adjusted for performance based on available RAM.
+
+Verification should only be turned off if importing from a trusted blockchain.
+
+```bash
+# use default settings to import blockchain.raw into database
+$ blockchain_import
+
+# fast import with large batch size, verification off
+$ blockchain_import --batch-size 100000 --verify off
+
+# LMDB flags can be set by appending them to the database type:
+# flags: nosync, nometasync, writemap, mapasync
+$ blockchain_import --database lmdb#nosync
+$ blockchain_import --database lmdb#nosync,nometasync
+```
+
+## Blockchain converter with batching
+`blockchain_converter` has also been updated and includes batching for faster writes. However, on lower RAM systems, this will be slower than using the exporter and importer utilities. The converter needs to keep the blockchain in memory for the duration of the conversion, like the original bitmonerod, thus leaving less memory available to the destination database to operate.
+
+```bash
+$ blockchain_converter --batch on --batch-size 20000
+```

src/blockchain_converter/blockchain_converter.cpp

@@ -44,14 +44,19 @@
 #include "version.h"
 #include <iostream>
 
+namespace
+{
+
 // CONFIG
-static bool opt_batch   = true;
-static bool opt_testnet = false;
+bool opt_batch   = true;
+bool opt_resume  = true;
+bool opt_testnet = false;
 
 // number of blocks per batch transaction
 // adjustable through command-line argument according to available RAM
-static uint64_t db_batch_size = 20000;
+uint64_t db_batch_size = 20000;
 
+}
 
 namespace po = boost::program_options;
 
@@ -113,11 +118,14 @@ int main(int argc, char* argv[])
 
   const command_line::arg_descriptor<bool> arg_batch  =  {"batch",
     "Batch transactions for faster import", true};
+  const command_line::arg_descriptor<bool> arg_resume =  {"resume",
+        "Resume from current height if output database already exists", true};
 
   // call add_options() directly for these arguments since command_line helpers
   // support only boolean switch, not boolean argument
   desc_cmd_sett.add_options()
-    (arg_batch.name, make_semantic(arg_batch), arg_batch.description)
+    (arg_batch.name,  make_semantic(arg_batch),  arg_batch.description)
+    (arg_resume.name, make_semantic(arg_resume), arg_resume.description)
     ;
 
   po::options_description desc_options("Allowed options");
@@ -137,6 +145,7 @@ int main(int argc, char* argv[])
 
   int log_level = command_line::get_arg(vm, arg_log_level);
   opt_batch     = command_line::get_arg(vm, arg_batch);
+  opt_resume    = command_line::get_arg(vm, arg_resume);
   db_batch_size = command_line::get_arg(vm, arg_batch_size);
 
   if (command_line::get_arg(vm, command_line::arg_help))
@@ -178,23 +187,41 @@ int main(int argc, char* argv[])
   {
     LOG_PRINT_L0("batch:   " << std::boolalpha << opt_batch << std::noboolalpha);
   }
+  LOG_PRINT_L0("resume:  " << std::boolalpha << opt_resume  << std::noboolalpha);
   LOG_PRINT_L0("testnet: " << std::boolalpha << opt_testnet << std::noboolalpha);
 
   fake_core c(src_folder, opt_testnet);
 
   height = c.m_storage.get_current_blockchain_height();
-  if (! num_blocks || num_blocks > height)
-    end_block = height - 1;
-  else
-    end_block = start_block + num_blocks - 1;
 
   BlockchainDB *blockchain;
   blockchain = new BlockchainLMDB(opt_batch);
   dest_folder /= blockchain->get_db_name();
   LOG_PRINT_L0("Source blockchain: " << src_folder);
   LOG_PRINT_L0("Dest blockchain:   " << dest_folder.string());
-  LOG_PRINT_L0("Opening LMDB: " << dest_folder.string());
+  LOG_PRINT_L0("Opening dest blockchain (BlockchainDB " << blockchain->get_db_name() << ")");
   blockchain->open(dest_folder.string());
+  LOG_PRINT_L0("Source blockchain height: " << height);
+  LOG_PRINT_L0("Dest blockchain height:   " << blockchain->height());
+
+  if (opt_resume)
+    // next block number to add is same as current height
+    start_block = blockchain->height();
+
+  if (! num_blocks || (start_block + num_blocks > height))
+    end_block = height - 1;
+  else
+    end_block = start_block + num_blocks - 1;
+
+  LOG_PRINT_L0("start height: " << start_block+1 << "  stop height: " <<
+            end_block+1);
+
+  if (start_block > end_block)
+  {
+    LOG_PRINT_L0("Finished: no blocks to add");
+    delete blockchain;
+    return 0;
+  }
 
   if (opt_batch)
     blockchain->batch_start();
@@ -247,7 +274,7 @@ int main(int argc, char* argv[])
     catch (const std::exception& e)
     {
       std::cout << ENDL;
-      std::cerr << "Error adding block to new blockchain: " << e.what() << ENDL;
+      std::cerr << "Error adding block " << i << " to new blockchain: " << e.what() << ENDL;
       delete blockchain;
       return 2;
     }

src/blockchain_converter/blockchain_export.cpp

@@ -148,7 +148,7 @@ void BlockchainExport::write_block(block& block)
 #if SOURCE_DB == DB_MEMORY
   const transaction* cb_tx_full = m_blockchain_storage->get_tx(coinbase_tx_hash);
 #else
-  transaction cb_tx_full = m_blockchain_storage->get_db()->get_tx(coinbase_tx_hash);
+  transaction cb_tx_full = m_blockchain_storage->get_db().get_tx(coinbase_tx_hash);
 #endif
 
 #if SOURCE_DB == DB_MEMORY
@@ -167,7 +167,7 @@ void BlockchainExport::write_block(block& block)
 #if SOURCE_DB == DB_MEMORY
     const transaction* tx = m_blockchain_storage->get_tx(tx_id);
 #else
-    transaction tx = m_blockchain_storage->get_db()->get_tx(tx_id);
+    transaction tx = m_blockchain_storage->get_db().get_tx(tx_id);
 #endif
 
 #if SOURCE_DB == DB_MEMORY
@@ -205,18 +205,18 @@ void BlockchainExport::write_block(block& block)
 #if SOURCE_DB == DB_MEMORY
     size_t block_size = m_blockchain_storage->get_block_size(block_height);
 #else
-    size_t block_size = m_blockchain_storage->get_db()->get_block_size(block_height);
+    size_t block_size = m_blockchain_storage->get_db().get_block_size(block_height);
 #endif
 #if SOURCE_DB == DB_MEMORY
     difficulty_type cumulative_difficulty = m_blockchain_storage->get_block_cumulative_difficulty(block_height);
 #else
-    difficulty_type cumulative_difficulty = m_blockchain_storage->get_db()->get_block_cumulative_difficulty(block_height);
+    difficulty_type cumulative_difficulty = m_blockchain_storage->get_db().get_block_cumulative_difficulty(block_height);
 #endif
 #if SOURCE_DB == DB_MEMORY
     uint64_t coins_generated = m_blockchain_storage->get_block_coins_generated(block_height);
 #else
     // TODO TEST to verify that this is the equivalent. make sure no off-by-one error with block height vs block number
-    uint64_t coins_generated = m_blockchain_storage->get_db()->get_block_already_generated_coins(block_height);
+    uint64_t coins_generated = m_blockchain_storage->get_db().get_block_already_generated_coins(block_height);
 #endif
 
     *m_raw_archive << block_size;
@@ -239,14 +239,16 @@ bool BlockchainExport::BlockchainExport::close()
 
 
 #if SOURCE_DB == DB_MEMORY
-bool BlockchainExport::store_blockchain_raw(blockchain_storage* _blockchain_storage, tx_memory_pool* _tx_pool, boost::filesystem::path& output_dir, uint64_t use_block_height)
+bool BlockchainExport::store_blockchain_raw(blockchain_storage* _blockchain_storage, tx_memory_pool* _tx_pool, boost::filesystem::path& output_dir, uint64_t requested_block_height)
 #else
-bool BlockchainExport::store_blockchain_raw(Blockchain* _blockchain_storage, tx_memory_pool* _tx_pool, boost::filesystem::path& output_dir, uint64_t use_block_height)
+bool BlockchainExport::store_blockchain_raw(Blockchain* _blockchain_storage, tx_memory_pool* _tx_pool, boost::filesystem::path& output_dir, uint64_t requested_block_height)
 #endif
 {
-  uint64_t use_block_height2 = 0;
+  uint64_t block_height = 0;
   m_blockchain_storage = _blockchain_storage;
   m_tx_pool = _tx_pool;
+  uint64_t progress_interval = 100;
+  std::string refresh_string = "\r                                    \r";
   LOG_PRINT_L0("Storing blocks raw data...");
   if (!BlockchainExport::open(output_dir))
   {
@@ -255,15 +257,15 @@ bool BlockchainExport::store_blockchain_raw(Blockchain* _blockchain_storage, tx_
   }
   block b;
   LOG_PRINT_L0("source blockchain height: " <<  m_blockchain_storage->get_current_blockchain_height());
-  LOG_PRINT_L0("requested block height: " << use_block_height);
-  if ((use_block_height > 0) && (use_block_height < m_blockchain_storage->get_current_blockchain_height()))
-    use_block_height2 = use_block_height;
+  LOG_PRINT_L0("requested block height: " << requested_block_height);
+  if ((requested_block_height > 0) && (requested_block_height < m_blockchain_storage->get_current_blockchain_height()))
+    block_height = requested_block_height;
   else
   {
-    use_block_height2 = m_blockchain_storage->get_current_blockchain_height();
-    LOG_PRINT_L0("using block height: " << use_block_height2);
+    block_height = m_blockchain_storage->get_current_blockchain_height();
+    LOG_PRINT_L0("Using block height of source blockchain: " << block_height);
   }
-  for (height=0; height < use_block_height2; ++height)
+  for (height=0; height < block_height; ++height)
   {
     crypto::hash hash = m_blockchain_storage->get_block_id_by_height(height);
     m_blockchain_storage->get_block_by_hash(hash, b);
@@ -271,11 +273,17 @@ bool BlockchainExport::store_blockchain_raw(Blockchain* _blockchain_storage, tx_
     if (height % NUM_BLOCKS_PER_CHUNK == 0) {
       flush_chunk();
     }
+    if (height % progress_interval == 0) {
+      std::cout << refresh_string;
+      std::cout << "height " << height << "/" << block_height << std::flush;
+    }
   }
   if (height % NUM_BLOCKS_PER_CHUNK != 0)
   {
     flush_chunk();
   }
+  std::cout << refresh_string;
+  std::cout << "height " << height << "/" << block_height << ENDL;
 
   LOG_PRINT_L0("longest chunk was " << max_chunk << " bytes");
   return BlockchainExport::close();

src/blockchain_converter/blockchain_import.cpp

@@ -57,6 +57,8 @@ static bool opt_testnet = true;
 // adjustable through command-line argument according to available RAM
 static uint64_t db_batch_size = 20000;
 
+static std::string refresh_string = "\r                                    \r";
+
 
 namespace po = boost::program_options;
 
@@ -146,7 +148,7 @@ int count_blocks(std::string& import_file_path)
     int chunk_size;
     import_file.read(buffer1, STR_LENGTH_OF_INT);
     if (!import_file) {
-      std::cout << "\r                   \r";
+      std::cout << refresh_string;
       LOG_PRINT_L1("End of import file reached");
       quit = true;
       break;
@@ -154,7 +156,7 @@ int count_blocks(std::string& import_file_path)
     h += NUM_BLOCKS_PER_CHUNK;
     if (h % progress_interval == 0)
     {
-      std::cout << "\r                   \r" << "block height: " << h <<
+      std::cout << refresh_string << "block height: " << h <<
         std::flush;
     }
     bytes_read += STR_LENGTH_OF_INT;
@@ -162,32 +164,33 @@ int count_blocks(std::string& import_file_path)
     chunk_size = atoi(buffer1);
     if (chunk_size > BUFFER_SIZE)
     {
-      std::cout << "\r                   \r";
+      std::cout << refresh_string;
       LOG_PRINT_L0("WARNING: chunk_size " << chunk_size << " > BUFFER_SIZE " << BUFFER_SIZE
           << "  height: " << h);
       throw std::runtime_error("Aborting: chunk size exceeds buffer size");
     }
     if (chunk_size > 100000)
     {
-      std::cout << "\r                   \r";
+      std::cout << refresh_string;
       LOG_PRINT_L0("WARNING: chunk_size " << chunk_size << " > 100000" << "  height: "
           << h);
     }
     else if (chunk_size <= 0) {
-      std::cout << "\r                   \r";
+      std::cout << refresh_string;
       LOG_PRINT_L0("ERROR: chunk_size " << chunk_size << " <= 0" << "  height: " << h);
       throw std::runtime_error("Aborting");
     }
     // skip to next expected block size value
     import_file.seekg(chunk_size, std::ios_base::cur);
     if (! import_file) {
-      std::cout << "\r                   \r";
+      std::cout << refresh_string;
       LOG_PRINT_L0("ERROR: unexpected end of import file: bytes read before error: "
           << import_file.gcount() << " of chunk_size " << chunk_size);
       throw std::runtime_error("Aborting");
     }
     bytes_read += chunk_size;
-    std::cout << "\r                   \r";
+    std::cout << refresh_string;
+
     LOG_PRINT_L3("Total bytes scanned: " << bytes_read);
   }
 
@@ -214,7 +217,7 @@ int import_from_file(FakeCore& simple_core, std::string& import_file_path)
     // Reset stats, in case we're using newly created db, accumulating stats
     // from addition of genesis block.
     // This aligns internal db counts with importer counts.
-    simple_core.m_storage.get_db()->reset_stats();
+    simple_core.m_storage.get_db().reset_stats();
   }
 #endif
   boost::filesystem::path raw_file_path(import_file_path);
@@ -285,7 +288,7 @@ int import_from_file(FakeCore& simple_core, std::string& import_file_path)
     int chunk_size;
     import_file.read(buffer1, STR_LENGTH_OF_INT);
     if (! import_file) {
-      std::cout << "\r                   \r";
+      std::cout << refresh_string;
       LOG_PRINT_L0("End of import file reached");
       quit = 1;
       break;
@@ -340,7 +343,7 @@ int import_from_file(FakeCore& simple_core, std::string& import_file_path)
         h++;
         if (h % display_interval == 0)
         {
-          std::cout << "\r                   \r";
+          std::cout << refresh_string;
           LOG_PRINT_L0("loading block height " << h);
         }
         else
@@ -352,7 +355,7 @@ int import_from_file(FakeCore& simple_core, std::string& import_file_path)
         }
         catch (const std::exception& e)
         {
-          std::cout << "\r                   \r";
+          std::cout << refresh_string;
           LOG_PRINT_RED_L0("exception while de-archiving block, height=" << h);
           quit = 1;
           break;
@@ -361,7 +364,7 @@ int import_from_file(FakeCore& simple_core, std::string& import_file_path)
 
         if (h % progress_interval == 0)
         {
-          std::cout << "\r                   \r" << "block " << h-1
+          std::cout << refresh_string << "block " << h-1
             << std::flush;
         }
 
@@ -374,7 +377,7 @@ int import_from_file(FakeCore& simple_core, std::string& import_file_path)
         }
         catch (const std::exception& e)
         {
-          std::cout << "\r                   \r";
+          std::cout << refresh_string;
           LOG_PRINT_RED_L0("exception while de-archiving tx-num, height=" << h);
           quit = 1;
           break;
@@ -469,7 +472,7 @@ int import_from_file(FakeCore& simple_core, std::string& import_file_path)
           a >> cumulative_difficulty;
           a >> coins_generated;
 
-          std::cout << "\r                   \r";
+          std::cout << refresh_string;
           LOG_PRINT_L2("block_size: " << block_size);
           LOG_PRINT_L2("cumulative_difficulty: " << cumulative_difficulty);
           LOG_PRINT_L2("coins_generated: " << coins_generated);
@@ -480,7 +483,7 @@ int import_from_file(FakeCore& simple_core, std::string& import_file_path)
           }
           catch (const std::exception& e)
           {
-            std::cout << "\r                   \r";
+            std::cout << refresh_string;
             LOG_PRINT_RED_L0("Error adding block to blockchain: " << e.what());
             quit = 2; // make sure we don't commit partial block data
             break;
@@ -491,13 +494,13 @@ int import_from_file(FakeCore& simple_core, std::string& import_file_path)
         {
           if (h % db_batch_size == 0)
           {
-            std::cout << "\r                   \r";
+            std::cout << refresh_string;
             std::cout << ENDL << "[- batch commit at height " << h << " -]" << ENDL;
             simple_core.batch_stop();
             simple_core.batch_start();
             std::cout << ENDL;
 #if !defined(BLOCKCHAIN_DB) || (BLOCKCHAIN_DB == DB_LMDB)
-            simple_core.m_storage.get_db()->show_stats();
+            simple_core.m_storage.get_db().show_stats();
 #endif
           }
         }
@@ -505,7 +508,7 @@ int import_from_file(FakeCore& simple_core, std::string& import_file_path)
     }
     catch (const std::exception& e)
     {
-      std::cout << "\r                   \r";
+      std::cout << refresh_string;
       LOG_PRINT_RED_L0("exception while reading from import file, height=" << h);
       return 2;
     }
@@ -525,7 +528,7 @@ int import_from_file(FakeCore& simple_core, std::string& import_file_path)
       simple_core.batch_stop();
     }
 #if !defined(BLOCKCHAIN_DB) || (BLOCKCHAIN_DB == DB_LMDB)
-    simple_core.m_storage.get_db()->show_stats();
+    simple_core.m_storage.get_db().show_stats();
 #endif
     if (h > 0)
       LOG_PRINT_L0("Finished at height: " << h << "  block: " << h-1);

src/blockchain_converter/fake_core.h

@@ -55,7 +55,7 @@ struct fake_core_lmdb
     m_pool.init(path.string());
     m_storage.init(path.string(), use_testnet, mdb_flags);
     if (do_batch)
-      m_storage.get_db()->set_batch_transactions(do_batch);
+      m_storage.get_db().set_batch_transactions(do_batch);
     support_batch = true;
     support_add_block = true;
   }
@@ -71,17 +71,17 @@ struct fake_core_lmdb
                             , const std::vector<transaction>& txs
                             )
   {
-    return m_storage.get_db()->add_block(blk, block_size, cumulative_difficulty, coins_generated, txs);
+    return m_storage.get_db().add_block(blk, block_size, cumulative_difficulty, coins_generated, txs);
   }
 
   void batch_start()
   {
-    m_storage.get_db()->batch_start();
+    m_storage.get_db().batch_start();
   }
 
   void batch_stop()
   {
-    m_storage.get_db()->batch_stop();
+    m_storage.get_db().batch_stop();
   }
 
 };

src/cryptonote_core/blockchain.h

@@ -145,9 +145,9 @@ namespace cryptonote
     void set_enforce_dns_checkpoints(bool enforce);
     bool update_checkpoints(const std::string& file_path, bool check_dns);
 
-    BlockchainDB* get_db()
+    BlockchainDB& get_db()
     {
-      return m_db;
+      return *m_db;
     }
 
   private: