Allow setting start block on export

And make import honor the starting block# recorded in a bootstrap file
This commit is contained in:
Howard Chu 2020-10-18 12:59:23 +01:00
parent bf65b07718
commit b7dd8349f4
No known key found for this signature in database
GPG key ID: FD2A70B44AB11BA7
4 changed files with 69 additions and 34 deletions

View file

@ -47,6 +47,7 @@ int main(int argc, char* argv[])
epee::string_tools::set_module_name_and_folder(argv[0]); epee::string_tools::set_module_name_and_folder(argv[0]);
uint32_t log_level = 0; uint32_t log_level = 0;
uint64_t block_start = 0;
uint64_t block_stop = 0; uint64_t block_stop = 0;
bool blocks_dat = false; bool blocks_dat = false;
@ -58,6 +59,7 @@ int main(int argc, char* argv[])
po::options_description desc_cmd_sett("Command line options and settings options"); po::options_description desc_cmd_sett("Command line options and settings options");
const command_line::arg_descriptor<std::string> arg_output_file = {"output-file", "Specify output file", "", true}; const command_line::arg_descriptor<std::string> arg_output_file = {"output-file", "Specify output file", "", true};
const command_line::arg_descriptor<std::string> arg_log_level = {"log-level", "0-4 or categories", ""}; const command_line::arg_descriptor<std::string> arg_log_level = {"log-level", "0-4 or categories", ""};
const command_line::arg_descriptor<uint64_t> arg_block_start = {"block-start", "Start at block number", block_start};
const command_line::arg_descriptor<uint64_t> arg_block_stop = {"block-stop", "Stop at block number", block_stop}; const command_line::arg_descriptor<uint64_t> arg_block_stop = {"block-stop", "Stop at block number", block_stop};
const command_line::arg_descriptor<bool> arg_blocks_dat = {"blocksdat", "Output in blocks.dat format", blocks_dat}; const command_line::arg_descriptor<bool> arg_blocks_dat = {"blocksdat", "Output in blocks.dat format", blocks_dat};
@ -67,6 +69,7 @@ int main(int argc, char* argv[])
command_line::add_arg(desc_cmd_sett, cryptonote::arg_testnet_on); command_line::add_arg(desc_cmd_sett, cryptonote::arg_testnet_on);
command_line::add_arg(desc_cmd_sett, cryptonote::arg_stagenet_on); command_line::add_arg(desc_cmd_sett, cryptonote::arg_stagenet_on);
command_line::add_arg(desc_cmd_sett, arg_log_level); command_line::add_arg(desc_cmd_sett, arg_log_level);
command_line::add_arg(desc_cmd_sett, arg_block_start);
command_line::add_arg(desc_cmd_sett, arg_block_stop); command_line::add_arg(desc_cmd_sett, arg_block_stop);
command_line::add_arg(desc_cmd_sett, arg_blocks_dat); command_line::add_arg(desc_cmd_sett, arg_blocks_dat);
@ -97,6 +100,7 @@ int main(int argc, char* argv[])
mlog_set_log(command_line::get_arg(vm, arg_log_level).c_str()); mlog_set_log(command_line::get_arg(vm, arg_log_level).c_str());
else else
mlog_set_log(std::string(std::to_string(log_level) + ",bcutil:INFO").c_str()); mlog_set_log(std::string(std::to_string(log_level) + ",bcutil:INFO").c_str());
block_start = command_line::get_arg(vm, arg_block_start);
block_stop = command_line::get_arg(vm, arg_block_stop); block_stop = command_line::get_arg(vm, arg_block_stop);
LOG_PRINT_L0("Starting..."); LOG_PRINT_L0("Starting...");
@ -178,7 +182,7 @@ int main(int argc, char* argv[])
else else
{ {
BootstrapFile bootstrap; BootstrapFile bootstrap;
r = bootstrap.store_blockchain_raw(core_storage, NULL, output_file_path, block_stop); r = bootstrap.store_blockchain_raw(core_storage, NULL, output_file_path, block_start, block_stop);
} }
CHECK_AND_ASSERT_MES(r, 1, "Failed to export blockchain raw data"); CHECK_AND_ASSERT_MES(r, 1, "Failed to export blockchain raw data");
LOG_PRINT_L0("Blockchain raw data exported OK"); LOG_PRINT_L0("Blockchain raw data exported OK");

View file

@ -227,6 +227,7 @@ int import_from_file(cryptonote::core& core, const std::string& import_file_path
return false; return false;
} }
uint64_t block_first, block_last;
uint64_t start_height = 1, seek_height; uint64_t start_height = 1, seek_height;
if (opt_resume) if (opt_resume)
start_height = core.get_blockchain_storage().get_current_blockchain_height(); start_height = core.get_blockchain_storage().get_current_blockchain_height();
@ -235,10 +236,10 @@ int import_from_file(cryptonote::core& core, const std::string& import_file_path
BootstrapFile bootstrap; BootstrapFile bootstrap;
std::streampos pos; std::streampos pos;
// BootstrapFile bootstrap(import_file_path); // BootstrapFile bootstrap(import_file_path);
uint64_t total_source_blocks = bootstrap.count_blocks(import_file_path, pos, seek_height); uint64_t total_source_blocks = bootstrap.count_blocks(import_file_path, pos, seek_height, block_first);
MINFO("bootstrap file last block number: " << total_source_blocks-1 << " (zero-based height) total blocks: " << total_source_blocks); MINFO("bootstrap file last block number: " << total_source_blocks+block_first-1 << " (zero-based height) total blocks: " << total_source_blocks);
if (total_source_blocks-1 <= start_height) if (total_source_blocks+block_first-1 <= start_height)
{ {
return false; return false;
} }
@ -260,7 +261,8 @@ int import_from_file(cryptonote::core& core, const std::string& import_file_path
// 4 byte magic + (currently) 1024 byte header structures // 4 byte magic + (currently) 1024 byte header structures
uint8_t major_version, minor_version; uint8_t major_version, minor_version;
bootstrap.seek_to_first_chunk(import_file, major_version, minor_version); uint64_t dummy;
bootstrap.seek_to_first_chunk(import_file, major_version, minor_version, dummy, dummy);
std::string str1; std::string str1;
char buffer1[1024]; char buffer1[1024];
@ -275,7 +277,7 @@ int import_from_file(cryptonote::core& core, const std::string& import_file_path
if (! block_stop) if (! block_stop)
{ {
block_stop = total_source_blocks - 1; block_stop = total_source_blocks+block_first - 1;
} }
// These are what we'll try to use, and they don't have to be a determination // These are what we'll try to use, and they don't have to be a determination

View file

@ -52,7 +52,7 @@ namespace
bool BootstrapFile::open_writer(const boost::filesystem::path& file_path) bool BootstrapFile::open_writer(const boost::filesystem::path& file_path, uint64_t start_block, uint64_t stop_block)
{ {
const boost::filesystem::path dir_path = file_path.parent_path(); const boost::filesystem::path dir_path = file_path.parent_path();
if (!dir_path.empty()) if (!dir_path.empty())
@ -78,7 +78,7 @@ bool BootstrapFile::open_writer(const boost::filesystem::path& file_path)
m_raw_data_file = new std::ofstream(); m_raw_data_file = new std::ofstream();
bool do_initialize_file = false; bool do_initialize_file = false;
uint64_t num_blocks = 0; uint64_t num_blocks = 0, block_first = 0;
if (! boost::filesystem::exists(file_path)) if (! boost::filesystem::exists(file_path))
{ {
@ -88,10 +88,12 @@ bool BootstrapFile::open_writer(const boost::filesystem::path& file_path)
} }
else else
{ {
num_blocks = count_blocks(file_path.string()); std::streampos dummy_pos;
MDEBUG("appending to existing file with height: " << num_blocks-1 << " total blocks: " << num_blocks); uint64_t dummy_height = 0;
num_blocks = count_blocks(file_path.string(), dummy_pos, dummy_height, block_first);
MDEBUG("appending to existing file with height: " << num_blocks+block_first-1 << " total blocks: " << num_blocks);
} }
m_height = num_blocks; m_height = num_blocks+block_first;
if (do_initialize_file) if (do_initialize_file)
m_raw_data_file->open(file_path.string(), std::ios_base::binary | std::ios_base::out | std::ios::trunc); m_raw_data_file->open(file_path.string(), std::ios_base::binary | std::ios_base::out | std::ios::trunc);
@ -106,13 +108,12 @@ bool BootstrapFile::open_writer(const boost::filesystem::path& file_path)
return false; return false;
if (do_initialize_file) if (do_initialize_file)
initialize_file(); initialize_file(start_block, stop_block);
return true; return true;
} }
bool BootstrapFile::initialize_file(uint64_t first_block, uint64_t last_block)
bool BootstrapFile::initialize_file()
{ {
const uint32_t file_magic = blockchain_raw_magic; const uint32_t file_magic = blockchain_raw_magic;
@ -129,8 +130,8 @@ bool BootstrapFile::initialize_file()
bfi.header_size = header_size; bfi.header_size = header_size;
bootstrap::blocks_info bbi; bootstrap::blocks_info bbi;
bbi.block_first = 0; bbi.block_first = first_block;
bbi.block_last = 0; bbi.block_last = last_block;
bbi.block_last_pos = 0; bbi.block_last_pos = 0;
buffer_type buffer2; buffer_type buffer2;
@ -261,7 +262,7 @@ bool BootstrapFile::close()
} }
bool BootstrapFile::store_blockchain_raw(Blockchain* _blockchain_storage, tx_memory_pool* _tx_pool, boost::filesystem::path& output_file, uint64_t requested_block_stop) bool BootstrapFile::store_blockchain_raw(Blockchain* _blockchain_storage, tx_memory_pool* _tx_pool, boost::filesystem::path& output_file, uint64_t start_block, uint64_t requested_block_stop)
{ {
uint64_t num_blocks_written = 0; uint64_t num_blocks_written = 0;
m_max_chunk = 0; m_max_chunk = 0;
@ -269,17 +270,11 @@ bool BootstrapFile::store_blockchain_raw(Blockchain* _blockchain_storage, tx_mem
m_tx_pool = _tx_pool; m_tx_pool = _tx_pool;
uint64_t progress_interval = 100; uint64_t progress_interval = 100;
MINFO("Storing blocks raw data..."); MINFO("Storing blocks raw data...");
if (!BootstrapFile::open_writer(output_file))
{
MFATAL("failed to open raw file for write");
return false;
}
block b; block b;
// block_start, block_stop use 0-based height. m_height uses 1-based height. So to resume export // block_start, block_stop use 0-based height. m_height uses 1-based height. So to resume export
// from last exported block, block_start doesn't need to add 1 here, as it's already at the next // from last exported block, block_start doesn't need to add 1 here, as it's already at the next
// height. // height.
uint64_t block_start = m_height;
uint64_t block_stop = 0; uint64_t block_stop = 0;
MINFO("source blockchain height: " << m_blockchain_storage->get_current_blockchain_height()-1); MINFO("source blockchain height: " << m_blockchain_storage->get_current_blockchain_height()-1);
if ((requested_block_stop > 0) && (requested_block_stop < m_blockchain_storage->get_current_blockchain_height())) if ((requested_block_stop > 0) && (requested_block_stop < m_blockchain_storage->get_current_blockchain_height()))
@ -292,6 +287,13 @@ bool BootstrapFile::store_blockchain_raw(Blockchain* _blockchain_storage, tx_mem
block_stop = m_blockchain_storage->get_current_blockchain_height() - 1; block_stop = m_blockchain_storage->get_current_blockchain_height() - 1;
MINFO("Using block height of source blockchain: " << block_stop); MINFO("Using block height of source blockchain: " << block_stop);
} }
if (!BootstrapFile::open_writer(output_file, start_block, block_stop))
{
MFATAL("failed to open raw file for write");
return false;
}
uint64_t block_start = m_height ? m_height : start_block;
MINFO("Starting block height: " << block_start);
for (m_cur_height = block_start; m_cur_height <= block_stop; ++m_cur_height) for (m_cur_height = block_start; m_cur_height <= block_stop; ++m_cur_height)
{ {
// this method's height refers to 0-based height (genesis block = height 0) // this method's height refers to 0-based height (genesis block = height 0)
@ -323,7 +325,8 @@ bool BootstrapFile::store_blockchain_raw(Blockchain* _blockchain_storage, tx_mem
return BootstrapFile::close(); return BootstrapFile::close();
} }
uint64_t BootstrapFile::seek_to_first_chunk(std::ifstream& import_file, uint8_t &major_version, uint8_t &minor_version) uint64_t BootstrapFile::seek_to_first_chunk(std::ifstream& import_file, uint8_t &major_version, uint8_t &minor_version,
uint64_t &block_first, uint64_t &block_last)
{ {
uint32_t file_magic; uint32_t file_magic;
@ -368,11 +371,35 @@ uint64_t BootstrapFile::seek_to_first_chunk(std::ifstream& import_file, uint8_t
MINFO("bootstrap magic size: " << sizeof(file_magic)); MINFO("bootstrap magic size: " << sizeof(file_magic));
MINFO("bootstrap header size: " << bfi.header_size); MINFO("bootstrap header size: " << bfi.header_size);
uint32_t buflen_blocks_info;
import_file.read(buf1, sizeof(buflen_blocks_info));
str1.assign(buf1, sizeof(buflen_blocks_info));
if (! import_file)
throw std::runtime_error("Error reading expected number of bytes");
if (! ::serialization::parse_binary(str1, buflen_blocks_info))
throw std::runtime_error("Error in deserialization of buflen_blocks_info");
MINFO("bootstrap::blocks_info size: " << buflen_blocks_info);
if (buflen_blocks_info > sizeof(buf1))
throw std::runtime_error("Error: bootstrap::blocks_info size exceeds buffer size");
import_file.read(buf1, buflen_blocks_info);
if (! import_file)
throw std::runtime_error("Error reading expected number of bytes");
str1.assign(buf1, buflen_blocks_info);
bootstrap::blocks_info bbi;
if (! ::serialization::parse_binary(str1, bbi))
throw std::runtime_error("Error in deserialization of bootstrap::blocks_info");
MINFO("bootstrap first block:" << bbi.block_first);
MINFO("bootstrap last block:" << bbi.block_last);
uint64_t full_header_size = sizeof(file_magic) + bfi.header_size; uint64_t full_header_size = sizeof(file_magic) + bfi.header_size;
import_file.seekg(full_header_size); import_file.seekg(full_header_size);
major_version = bfi.major_version; major_version = bfi.major_version;
minor_version = bfi.minor_version; minor_version = bfi.minor_version;
block_first = bbi.block_first;
block_last = bbi.block_last;
return full_header_size; return full_header_size;
} }
@ -436,13 +463,14 @@ uint64_t BootstrapFile::count_blocks(const std::string& import_file_path)
{ {
std::streampos dummy_pos; std::streampos dummy_pos;
uint64_t dummy_height = 0; uint64_t dummy_height = 0;
return count_blocks(import_file_path, dummy_pos, dummy_height); return count_blocks(import_file_path, dummy_pos, dummy_height, dummy_height);
} }
// If seek_height is non-zero on entry, return a stream position <= this height when finished. // If seek_height is non-zero on entry, return a stream position <= this height when finished.
// And return the actual height corresponding to this position. Allows the caller to locate its // And return the actual height corresponding to this position. Allows the caller to locate its
// starting position without having to reread the entire file again. // starting position without having to reread the entire file again.
uint64_t BootstrapFile::count_blocks(const std::string& import_file_path, std::streampos &start_pos, uint64_t& seek_height) uint64_t BootstrapFile::count_blocks(const std::string& import_file_path, std::streampos &start_pos,
uint64_t& seek_height, uint64_t &block_first)
{ {
boost::filesystem::path raw_file_path(import_file_path); boost::filesystem::path raw_file_path(import_file_path);
boost::system::error_code ec; boost::system::error_code ec;
@ -464,7 +492,8 @@ uint64_t BootstrapFile::count_blocks(const std::string& import_file_path, std::s
uint64_t full_header_size; // 4 byte magic + length of header structures uint64_t full_header_size; // 4 byte magic + length of header structures
uint8_t major_version, minor_version; uint8_t major_version, minor_version;
full_header_size = seek_to_first_chunk(import_file, major_version, minor_version); uint64_t block_last;
full_header_size = seek_to_first_chunk(import_file, major_version, minor_version, block_first, block_last);
MINFO("Scanning blockchain from bootstrap file..."); MINFO("Scanning blockchain from bootstrap file...");
bool quit = false; bool quit = false;
@ -473,11 +502,11 @@ uint64_t BootstrapFile::count_blocks(const std::string& import_file_path, std::s
while (! quit) while (! quit)
{ {
if (start_height && h + progress_interval >= start_height - 1) if (start_height && h + block_first + progress_interval >= start_height - 1)
{ {
start_height = 0; start_height = 0;
start_pos = import_file.tellg(); start_pos = import_file.tellg();
seek_height = h; seek_height = h + block_first;
} }
bytes_read += count_bytes(import_file, progress_interval, blocks, quit); bytes_read += count_bytes(import_file, progress_interval, blocks, quit);
h += blocks; h += blocks;

View file

@ -57,12 +57,12 @@ class BootstrapFile
public: public:
uint64_t count_bytes(std::ifstream& import_file, uint64_t blocks, uint64_t& h, bool& quit); uint64_t count_bytes(std::ifstream& import_file, uint64_t blocks, uint64_t& h, bool& quit);
uint64_t count_blocks(const std::string& dir_path, std::streampos& start_pos, uint64_t& seek_height); uint64_t count_blocks(const std::string& dir_path, std::streampos& start_pos, uint64_t& seek_height, uint64_t& block_first);
uint64_t count_blocks(const std::string& dir_path); uint64_t count_blocks(const std::string& dir_path);
uint64_t seek_to_first_chunk(std::ifstream& import_file, uint8_t &major_version, uint8_t &minor_version); uint64_t seek_to_first_chunk(std::ifstream& import_file, uint8_t &major_version, uint8_t &minor_version, uint64_t &block_first, uint64_t &block_last);
bool store_blockchain_raw(cryptonote::Blockchain* cs, cryptonote::tx_memory_pool* txp, bool store_blockchain_raw(cryptonote::Blockchain* cs, cryptonote::tx_memory_pool* txp,
boost::filesystem::path& output_file, uint64_t use_block_height=0); boost::filesystem::path& output_file, uint64_t start_block=0, uint64_t stop_block=0);
protected: protected:
@ -75,8 +75,8 @@ protected:
boost::iostreams::stream<boost::iostreams::back_insert_device<buffer_type>>* m_output_stream; boost::iostreams::stream<boost::iostreams::back_insert_device<buffer_type>>* m_output_stream;
// open export file for write // open export file for write
bool open_writer(const boost::filesystem::path& file_path); bool open_writer(const boost::filesystem::path& file_path, uint64_t start_block, uint64_t stop_block);
bool initialize_file(); bool initialize_file(uint64_t start_block, uint64_t stop_block);
bool close(); bool close();
void write_block(block& block); void write_block(block& block);
void flush_chunk(); void flush_chunk();