diff --git a/src/cryptonote_basic/connection_context.h b/src/cryptonote_basic/connection_context.h index a7d688300..ee26a0c07 100644 --- a/src/cryptonote_basic/connection_context.h +++ b/src/cryptonote_basic/connection_context.h @@ -77,6 +77,8 @@ namespace cryptonote int m_expect_response; uint64_t m_expect_height; size_t m_num_requested; + epee::copyable_atomic m_new_stripe_notification{0}; + epee::copyable_atomic m_idle_peer_notification{0}; }; inline std::string get_protocol_state_string(cryptonote_connection_context::state s) diff --git a/src/cryptonote_protocol/cryptonote_protocol_handler.inl b/src/cryptonote_protocol/cryptonote_protocol_handler.inl index 5580998fb..69ef0970b 100644 --- a/src/cryptonote_protocol/cryptonote_protocol_handler.inl +++ b/src/cryptonote_protocol/cryptonote_protocol_handler.inl @@ -137,6 +137,41 @@ namespace cryptonote CHECK_AND_ASSERT_MES_CC( context.m_callback_request_count > 0, false, "false callback fired, but context.m_callback_request_count=" << context.m_callback_request_count); --context.m_callback_request_count; + uint32_t notified = true; + if (context.m_idle_peer_notification.compare_exchange_strong(notified, not notified)) + { + if (context.m_state == cryptonote_connection_context::state_synchronizing && context.m_last_request_time != boost::date_time::not_a_date_time) + { + const boost::posix_time::ptime now = boost::posix_time::microsec_clock::universal_time(); + const boost::posix_time::time_duration dt = now - context.m_last_request_time; + const auto ms = dt.total_microseconds(); + if (ms > IDLE_PEER_KICK_TIME || (context.m_expect_response && ms > NON_RESPONSIVE_PEER_KICK_TIME)) + { + if (context.m_score-- >= 0) + { + MINFO(context << " kicking idle peer, last update " << (dt.total_microseconds() / 1.e6) << " seconds ago, expecting " << (int)context.m_expect_response); + context.m_last_request_time = boost::date_time::not_a_date_time; + context.m_expect_response = 0; + context.m_expect_height = 0; + context.m_state = cryptonote_connection_context::state_standby; // we'll go back to adding, then (if we can't), download + } + else + { + MINFO(context << "dropping idle peer with negative score"); + drop_connection_with_score(context, context.m_expect_response == 0 ? 1 : 5, false); + return false; + } + } + } + } + + notified = true; + if (context.m_new_stripe_notification.compare_exchange_strong(notified, not notified)) + { + if (context.m_state == cryptonote_connection_context::state_normal) + context.m_state = cryptonote_connection_context::state_synchronizing; + } + if(context.m_state == cryptonote_connection_context::state_synchronizing && context.m_last_request_time == boost::posix_time::not_a_date_time) { NOTIFY_REQUEST_CHAIN::request r = {}; @@ -1687,7 +1722,7 @@ skip: const uint32_t peer_stripe = tools::get_pruning_stripe(context.m_pruning_seed); if (stripe && peer_stripe && peer_stripe != stripe) return true; - context.m_state = cryptonote_connection_context::state_synchronizing; + context.m_new_stripe_notification = true; LOG_PRINT_CCONTEXT_L2("requesting callback"); ++context.m_callback_request_count; m_p2p->request_callback(context); @@ -1710,7 +1745,6 @@ skip: bool t_cryptonote_protocol_handler::kick_idle_peers() { MTRACE("Checking for idle peers..."); - std::vector> idle_peers; m_p2p->for_each_connection([&](cryptonote_connection_context& context, nodetool::peerid_type peer_id, uint32_t support_flags)->bool { if (context.m_state == cryptonote_connection_context::state_synchronizing && context.m_last_request_time != boost::date_time::not_a_date_time) @@ -1720,36 +1754,16 @@ skip: const auto ms = dt.total_microseconds(); if (ms > IDLE_PEER_KICK_TIME || (context.m_expect_response && ms > NON_RESPONSIVE_PEER_KICK_TIME)) { - if (context.m_score-- >= 0) - { - MINFO(context << " kicking idle peer, last update " << (dt.total_microseconds() / 1.e6) << " seconds ago, expecting " << (int)context.m_expect_response); - LOG_PRINT_CCONTEXT_L2("requesting callback"); - context.m_last_request_time = boost::date_time::not_a_date_time; - context.m_expect_response = 0; - context.m_expect_height = 0; - context.m_state = cryptonote_connection_context::state_standby; // we'll go back to adding, then (if we can't), download - ++context.m_callback_request_count; - m_p2p->request_callback(context); - } - else - { - idle_peers.push_back(std::make_pair(context.m_connection_id, context.m_expect_response == 0 ? 1 : 5)); - } + context.m_idle_peer_notification = true; + LOG_PRINT_CCONTEXT_L2("requesting callback"); + ++context.m_callback_request_count; + m_p2p->request_callback(context); + MLOG_PEER_STATE("requesting callback"); } } return true; }); - for (const auto &e: idle_peers) - { - const auto &uuid = e.first; - m_p2p->for_connection(uuid, [&](cryptonote_connection_context& ctx, nodetool::peerid_type peer_id, uint32_t f)->bool{ - MINFO(ctx << "dropping idle peer with negative score"); - drop_connection_with_score(ctx, e.second, false); - return true; - }); - } - return true; } //------------------------------------------------------------------------------------------------------------------------ diff --git a/tests/unit_tests/node_server.cpp b/tests/unit_tests/node_server.cpp index af297d01a..fc7886485 100644 --- a/tests/unit_tests/node_server.cpp +++ b/tests/unit_tests/node_server.cpp @@ -310,5 +310,597 @@ TEST(node_server, bind_same_p2p_port) EXPECT_TRUE(init(new_node(), port_another)); } +TEST(cryptonote_protocol_handler, race_condition) +{ + struct contexts { + using basic = epee::net_utils::connection_context_base; + using cryptonote = cryptonote::cryptonote_connection_context; + using p2p = nodetool::p2p_connection_context_t; + }; + using context_t = contexts::p2p; + using handler_t = epee::levin::async_protocol_handler; + using connection_t = epee::net_utils::connection; + using connection_ptr = boost::shared_ptr; + using connections_t = std::vector; + using shared_state_t = typename connection_t::shared_state; + using shared_state_ptr = std::shared_ptr; + using io_context_t = boost::asio::io_service; + using event_t = epee::simple_event; + using ec_t = boost::system::error_code; + auto create_conn_pair = [](connection_ptr in, connection_ptr out) { + using endpoint_t = boost::asio::ip::tcp::endpoint; + using acceptor_t = boost::asio::ip::tcp::acceptor; + io_context_t io_context; + endpoint_t endpoint(boost::asio::ip::address::from_string("127.0.0.1"), 5262); + acceptor_t acceptor(io_context); + ec_t ec; + acceptor.open(endpoint.protocol(), ec); + EXPECT_EQ(ec.value(), 0); + acceptor.set_option(boost::asio::ip::tcp::acceptor::reuse_address(true)); + acceptor.bind(endpoint, ec); + EXPECT_EQ(ec.value(), 0); + acceptor.listen(boost::asio::socket_base::max_listen_connections, ec); + EXPECT_EQ(ec.value(), 0); + out->socket().open(endpoint.protocol(), ec); + EXPECT_EQ(ec.value(), 0); + acceptor.async_accept(in->socket(), [](const ec_t &ec){}); + out->socket().async_connect(endpoint, [](const ec_t &ec){}); + io_context.run(); + acceptor.close(ec); + EXPECT_EQ(ec.value(), 0); + EXPECT_TRUE(in->start(true, true)); + EXPECT_TRUE(out->start(false, true)); + return std::make_pair<>(std::move(in), std::move(out)); + }; + auto get_conn_tag = [](connection_t &conn){ + context_t context; + conn.get_context(context); + return context.m_connection_id; + }; + using work_t = boost::asio::io_service::work; + using work_ptr = std::shared_ptr; + using workers_t = std::vector; + using commands_handler_t = epee::levin::levin_commands_handler; + using p2p_endpoint_t = nodetool::i_p2p_endpoint; + using core_t = cryptonote::core; + using core_ptr = std::unique_ptr; + using core_protocol_t = cryptonote::t_cryptonote_protocol_handler; + using core_protocol_ptr = std::shared_ptr; + using block_t = cryptonote::block; + using diff_t = cryptonote::difficulty_type; + using reward_t = uint64_t; + using height_t = uint64_t; + struct span { + using blocks = epee::span; + }; + auto get_block_template = []( + core_t &core, + block_t &block, + diff_t &diff, + reward_t &reward + ){ + auto &storage = core.get_blockchain_storage(); + const auto height = storage.get_current_blockchain_height(); + const auto hardfork = storage.get_current_hard_fork_version(); + block.major_version = hardfork; + block.minor_version = storage.get_ideal_hard_fork_version(); + block.prev_id = storage.get_tail_id(); + auto &db = storage.get_db(); + block.timestamp = db.get_top_block_timestamp(); + block.nonce = 0xACAB; + block.miner_tx.vin.clear(); + block.miner_tx.vout.clear(); + block.miner_tx.extra.clear(); + block.miner_tx.version = hardfork >= 4 ? 2 : 1; + block.miner_tx.unlock_time = height + CRYPTONOTE_MINED_MONEY_UNLOCK_WINDOW; + block.miner_tx.vin.push_back(cryptonote::txin_gen{height}); + cryptonote::add_tx_pub_key_to_extra(block.miner_tx, {}); + cryptonote::get_block_reward( + db.get_block_weight(height - 1), + {}, + db.get_block_already_generated_coins(height - 1), + reward, + hardfork + ); + block.miner_tx.vout.push_back(cryptonote::tx_out{reward, cryptonote::txout_to_key{}}); + diff = storage.get_difficulty_for_next_block(); + }; + struct stat { + struct chain { + diff_t diff; + reward_t reward; + }; + }; + auto add_block = []( + core_t &core, + const block_t &block, + const stat::chain &stat + ){ + core.get_blockchain_storage().get_db().batch_start({}, {}); + core.get_blockchain_storage().get_db().add_block( + {block, cryptonote::block_to_blob(block)}, + cryptonote::get_transaction_weight(block.miner_tx), + core.get_blockchain_storage().get_next_long_term_block_weight( + cryptonote::get_transaction_weight(block.miner_tx) + ), + stat.diff, + stat.reward, + {} + ); + core.get_blockchain_storage().get_db().batch_stop(); + }; + struct messages { + struct core { + using sync = cryptonote::CORE_SYNC_DATA; + }; + using handshake = nodetool::COMMAND_HANDSHAKE_T; + }; + struct net_node_t: commands_handler_t, p2p_endpoint_t { + using span_t = epee::span; + using string_t = std::string; + using zone_t = epee::net_utils::zone; + using uuid_t = boost::uuids::uuid; + using relay_t = cryptonote::relay_method; + using blobs_t = std::vector; + using id_t = nodetool::peerid_type; + using callback_t = std::function; + using address_t = epee::net_utils::network_address; + using connections_t = std::vector>; + struct bans { + using subnets = std::map; + using hosts = std::map; + }; + struct slice { + using bytes = epee::byte_slice; + }; + shared_state_ptr shared_state; + core_protocol_ptr core_protocol; + virtual int invoke(int command, const span_t in, slice::bytes &out, context_t &context) override { + if (core_protocol) { + if (command == messages::handshake::ID) { + return epee::net_utils::buff_to_t_adapter( + command, + in, + out, + [this](int command, typename messages::handshake::request &in, typename messages::handshake::response &out, context_t &context){ + core_protocol->process_payload_sync_data(in.payload_data, context, true); + core_protocol->get_payload_sync_data(out.payload_data); + return 1; + }, + context + ); + } + bool handled; + return core_protocol->handle_invoke_map(false, command, in, out, context, handled); + } + else + return {}; + } + virtual int notify(int command, const span_t in, context_t &context) override { + if (core_protocol) { + bool handled; + slice::bytes out; + return core_protocol->handle_invoke_map(true, command, in, out, context, handled); + } + else + return {}; + } + virtual void callback(context_t &context) override { + if (core_protocol) + core_protocol->on_callback(context); + } + virtual void on_connection_new(context_t&) override {} + virtual void on_connection_close(context_t &context) override { + if (core_protocol) + core_protocol->on_connection_close(context); + } + virtual ~net_node_t() override {} + virtual bool add_host_fail(const address_t&, unsigned int = {}) override { + return {}; + } + virtual bool block_host(address_t address, time_t = {}, bool = {}) override { + return {}; + } + virtual bool drop_connection(const contexts::basic& context) override { + if (shared_state) + return shared_state->close(context.m_connection_id); + else + return {}; + } + virtual bool for_connection(const uuid_t& uuid, callback_t f) override { + if (shared_state) + return shared_state->for_connection(uuid,[&f](context_t &context){ + return f(context, context.peer_id, context.support_flags); + }); + else + return {}; + } + virtual bool invoke_command_to_peer(int command, const span_t in, string_t& out, const contexts::basic& context) override { + if (shared_state) + return shared_state->invoke(command, in, out, context.m_connection_id); + else + return {}; + } + virtual bool invoke_notify_to_peer(int command, const span_t in, const contexts::basic& context) override { + if (shared_state) + return shared_state->notify(command, in, context.m_connection_id); + else + return {}; + } + virtual bool relay_notify_to_list(int command, const span_t in, connections_t connections) override { + if (shared_state) { + for (auto &e: connections) + shared_state->notify(command, in, e.second); + } + return {}; + } + virtual bool unblock_host(const address_t&) override { + return {}; + } + virtual zone_t send_txs(blobs_t, const zone_t, const uuid_t&, relay_t) override { + return {}; + } + virtual bans::subnets get_blocked_subnets() override { + return {}; + } + virtual bans::hosts get_blocked_hosts() override { + return {}; + } + virtual uint64_t get_public_connections_count() override { + if (shared_state) + return shared_state->get_connections_count(); + else + return {}; + } + virtual void add_used_stripe_peer(const contexts::cryptonote&) override {} + virtual void clear_used_stripe_peers() override {} + virtual void remove_used_stripe_peer(const contexts::cryptonote&) override {} + virtual void for_each_connection(callback_t f) override { + if (shared_state) + shared_state->foreach_connection([&f](context_t &context){ + return f(context, context.peer_id, context.support_flags); + }); + } + virtual void request_callback(const contexts::basic &context) override { + if (shared_state) + shared_state->request_callback(context.m_connection_id); + } + }; + auto conduct_handshake = [get_conn_tag](net_node_t &net_node, connection_ptr conn){ + event_t handshaked; + net_node.shared_state->for_connection( + get_conn_tag(*conn), + [&handshaked, &net_node](context_t &context){ + typename messages::handshake::request msg; + net_node.core_protocol->get_payload_sync_data(msg.payload_data); + epee::net_utils::async_invoke_remote_command2( + context, + messages::handshake::ID, + msg, + *net_node.shared_state, + [&handshaked, &net_node](int code, const typename messages::handshake::response &msg, context_t &context){ + EXPECT_TRUE(code >= 0); + net_node.core_protocol->process_payload_sync_data(msg.payload_data, context, true); + handshaked.raise(); + }, + P2P_DEFAULT_HANDSHAKE_INVOKE_TIMEOUT + ); + return true; + } + ); + handshaked.wait(); + }; + using path_t = boost::filesystem::path; + auto create_dir = []{ + ec_t ec; + path_t path = boost::filesystem::temp_directory_path() / boost::filesystem::unique_path("daemon-%%%%%%%%%%%%%%%%", ec); + if (ec) + return path_t{}; + auto success = boost::filesystem::create_directory(path, ec); + if (not ec && success) + return path; + return path_t{}; + }; + auto remove_tree = [](const path_t &path){ + ec_t ec; + boost::filesystem::remove_all(path, ec); + }; + using options_t = boost::program_options::variables_map; + struct daemon_t { + options_t options; + core_ptr core; + core_protocol_ptr core_protocol; + net_node_t net_node; + shared_state_ptr shared_state; + connections_t conn; + }; + struct daemons_t { + daemon_t main; + daemon_t alt; + }; + using options_description_t = boost::program_options::options_description; + + const auto dir = create_dir(); + ASSERT_TRUE(not dir.empty()); + + daemons_t daemon{ + { + [&dir]{ + options_t options; + boost::program_options::store( + boost::program_options::command_line_parser({ + "--data-dir", + (dir / "main").string(), + "--disable-dns-checkpoints", + "--check-updates=disabled", + "--fixed-difficulty=1", + "--block-sync-size=1", + "--db-sync-mode=fastest:async:50000", + }).options([]{ + options_description_t options_description{}; + cryptonote::core::init_options(options_description); + return options_description; + }()).run(), + options + ); + return options; + }(), + {}, + {}, + {}, + {}, + {}, + }, + { + [&dir]{ + options_t options; + boost::program_options::store( + boost::program_options::command_line_parser({ + "--data-dir", + (dir / "alt").string(), + "--disable-dns-checkpoints", + "--check-updates=disabled", + "--fixed-difficulty=1", + "--block-sync-size=1", + "--db-sync-mode=fastest:async:50000", + }).options([]{ + options_description_t options_description{}; + cryptonote::core::init_options(options_description); + return options_description; + }()).run(), + options + ); + return options; + }(), + {}, + {}, + {}, + {}, + {}, + }, + }; + + io_context_t io_context; + work_ptr work = std::make_shared(io_context); + workers_t workers; + while (workers.size() < 4) { + workers.emplace_back([&io_context]{ + io_context.run(); + }); + } + + connection_t::set_rate_up_limit(std::numeric_limits::max()); + connection_t::set_rate_down_limit(std::numeric_limits::max()); + + { + daemon.main.core = core_ptr(new core_t(nullptr)); + daemon.main.core->init(daemon.main.options, nullptr, nullptr); + daemon.main.net_node.core_protocol = daemon.main.core_protocol = core_protocol_ptr(new core_protocol_t( + *daemon.main.core, &daemon.main.net_node, {} + )); + daemon.main.core->set_cryptonote_protocol(daemon.main.core_protocol.get()); + daemon.main.core_protocol->init(daemon.main.options); + daemon.main.net_node.shared_state = daemon.main.shared_state = std::make_shared(); + daemon.main.shared_state->set_handler(&daemon.main.net_node); + daemon.alt.shared_state = std::make_shared(); + daemon.alt.shared_state->set_handler(&daemon.alt.net_node); + + struct { + event_t prepare; + event_t check; + event_t finish; + } events; + auto connections = create_conn_pair( + connection_ptr(new connection_t(io_context, daemon.main.shared_state, {}, {})), + connection_ptr(new connection_t(io_context, daemon.alt.shared_state, {}, {})) + ); + { + auto conn = connections.first; + auto shared_state = daemon.main.shared_state; + const auto tag = get_conn_tag(*conn); + conn->strand_.post([tag, conn, shared_state, &events]{ + shared_state->for_connection(tag, [](context_t &context){ + context.m_expect_height = -1; + context.m_expect_response = -1; + context.m_last_request_time = boost::date_time::min_date_time; + context.m_score = 0; + context.m_state = contexts::cryptonote::state_synchronizing; + return true; + }); + events.prepare.raise(); + events.check.wait(); + shared_state->for_connection(tag, [](context_t &context){ + EXPECT_TRUE(context.m_expect_height == -1); + EXPECT_TRUE(context.m_expect_response == -1); + EXPECT_TRUE(context.m_last_request_time == boost::date_time::min_date_time); + EXPECT_TRUE(context.m_score == 0); + EXPECT_TRUE(context.m_state == contexts::cryptonote::state_synchronizing); + return true; + }); + events.finish.raise(); + }); + } + events.prepare.wait(); + daemon.main.core_protocol->on_idle(); + events.check.raise(); + events.finish.wait(); + + connections.first->strand_.post([connections]{ + connections.first->cancel(); + }); + connections.second->strand_.post([connections]{ + connections.second->cancel(); + }); + connections.first.reset(); + connections.second.reset(); + while (daemon.main.shared_state->sock_count); + while (daemon.alt.shared_state->sock_count); + daemon.main.core_protocol->deinit(); + daemon.main.core->stop(); + daemon.main.core->deinit(); + daemon.main.net_node.shared_state.reset(); + daemon.main.shared_state.reset(); + daemon.main.core_protocol.reset(); + daemon.main.core.reset(); + daemon.alt.shared_state.reset(); + } + + { + daemon.main.core = core_ptr(new core_t(nullptr)); + daemon.main.core->init(daemon.main.options, nullptr, nullptr); + daemon.main.net_node.core_protocol = daemon.main.core_protocol = core_protocol_ptr(new core_protocol_t( + *daemon.main.core, &daemon.main.net_node, {} + )); + daemon.main.core->set_cryptonote_protocol(daemon.main.core_protocol.get()); + daemon.main.core->set_checkpoints({}); + daemon.main.core_protocol->init(daemon.main.options); + daemon.main.net_node.shared_state = daemon.main.shared_state = std::make_shared(); + daemon.main.shared_state->set_handler(&daemon.main.net_node); + daemon.alt.core = core_ptr(new core_t(nullptr)); + daemon.alt.core->init(daemon.alt.options, nullptr, nullptr); + daemon.alt.net_node.core_protocol = daemon.alt.core_protocol = core_protocol_ptr(new core_protocol_t( + *daemon.alt.core, &daemon.alt.net_node, {} + )); + daemon.alt.core->set_cryptonote_protocol(daemon.alt.core_protocol.get()); + daemon.alt.core->set_checkpoints({}); + daemon.alt.core_protocol->init(daemon.alt.options); + daemon.alt.net_node.shared_state = daemon.alt.shared_state = std::make_shared(); + daemon.alt.shared_state->set_handler(&daemon.alt.net_node); + + struct { + io_context_t io_context; + work_ptr work; + workers_t workers; + } check; + check.work = std::make_shared(check.io_context); + check.workers.emplace_back([&check]{ + check.io_context.run(); + }); + while (daemon.main.conn.size() < 1) { + daemon.main.conn.emplace_back(new connection_t(check.io_context, daemon.main.shared_state, {}, {})); + daemon.alt.conn.emplace_back(new connection_t(io_context, daemon.alt.shared_state, {}, {})); + create_conn_pair(daemon.main.conn.back(), daemon.alt.conn.back()); + conduct_handshake(daemon.alt.net_node, daemon.alt.conn.back()); + } + struct { + event_t prepare; + event_t sync; + event_t finish; + } events; + { + auto conn = daemon.main.conn.back(); + auto shared_state = daemon.main.shared_state; + const auto tag = get_conn_tag(*conn); + conn->strand_.post([tag, conn, shared_state, &events]{ + shared_state->for_connection(tag, [](context_t &context){ + EXPECT_TRUE(context.m_state == contexts::cryptonote::state_normal); + return true; + }); + events.prepare.raise(); + events.sync.wait(); + shared_state->for_connection(tag, [](context_t &context){ + EXPECT_TRUE(context.m_state == contexts::cryptonote::state_normal); + return true; + }); + events.finish.raise(); + }); + } + events.prepare.wait(); + daemon.main.core->get_blockchain_storage().add_block_notify( + [&events](height_t height, span::blocks blocks){ + if (height >= CRYPTONOTE_PRUNING_STRIPE_SIZE) + events.sync.raise(); + } + ); + { + stat::chain stat{ + daemon.alt.core->get_blockchain_storage().get_db().get_block_cumulative_difficulty( + daemon.alt.core->get_current_blockchain_height() - 1 + ), + daemon.alt.core->get_blockchain_storage().get_db().get_block_already_generated_coins( + daemon.alt.core->get_current_blockchain_height() - 1 + ), + }; + while (daemon.alt.core->get_current_blockchain_height() < CRYPTONOTE_PRUNING_STRIPE_SIZE + CRYPTONOTE_PRUNING_TIP_BLOCKS) { + block_t block; + diff_t diff; + reward_t reward; + get_block_template(*daemon.alt.core, block, diff, reward); + stat.diff += diff; + stat.reward = stat.reward < (MONEY_SUPPLY - stat.reward) ? stat.reward + reward : MONEY_SUPPLY; + add_block(*daemon.alt.core, block, stat); + if (daemon.main.core->get_current_blockchain_height() + 1 < CRYPTONOTE_PRUNING_STRIPE_SIZE) + add_block(*daemon.main.core, block, stat); + } + } + while (daemon.main.conn.size() < 2) { + daemon.main.conn.emplace_back(new connection_t(io_context, daemon.main.shared_state, {}, {})); + daemon.alt.conn.emplace_back(new connection_t(io_context, daemon.alt.shared_state, {}, {})); + create_conn_pair(daemon.main.conn.back(), daemon.alt.conn.back()); + conduct_handshake(daemon.alt.net_node, daemon.alt.conn.back()); + } + events.finish.wait(); + + for (;daemon.main.conn.size(); daemon.main.conn.pop_back()) { + auto conn = daemon.main.conn.back(); + conn->strand_.post([conn]{ + conn->cancel(); + }); + } + for (;daemon.alt.conn.size(); daemon.alt.conn.pop_back()) { + auto conn = daemon.alt.conn.back(); + conn->strand_.post([conn]{ + conn->cancel(); + }); + } + while (daemon.main.shared_state->sock_count); + while (daemon.alt.shared_state->sock_count); + daemon.main.core_protocol->deinit(); + daemon.main.core->stop(); + daemon.main.core->deinit(); + daemon.main.net_node.shared_state.reset(); + daemon.main.shared_state.reset(); + daemon.main.core_protocol.reset(); + daemon.main.core.reset(); + daemon.alt.core_protocol->deinit(); + daemon.alt.core->stop(); + daemon.alt.core->deinit(); + daemon.alt.net_node.shared_state.reset(); + daemon.alt.shared_state.reset(); + daemon.alt.core_protocol.reset(); + daemon.alt.core.reset(); + check.work.reset(); + for (auto& w: check.workers) { + w.join(); + } + } + + work.reset(); + for (auto& w: workers) { + w.join(); + } + remove_tree(dir); +} + namespace nodetool { template class node_server>; } namespace cryptonote { template class t_cryptonote_protocol_handler; }