fix serialization being different on mac

On Mac, size_t is a distinct type from uint64_t, and some
types (in wallet cache as well as cold/hot wallet transfer
data) use pairs/containers with size_t as fields. Mac would
save those as full size, while other platforms would save
them as varints. Might apply to other platforms where the
types are distinct.

There's a nasty hack for backward compatibility, which can
go after a couple forks.
This commit is contained in:
moneromooo-monero 2021-01-12 18:24:55 +00:00
parent 25670398b1
commit 4a9ae3eb8b
No known key found for this signature in database
GPG key ID: 686F07454D6CEFC3
9 changed files with 64 additions and 27 deletions

View file

@ -44,10 +44,10 @@ namespace cryptonote
typedef std::pair<uint64_t, rct::ctkey> output_entry; typedef std::pair<uint64_t, rct::ctkey> output_entry;
std::vector<output_entry> outputs; //index + key + optional ringct commitment std::vector<output_entry> outputs; //index + key + optional ringct commitment
size_t real_output; //index in outputs vector of real output_entry uint64_t real_output; //index in outputs vector of real output_entry
crypto::public_key real_out_tx_key; //incoming real tx public key crypto::public_key real_out_tx_key; //incoming real tx public key
std::vector<crypto::public_key> real_out_additional_tx_keys; //incoming real tx additional public keys std::vector<crypto::public_key> real_out_additional_tx_keys; //incoming real tx additional public keys
size_t real_output_in_tx_index; //index in transaction outputs vector uint64_t real_output_in_tx_index; //index in transaction outputs vector
uint64_t amount; //money uint64_t amount; //money
bool rct; //true if the output is rct bool rct; //true if the output is rct
rct::key mask; //ringct amount mask rct::key mask; //ringct amount mask

View file

@ -98,7 +98,7 @@ template <>
struct binary_archive<false> : public binary_archive_base<std::istream, false> struct binary_archive<false> : public binary_archive_base<std::istream, false>
{ {
explicit binary_archive(stream_type &s) : base_type(s) { explicit binary_archive(stream_type &s) : base_type(s), varint_bug_backward_compatibility_(false) {
stream_type::pos_type pos = stream_.tellg(); stream_type::pos_type pos = stream_.tellg();
stream_.seekg(0, std::ios_base::end); stream_.seekg(0, std::ios_base::end);
eof_pos_ = stream_.tellg(); eof_pos_ = stream_.tellg();
@ -173,8 +173,13 @@ struct binary_archive<false> : public binary_archive_base<std::istream, false>
assert(stream_.tellg() <= eof_pos_); assert(stream_.tellg() <= eof_pos_);
return eof_pos_ - stream_.tellg(); return eof_pos_ - stream_.tellg();
} }
void enable_varint_bug_backward_compatibility() { varint_bug_backward_compatibility_ = true; }
bool varint_bug_backward_compatibility_enabled() const { return varint_bug_backward_compatibility_; }
protected: protected:
std::streamoff eof_pos_; std::streamoff eof_pos_;
bool varint_bug_backward_compatibility_;
}; };
template <> template <>
@ -227,6 +232,8 @@ struct binary_archive<true> : public binary_archive_base<std::ostream, true>
void write_variant_tag(variant_tag_type t) { void write_variant_tag(variant_tag_type t) {
serialize_int(t); serialize_int(t);
} }
bool varint_bug_backward_compatibility_enabled() const { return false; }
}; };
POP_WARNINGS POP_WARNINGS

View file

@ -32,22 +32,27 @@ namespace serialization
{ {
namespace detail namespace detail
{ {
template<typename T>
inline constexpr bool use_container_varint() noexcept
{
return std::is_integral<T>::value && std::is_unsigned<T>::value && sizeof(T) > 1;
}
template <typename Archive, class T> template <typename Archive, class T>
bool serialize_container_element(Archive& ar, T& e) typename std::enable_if<!use_container_varint<T>(), bool>::type
serialize_container_element(Archive& ar, T& e)
{ {
return ::do_serialize(ar, e); return ::do_serialize(ar, e);
} }
template <typename Archive> template<typename Archive, typename T>
bool serialize_container_element(Archive& ar, uint32_t& e) typename std::enable_if<use_container_varint<T>(), bool>::type
serialize_container_element(Archive& ar, T& e)
{ {
ar.serialize_varint(e); static constexpr const bool previously_varint = std::is_same<uint64_t, T>() || std::is_same<uint32_t, T>();
return true;
}
template <typename Archive> if (!previously_varint && ar.varint_bug_backward_compatibility_enabled() && !typename Archive::is_saving())
bool serialize_container_element(Archive& ar, uint64_t& e) return ::do_serialize(ar, e);
{
ar.serialize_varint(e); ar.serialize_varint(e);
return true; return true;
} }

View file

@ -84,6 +84,8 @@ struct json_archive_base
void end_variant() { end_object(); } void end_variant() { end_object(); }
Stream &stream() { return stream_; } Stream &stream() { return stream_; }
bool varint_bug_backward_compatibility_enabled() const { return false; }
protected: protected:
void make_indent() void make_indent()
{ {

View file

@ -30,21 +30,34 @@
#pragma once #pragma once
#include <memory> #include <memory>
#include <boost/type_traits/make_unsigned.hpp>
#include "serialization.h" #include "serialization.h"
namespace serialization namespace serialization
{ {
namespace detail namespace detail
{ {
template<typename T>
inline constexpr bool use_pair_varint() noexcept
{
return std::is_integral<T>::value && std::is_unsigned<T>::value && sizeof(T) > 1;
}
template <typename Archive, class T> template <typename Archive, class T>
bool serialize_pair_element(Archive& ar, T& e) typename std::enable_if<!use_pair_varint<T>(), bool>::type
serialize_pair_element(Archive& ar, T& e)
{ {
return ::do_serialize(ar, e); return ::do_serialize(ar, e);
} }
template <typename Archive> template<typename Archive, typename T>
bool serialize_pair_element(Archive& ar, uint64_t& e) typename std::enable_if<use_pair_varint<T>(), bool>::type
serialize_pair_element(Archive& ar, T& e)
{ {
static constexpr const bool previously_varint = std::is_same<uint64_t, T>();
if (!previously_varint && ar.varint_bug_backward_compatibility_enabled() && !typename Archive::is_saving())
return ::do_serialize(ar, e);
ar.serialize_varint(e); ar.serialize_varint(e);
return true; return true;
} }

View file

@ -5649,6 +5649,16 @@ void wallet2::load(const std::string& wallet_, const epee::wipeable_string& pass
if (::serialization::serialize(ar, *this)) if (::serialization::serialize(ar, *this))
if (::serialization::check_stream_state(ar)) if (::serialization::check_stream_state(ar))
loaded = true; loaded = true;
if (!loaded)
{
std::stringstream iss;
iss << cache_data;
binary_archive<false> ar(iss);
ar.enable_varint_bug_backward_compatibility();
if (::serialization::serialize(ar, *this))
if (::serialization::check_stream_state(ar))
loaded = true;
}
} }
catch(...) { } catch(...) { }
@ -12439,7 +12449,7 @@ crypto::public_key wallet2::get_tx_pub_key_from_received_outs(const tools::walle
bool wallet2::export_key_images(const std::string &filename, bool all) const bool wallet2::export_key_images(const std::string &filename, bool all) const
{ {
PERF_TIMER(export_key_images); PERF_TIMER(export_key_images);
std::pair<size_t, std::vector<std::pair<crypto::key_image, crypto::signature>>> ski = export_key_images(all); std::pair<uint64_t, std::vector<std::pair<crypto::key_image, crypto::signature>>> ski = export_key_images(all);
std::string magic(KEY_IMAGE_EXPORT_FILE_MAGIC, strlen(KEY_IMAGE_EXPORT_FILE_MAGIC)); std::string magic(KEY_IMAGE_EXPORT_FILE_MAGIC, strlen(KEY_IMAGE_EXPORT_FILE_MAGIC));
const cryptonote::account_public_address &keys = get_account().get_keys().m_account_address; const cryptonote::account_public_address &keys = get_account().get_keys().m_account_address;
const uint32_t offset = ski.first; const uint32_t offset = ski.first;
@ -12466,7 +12476,7 @@ bool wallet2::export_key_images(const std::string &filename, bool all) const
} }
//---------------------------------------------------------------------------------------------------- //----------------------------------------------------------------------------------------------------
std::pair<size_t, std::vector<std::pair<crypto::key_image, crypto::signature>>> wallet2::export_key_images(bool all) const std::pair<uint64_t, std::vector<std::pair<crypto::key_image, crypto::signature>>> wallet2::export_key_images(bool all) const
{ {
PERF_TIMER(export_key_images_raw); PERF_TIMER(export_key_images_raw);
std::vector<std::pair<crypto::key_image, crypto::signature>> ski; std::vector<std::pair<crypto::key_image, crypto::signature>> ski;
@ -12963,7 +12973,7 @@ void wallet2::import_blockchain(const std::tuple<size_t, crypto::hash, std::vect
m_last_block_reward = cryptonote::get_outs_money_amount(genesis.miner_tx); m_last_block_reward = cryptonote::get_outs_money_amount(genesis.miner_tx);
} }
//---------------------------------------------------------------------------------------------------- //----------------------------------------------------------------------------------------------------
std::pair<size_t, std::vector<tools::wallet2::transfer_details>> wallet2::export_outputs(bool all) const std::pair<uint64_t, std::vector<tools::wallet2::transfer_details>> wallet2::export_outputs(bool all) const
{ {
PERF_TIMER(export_outputs); PERF_TIMER(export_outputs);
std::vector<tools::wallet2::transfer_details> outs; std::vector<tools::wallet2::transfer_details> outs;
@ -13003,7 +13013,7 @@ std::string wallet2::export_outputs_to_str(bool all) const
return magic + ciphertext; return magic + ciphertext;
} }
//---------------------------------------------------------------------------------------------------- //----------------------------------------------------------------------------------------------------
size_t wallet2::import_outputs(const std::pair<size_t, std::vector<tools::wallet2::transfer_details>> &outputs) size_t wallet2::import_outputs(const std::pair<uint64_t, std::vector<tools::wallet2::transfer_details>> &outputs)
{ {
PERF_TIMER(import_outputs); PERF_TIMER(import_outputs);
@ -13109,7 +13119,7 @@ size_t wallet2::import_outputs_from_str(const std::string &outputs_st)
try try
{ {
std::string body(data, headerlen); std::string body(data, headerlen);
std::pair<size_t, std::vector<tools::wallet2::transfer_details>> outputs; std::pair<uint64_t, std::vector<tools::wallet2::transfer_details>> outputs;
try try
{ {
std::stringstream iss; std::stringstream iss;

View file

@ -327,7 +327,7 @@ private:
uint64_t m_block_height; uint64_t m_block_height;
cryptonote::transaction_prefix m_tx; cryptonote::transaction_prefix m_tx;
crypto::hash m_txid; crypto::hash m_txid;
size_t m_internal_output_index; uint64_t m_internal_output_index;
uint64_t m_global_output_index; uint64_t m_global_output_index;
bool m_spent; bool m_spent;
bool m_frozen; bool m_frozen;
@ -338,7 +338,7 @@ private:
bool m_rct; bool m_rct;
bool m_key_image_known; bool m_key_image_known;
bool m_key_image_request; // view wallets: we want to request it; cold wallets: it was requested bool m_key_image_request; // view wallets: we want to request it; cold wallets: it was requested
size_t m_pk_index; uint64_t m_pk_index;
cryptonote::subaddress_index m_subaddr_index; cryptonote::subaddress_index m_subaddr_index;
bool m_key_image_partial; bool m_key_image_partial;
std::vector<rct::key> m_multisig_k; std::vector<rct::key> m_multisig_k;
@ -1367,9 +1367,9 @@ private:
bool verify_with_public_key(const std::string &data, const crypto::public_key &public_key, const std::string &signature) const; bool verify_with_public_key(const std::string &data, const crypto::public_key &public_key, const std::string &signature) const;
// Import/Export wallet data // Import/Export wallet data
std::pair<size_t, std::vector<tools::wallet2::transfer_details>> export_outputs(bool all = false) const; std::pair<uint64_t, std::vector<tools::wallet2::transfer_details>> export_outputs(bool all = false) const;
std::string export_outputs_to_str(bool all = false) const; std::string export_outputs_to_str(bool all = false) const;
size_t import_outputs(const std::pair<size_t, std::vector<tools::wallet2::transfer_details>> &outputs); size_t import_outputs(const std::pair<uint64_t, std::vector<tools::wallet2::transfer_details>> &outputs);
size_t import_outputs_from_str(const std::string &outputs_st); size_t import_outputs_from_str(const std::string &outputs_st);
payment_container export_payments() const; payment_container export_payments() const;
void import_payments(const payment_container &payments); void import_payments(const payment_container &payments);
@ -1377,7 +1377,7 @@ private:
std::tuple<size_t, crypto::hash, std::vector<crypto::hash>> export_blockchain() const; std::tuple<size_t, crypto::hash, std::vector<crypto::hash>> export_blockchain() const;
void import_blockchain(const std::tuple<size_t, crypto::hash, std::vector<crypto::hash>> &bc); void import_blockchain(const std::tuple<size_t, crypto::hash, std::vector<crypto::hash>> &bc);
bool export_key_images(const std::string &filename, bool all = false) const; bool export_key_images(const std::string &filename, bool all = false) const;
std::pair<size_t, std::vector<std::pair<crypto::key_image, crypto::signature>>> export_key_images(bool all = false) const; std::pair<uint64_t, std::vector<std::pair<crypto::key_image, crypto::signature>>> export_key_images(bool all = false) const;
uint64_t import_key_images(const std::vector<std::pair<crypto::key_image, crypto::signature>> &signed_key_images, size_t offset, uint64_t &spent, uint64_t &unspent, bool check_spent = true); uint64_t import_key_images(const std::vector<std::pair<crypto::key_image, crypto::signature>> &signed_key_images, size_t offset, uint64_t &spent, uint64_t &unspent, bool check_spent = true);
uint64_t import_key_images(const std::string &filename, uint64_t &spent, uint64_t &unspent); uint64_t import_key_images(const std::string &filename, uint64_t &spent, uint64_t &unspent);
bool import_key_images(std::vector<crypto::key_image> key_images, size_t offset=0, boost::optional<std::unordered_set<size_t>> selected_transfers=boost::none); bool import_key_images(std::vector<crypto::key_image> key_images, size_t offset=0, boost::optional<std::unordered_set<size_t>> selected_transfers=boost::none);

View file

@ -2702,7 +2702,7 @@ namespace tools
if (!m_wallet) return not_open(er); if (!m_wallet) return not_open(er);
try try
{ {
std::pair<size_t, std::vector<std::pair<crypto::key_image, crypto::signature>>> ski = m_wallet->export_key_images(req.all); std::pair<uint64_t, std::vector<std::pair<crypto::key_image, crypto::signature>>> ski = m_wallet->export_key_images(req.all);
res.offset = ski.first; res.offset = ski.first;
res.signed_key_images.resize(ski.second.size()); res.signed_key_images.resize(ski.second.size());
for (size_t n = 0; n < ski.second.size(); ++n) for (size_t n = 0; n < ski.second.size(); ++n)

View file

@ -51,7 +51,7 @@ END_INIT_SIMPLE_FUZZER()
BEGIN_SIMPLE_FUZZER() BEGIN_SIMPLE_FUZZER()
std::string s((const char*)buf, len); std::string s((const char*)buf, len);
std::pair<size_t, std::vector<tools::wallet2::transfer_details>> outputs; std::pair<uint64_t, std::vector<tools::wallet2::transfer_details>> outputs;
std::stringstream iss; std::stringstream iss;
iss << s; iss << s;
binary_archive<false> ar(iss); binary_archive<false> ar(iss);