d0s3t3ch/wownero
1
0
Fork 0
mirror of https://git.wownero.com/wownero/wownero.git synced 2024-08-15 01:03:23 +00:00
Code Issues Projects Releases Wiki Activity

Adding classes, functions, and utilities for common LMDB operations.

Browse source
This commit is contained in:
Lee Clagett 2018-09-26 23:54:27 -04:00
parent f2f725d8db
commit 0c7e7bce18
15 changed files with 1959 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
src/CMakeLists.txt
View file

@ -109,6 +109,7 @@ add_subdirectory(ringct)
add_subdirectory(checkpoints)
add_subdirectory(cryptonote_basic)
add_subdirectory(cryptonote_core)
add_subdirectory(lmdb)
add_subdirectory(multisig)
add_subdirectory(net)
if(NOT IOS)

33
src/lmdb/CMakeLists.txt Normal file
View file

@ -0,0 +1,33 @@
# Copyright (c) 2014-2018, The Monero Project
#
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without modification, are
# permitted provided that the following conditions are met:
#
# 1. Redistributions of source code must retain the above copyright notice, this list of
# conditions and the following disclaimer.
#
# 2. Redistributions in binary form must reproduce the above copyright notice, this list
# of conditions and the following disclaimer in the documentation and/or other
# materials provided with the distribution.
#
# 3. Neither the name of the copyright holder nor the names of its contributors may be
# used to endorse or promote products derived from this software without specific
# prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
# THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
# THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
set(lmdb_sources database.cpp error.cpp table.cpp value_stream.cpp)
set(lmdb_headers database.h error.h key_stream.h table.h transaction.h util.h value_stream.h)
monero_add_library(lmdb_lib ${lmdb_sources} ${lmdb_headers})
target_link_libraries(lmdb_lib common ${LMDB_LIBRARY})

187
src/lmdb/database.cpp Normal file
View file

@ -0,0 +1,187 @@
// Copyright (c) 2014-2018, The Monero Project
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without modification, are
// permitted provided that the following conditions are met:
//
// 1. Redistributions of source code must retain the above copyright notice, this list of
// conditions and the following disclaimer.
//
// 2. Redistributions in binary form must reproduce the above copyright notice, this list
// of conditions and the following disclaimer in the documentation and/or other
// materials provided with the distribution.
//
// 3. Neither the name of the copyright holder nor the names of its contributors may be
// used to endorse or promote products derived from this software without specific
// prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include "database.h"
#include "lmdb/error.h"
#include "lmdb/util.h"
#ifdef _WIN32
namespace
{
constexpr const mdb_mode_t open_flags = 0;
}
#else
#include <sys/stat.h>
namespace
{
constexpr const mdb_mode_t open_flags = (S_IRUSR | S_IWUSR);
}
#endif
namespace lmdb
{
namespace
{
constexpr const std::size_t max_resize = 1 * 1024 * 1024 * 1024; // 1 GB
void acquire_context(context& ctx) noexcept
{
while (ctx.lock.test_and_set());
++(ctx.active);
ctx.lock.clear();
}
void release_context(context& ctx) noexcept
{
--(ctx.active);
}
}
void release_read_txn::operator()(MDB_txn* ptr) const noexcept
{
if (ptr)
{
MDB_env* const env = mdb_txn_env(ptr);
abort_txn{}(ptr);
if (env)
{
context* ctx = reinterpret_cast<context*>(mdb_env_get_userctx(env));
if (ctx)
release_context(*ctx);
}
}
}
expect<environment> open_environment(const char* path, MDB_dbi max_dbs) noexcept
{
MONERO_PRECOND(path != nullptr);
MDB_env* obj = nullptr;
MONERO_LMDB_CHECK(mdb_env_create(std::addressof(obj)));
environment out{obj};
MONERO_LMDB_CHECK(mdb_env_set_maxdbs(out.get(), max_dbs));
MONERO_LMDB_CHECK(mdb_env_open(out.get(), path, 0, open_flags));
return {std::move(out)};
}
expect<write_txn> database::do_create_txn(unsigned int flags) noexcept
{
MONERO_PRECOND(handle() != nullptr);
for (unsigned attempts = 0; attempts < 3; ++attempts)
{
acquire_context(ctx);
MDB_txn* txn = nullptr;
const int err =
mdb_txn_begin(handle(), nullptr, flags, &txn);
if (!err && txn != nullptr)
return write_txn{txn};
release_context(ctx);
if (err != MDB_MAP_RESIZED)
return {lmdb::error(err)};
MONERO_CHECK(this->resize());
}
return {lmdb::error(MDB_MAP_RESIZED)};
}
database::database(environment env)
: env(std::move(env)), ctx{{}, ATOMIC_FLAG_INIT}
{
if (handle())
{
const int err = mdb_env_set_userctx(handle(), std::addressof(ctx));
if (err)
MONERO_THROW(lmdb::error(err), "Failed to set user context");
}
}
database::~database() noexcept
{
while (ctx.active);
}
expect<void> database::resize() noexcept
{
MONERO_PRECOND(handle() != nullptr);
while (ctx.lock.test_and_set());
while (ctx.active);
MDB_envinfo info{};
MONERO_LMDB_CHECK(mdb_env_info(handle(), &info));
const std::size_t resize = std::min(info.me_mapsize, max_resize);
const int err = mdb_env_set_mapsize(handle(), info.me_mapsize + resize);
ctx.lock.clear();
if (err)
return {lmdb::error(err)};
return success();
}
expect<read_txn> database::create_read_txn(suspended_txn txn) noexcept
{
if (txn)
{
acquire_context(ctx);
const int err = mdb_txn_renew(txn.get());
if (err)
{
release_context(ctx);
return {lmdb::error(err)};
}
return read_txn{txn.release()};
}
auto new_txn = do_create_txn(MDB_RDONLY);
if (new_txn)
return read_txn{new_txn->release()};
return new_txn.error();
}
expect<suspended_txn> database::reset_txn(read_txn txn) noexcept
{
MONERO_PRECOND(txn != nullptr);
mdb_txn_reset(txn.get());
release_context(ctx);
return suspended_txn{txn.release()};
}
expect<write_txn> database::create_write_txn() noexcept
{
return do_create_txn(0);
}
expect<void> database::commit(write_txn txn) noexcept
{
MONERO_PRECOND(txn != nullptr);
MONERO_LMDB_CHECK(mdb_txn_commit(txn.get()));
txn.release();
release_context(ctx);
return success();
}
} // lmdb

138
src/lmdb/database.h Normal file
View file

@ -0,0 +1,138 @@
// Copyright (c) 2018, The Monero Project
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without modification, are
// permitted provided that the following conditions are met:
//
// 1. Redistributions of source code must retain the above copyright notice, this list of
// conditions and the following disclaimer.
//
// 2. Redistributions in binary form must reproduce the above copyright notice, this list
// of conditions and the following disclaimer in the documentation and/or other
// materials provided with the distribution.
//
// 3. Neither the name of the copyright holder nor the names of its contributors may be
// used to endorse or promote products derived from this software without specific
// prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#pragma once
#include <atomic>
#include <cstddef>
#include <lmdb.h>
#include <memory>
#include <type_traits>
#include "common/expect.h"
#include "lmdb/error.h"
#include "lmdb/transaction.h"
namespace lmdb
{
//! Closes LMDB environment handle.
struct close_env
{
void operator()(MDB_env* ptr) const noexcept
{
if (ptr)
mdb_env_close(ptr);
}
};
using environment = std::unique_ptr<MDB_env, close_env>;
//! \return LMDB environment at `path` with a max of `max_dbs` tables.
expect<environment> open_environment(const char* path, MDB_dbi max_dbs) noexcept;
//! Context given to LMDB.
struct context
{
std::atomic<std::size_t> active;
std::atomic_flag lock;
};
//! Manages a LMDB environment for safe memory-map resizing. Thread-safe.
class database
{
environment env;
context ctx;
//! \return The LMDB environment associated with the object.
MDB_env* handle() const noexcept { return env.get(); }
expect<write_txn> do_create_txn(unsigned int flags) noexcept;
public:
database(environment env);
database(database&&) = delete;
database(database const&) = delete;
virtual ~database() noexcept;
database& operator=(database&&) = delete;
database& operator=(database const&) = delete;
/*!
Resize the memory map for the LMDB environment. Will block until
all reads/writes on the environment complete.
*/
expect<void> resize() noexcept;
//! \return A read only LMDB transaction, reusing `txn` if provided.
expect<read_txn> create_read_txn(suspended_txn txn = nullptr) noexcept;
//! \return `txn` after releasing context.
expect<suspended_txn> reset_txn(read_txn txn) noexcept;
//! \return A read-write LMDB transaction.
expect<write_txn> create_write_txn() noexcept;
//! Commit the read-write transaction.
expect<void> commit(write_txn txn) noexcept;
/*!
Create a write transaction, pass it to `f`, then try to commit
the write if `f` succeeds.
\tparam F must be callable with signature `expect<T>(MDB_txn&)`.
\param f must be re-startable if `lmdb::error(MDB_MAP_FULL)`.
\return The result of calling `f`.
*/
template<typename F>
typename std::result_of<F(MDB_txn&)>::type try_write(F f, unsigned attempts = 3)
{
for (unsigned i = 0; i < attempts; ++i)
{
expect<write_txn> txn = create_write_txn();
if (!txn)
return txn.error();
MONERO_PRECOND(*txn != nullptr);
const auto wrote = f(*(*txn));
if (wrote)
{
MONERO_CHECK(commit(std::move(*txn)));
return wrote;
}
if (wrote != lmdb::error(MDB_MAP_FULL))
return wrote;
txn->reset();
MONERO_CHECK(this->resize());
}
return {lmdb::error(MDB_MAP_FULL)};
}
};
} // lmdb

98
src/lmdb/error.cpp Normal file
View file

@ -0,0 +1,98 @@
// Copyright (c) 2014-2018, The Monero Project
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without modification, are
// permitted provided that the following conditions are met:
//
// 1. Redistributions of source code must retain the above copyright notice, this list of
// conditions and the following disclaimer.
//
// 2. Redistributions in binary form must reproduce the above copyright notice, this list
// of conditions and the following disclaimer in the documentation and/or other
// materials provided with the distribution.
//
// 3. Neither the name of the copyright holder nor the names of its contributors may be
// used to endorse or promote products derived from this software without specific
// prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include "error.h"
#include <lmdb.h>
#include <string>
namespace {
struct category final : std::error_category
{
virtual const char* name() const noexcept override final
{
return "lmdb::error_category()";
}
virtual std::string message(int value) const override final
{
char const* const msg = mdb_strerror(value);
if (msg)
return msg;
return "Unknown lmdb::error_category() value";
}
virtual std::error_condition default_error_condition(int value) const noexcept override final
{
switch (value)
{
case MDB_KEYEXIST:
case MDB_NOTFOUND:
break; // map to nothing generic
case MDB_PAGE_NOTFOUND:
case MDB_CORRUPTED:
return std::errc::state_not_recoverable;
case MDB_PANIC:
case MDB_VERSION_MISMATCH:
case MDB_INVALID:
break; // map to nothing generic
case MDB_MAP_FULL:
return std::errc::no_buffer_space;
case MDB_DBS_FULL:
break; // map to nothing generic
case MDB_READERS_FULL:
case MDB_TLS_FULL:
return std::errc::no_lock_available;
case MDB_TXN_FULL:
case MDB_CURSOR_FULL:
case MDB_PAGE_FULL:
case MDB_MAP_RESIZED:
break; // map to nothing generic
case MDB_INCOMPATIBLE:
return std::errc::invalid_argument;
case MDB_BAD_RSLOT:
case MDB_BAD_TXN:
case MDB_BAD_VALSIZE:
case MDB_BAD_DBI:
return std::errc::invalid_argument;
default:
return std::error_condition{value, std::generic_category()};
}
return std::error_condition{value, *this};
}
};
}
namespace lmdb
{
std::error_category const& error_category() noexcept
{
static const category instance{};
return instance;
}
}

64
src/lmdb/error.h Normal file
View file

@ -0,0 +1,64 @@
// Copyright (c) 2014-2018, The Monero Project
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without modification, are
// permitted provided that the following conditions are met:
//
// 1. Redistributions of source code must retain the above copyright notice, this list of
// conditions and the following disclaimer.
//
// 2. Redistributions in binary form must reproduce the above copyright notice, this list
// of conditions and the following disclaimer in the documentation and/or other
// materials provided with the distribution.
//
// 3. Neither the name of the copyright holder nor the names of its contributors may be
// used to endorse or promote products derived from this software without specific
// prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#pragma once
#include <system_error>
#include <type_traits>
//! Executes a LMDB command, and returns errors via `lmdb::error` enum.
#define MONERO_LMDB_CHECK(...) \
do \
{ \
const int err = __VA_ARGS__ ; \
if (err) \
return {lmdb::error(err)}; \
} while (0)
namespace lmdb
{
//! Tracks LMDB error codes.
enum class error : int
{
// 0 is reserved for no error, as per expect<T>
// All other errors are the values reported by LMDB
};
std::error_category const& error_category() noexcept;
inline std::error_code make_error_code(error value) noexcept
{
return std::error_code{int(value), error_category()};
}
}
namespace std
{
template<>
struct is_error_code_enum<::lmdb::error>
: true_type
{};
}

264
src/lmdb/key_stream.h Normal file
View file

@ -0,0 +1,264 @@
// Copyright (c) 2018, The Monero Project
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without modification, are
// permitted provided that the following conditions are met:
//
// 1. Redistributions of source code must retain the above copyright notice, this list of
// conditions and the following disclaimer.
//
// 2. Redistributions in binary form must reproduce the above copyright notice, this list
// of conditions and the following disclaimer in the documentation and/or other
// materials provided with the distribution.
//
// 3. Neither the name of the copyright holder nor the names of its contributors may be
// used to endorse or promote products derived from this software without specific
// prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#pragma once
#include <boost/range/iterator_range.hpp>
#include <cstdint>
#include <cstring>
#include <iterator>
#include <lmdb.h>
#include <utility>
#include "lmdb/value_stream.h"
#include "span.h"
namespace lmdb
{
/*!
An InputIterator for a fixed-sized LMDB key and value. `operator++`
iterates over keys.
\tparam K Key type in database records.
\tparam V Value type in database records.
\note This meets requirements for an InputIterator only. The iterator
can only be incremented and dereferenced. All copies of an iterator
share the same LMDB cursor, and therefore incrementing any copy will
change the cursor state for all (incrementing an iterator will
invalidate all prior copies of the iterator). Usage is identical
to `std::istream_iterator`.
*/
template<typename K, typename V>
class key_iterator
{
MDB_cursor* cur;
epee::span<const std::uint8_t> key;
void increment()
{
// MDB_NEXT_MULTIPLE doesn't work if only one value is stored :/
if (cur)
key = lmdb::stream::get(*cur, MDB_NEXT_NODUP, sizeof(K), sizeof(V)).first;
}
public:
using value_type = std::pair<K, boost::iterator_range<value_iterator<V>>>;
using reference = value_type;
using pointer = void;
using difference_type = std::size_t;
using iterator_category = std::input_iterator_tag;
//! Construct an "end" iterator.
key_iterator() noexcept
: cur(nullptr), key()
{}
/*!
\param cur Iterate over keys starting at this cursor position.
\throw std::system_error if unexpected LMDB error. This can happen
if `cur` is invalid.
*/
key_iterator(MDB_cursor* cur)
: cur(cur), key()
{
if (cur)
key = lmdb::stream::get(*cur, MDB_GET_CURRENT, sizeof(K), sizeof(V)).first;
}
//! \return True if `this` is one-past the last key.
bool is_end() const noexcept { return key.empty(); }
//! \return True iff `rhs` is referencing `this` key.
bool equal(key_iterator const& rhs) const noexcept
{
return
(key.empty() && rhs.key.empty()) ||
key.data() == rhs.key.data();
}
/*!
Moves iterator to next key or end. Invalidates all prior copies of
the iterator.
*/
key_iterator& operator++()
{
increment();
return *this;
}
/*!
Moves iterator to next key or end.
\return A copy that is already invalidated, ignore
*/
key_iterator operator++(int)
{
key_iterator out{*this};
increment();
return out;
}
//! \pre `!is_end()` \return {current key, current value range}
value_type operator*() const
{
return {get_key(), make_value_range()};
}
//! \pre `!is_end()` \return Current key
K get_key() const noexcept
{
assert(!is_end());
K out;
std::memcpy(std::addressof(out), key.data(), sizeof(out));
return out;
}
/*!
Return a C++ iterator over database values from current cursor
position that will reach `.is_end()` after the last duplicate key
record. Calling `make_iterator()` will return an iterator whose
`operator*` will return an entire value (`V`).
`make_iterator<MONERO_FIELD(account, id)>()` will return an
iterator whose `operator*` will return a `decltype(account.id)`
object - the other fields in the struct `account` are never copied
from the database.
\throw std::system_error if LMDB has unexpected errors.
\return C++ iterator starting at current cursor position.
*/
template<typename T = V, typename F = T, std::size_t offset = 0>
value_iterator<T, F, offset> make_value_iterator() const
{
static_assert(std::is_same<T, V>(), "bad MONERO_FIELD usage?");
return {cur};
}
/*!
Return a range from current cursor position until last duplicate
key record. Useful in for-each range loops or in templated code
expecting a range of elements. Calling `make_range()` will return
a range of `T` objects. `make_range<MONERO_FIELD(account, id)>()`
will return a range of `decltype(account.id)` objects - the other
fields in the struct `account` are never copied from the database.
\throw std::system_error if LMDB has unexpected errors.
\return An InputIterator range over values at cursor position.
*/
template<typename T = V, typename F = T, std::size_t offset = 0>
boost::iterator_range<value_iterator<T, F, offset>> make_value_range() const
{
return {make_value_iterator<T, F, offset>(), value_iterator<T, F, offset>{}};
}
};
/*!
C++ wrapper for a LMDB read-only cursor on a fixed-sized key `K` and
value `V`.
\tparam K key type being stored by each record.
\tparam V value type being stored by each record.
\tparam D cleanup functor for the cursor; usually unique per db/table.
*/
template<typename K, typename V, typename D>
class key_stream
{
std::unique_ptr<MDB_cursor, D> cur;
public:
//! Take ownership of `cur` without changing position. `nullptr` valid.
explicit key_stream(std::unique_ptr<MDB_cursor, D> cur)
: cur(std::move(cur))
{}
key_stream(key_stream&&) = default;
key_stream(key_stream const&) = delete;
~key_stream() = default;
key_stream& operator=(key_stream&&) = default;
key_stream& operator=(key_stream const&) = delete;
/*!
Give up ownership of the cursor. `make_iterator()` and
`make_range()` can still be invoked, but return the empty set.
\return Currently owned LMDB cursor.
*/
std::unique_ptr<MDB_cursor, D> give_cursor() noexcept
{
return {std::move(cur)};
}
/*!
Place the stream back at the first key/value. Newly created
iterators will start at the first value again.
\note Invalidates all current iterators, including those created
with `make_iterator` or `make_range`. Also invalidates all
`value_iterator`s created with `key_iterator`.
*/
void reset()
{
if (cur)
lmdb::stream::get(*cur, MDB_FIRST, 0, 0);
}
/*!
\throw std::system_error if LMDB has unexpected errors.
\return C++ iterator over database keys from current cursor
position that will reach `.is_end()` after the last key.
*/
key_iterator<K, V> make_iterator() const
{
return {cur.get()};
}
/*!
\throw std::system_error if LMDB has unexpected errors.
\return Range from current cursor position until last key record.
Useful in for-each range loops or in templated code
*/
boost::iterator_range<key_iterator<K, V>> make_range() const
{
return {make_iterator(), key_iterator<K, V>{}};
}
};
template<typename K, typename V>
inline
bool operator==(key_iterator<K, V> const& lhs, key_iterator<K, V> const& rhs) noexcept
{
return lhs.equal(rhs);
}
template<typename K, typename V>
inline
bool operator!=(key_iterator<K, V> const& lhs, key_iterator<K, V> const& rhs) noexcept
{
return !lhs.equal(rhs);
}
} // lmdb

43
src/lmdb/table.cpp Normal file
View file

@ -0,0 +1,43 @@
// Copyright (c) 2018, The Monero Project
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without modification, are
// permitted provided that the following conditions are met:
//
// 1. Redistributions of source code must retain the above copyright notice, this list of
// conditions and the following disclaimer.
//
// 2. Redistributions in binary form must reproduce the above copyright notice, this list
// of conditions and the following disclaimer in the documentation and/or other
// materials provided with the distribution.
//
// 3. Neither the name of the copyright holder nor the names of its contributors may be
// used to endorse or promote products derived from this software without specific
// prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include "table.h"
namespace lmdb
{
expect<MDB_dbi> table::open(MDB_txn& write_txn) const noexcept
{
MONERO_PRECOND(name != nullptr);
MDB_dbi out;
MONERO_LMDB_CHECK(mdb_dbi_open(&write_txn, name, flags, &out));
if (key_cmp && !(flags & MDB_INTEGERKEY))
MONERO_LMDB_CHECK(mdb_set_compare(&write_txn, out, key_cmp));
if (value_cmp && !(flags & MDB_INTEGERDUP))
MONERO_LMDB_CHECK(mdb_set_dupsort(&write_txn, out, value_cmp));
return out;
}
}

120
src/lmdb/table.h Normal file
View file

@ -0,0 +1,120 @@
#pragma once
#include <utility>
#include "common/expect.h"
#include "lmdb/error.h"
#include "lmdb/key_stream.h"
#include "lmdb/util.h"
#include "lmdb/value_stream.h"
namespace lmdb
{
//! Helper for grouping typical LMDB DBI options.
struct table
{
char const* const name;
const unsigned flags;
MDB_cmp_func const* const key_cmp;
MDB_cmp_func const* const value_cmp;
//! \pre `name != nullptr` \return Open table.
expect<MDB_dbi> open(MDB_txn& write_txn) const noexcept;
};
//! Helper for grouping typical LMDB DBI options when key and value are fixed types.
template<typename K, typename V>
struct basic_table : table
{
using key_type = K;
using value_type = V;
//! \return Additional LMDB flags based on `flags` value.
static constexpr unsigned compute_flags(const unsigned flags) noexcept
{
return flags | ((flags & MDB_DUPSORT) ? MDB_DUPFIXED : 0);
}
constexpr explicit basic_table(const char* name, unsigned flags = 0, MDB_cmp_func value_cmp = nullptr) noexcept
: table{name, compute_flags(flags), &lmdb::less<lmdb::native_type<K>>, value_cmp}
{}
/*!
\tparam U must be same as `V`; used for sanity checking.
\tparam F is the type within `U` that is being extracted.
\tparam offset to `F` within `U`.
\note If using `F` and `offset` to retrieve a specific field, use
`MONERO_FIELD` macro in `src/lmdb/util.h` which calculates the
offset automatically.
\return Value of type `F` at `offset` within `value` which has
type `U`.
*/
template<typename U, typename F = U, std::size_t offset = 0>
static expect<F> get_value(MDB_val value) noexcept
{
static_assert(std::is_same<U, V>(), "bad MONERO_FIELD?");
static_assert(std::is_pod<F>(), "F must be POD");
static_assert(sizeof(F) + offset <= sizeof(U), "bad field type and/or offset");
if (value.mv_size != sizeof(U))
return {lmdb::error(MDB_BAD_VALSIZE)};
F out;
std::memcpy(std::addressof(out), static_cast<char*>(value.mv_data) + offset, sizeof(out));
return out;
}
/*!
\pre `cur != nullptr`.
\param cur Active cursor on table. Returned in object on success,
otherwise destroyed.
\return A handle to the first key/value in the table linked
to `cur` or an empty `key_stream`.
*/
template<typename D>
expect<key_stream<K, V, D>>
static get_key_stream(std::unique_ptr<MDB_cursor, D> cur) noexcept
{
MONERO_PRECOND(cur != nullptr);
MDB_val key;
MDB_val value;
const int err = mdb_cursor_get(cur.get(), &key, &value, MDB_FIRST);
if (err)
{
if (err != MDB_NOTFOUND)
return {lmdb::error(err)};
cur.reset(); // return empty set
}
return key_stream<K, V, D>{std::move(cur)};
}
/*!
\pre `cur != nullptr`.
\param cur Active cursor on table. Returned in object on success,
otherwise destroyed.
\return A handle to the first value at `key` in the table linked
to `cur` or an empty `value_stream`.
*/
template<typename D>
expect<value_stream<V, D>>
static get_value_stream(K const& key, std::unique_ptr<MDB_cursor, D> cur) noexcept
{
MONERO_PRECOND(cur != nullptr);
MDB_val key_bytes = lmdb::to_val(key);
MDB_val value;
const int err = mdb_cursor_get(cur.get(), &key_bytes, &value, MDB_SET);
if (err)
{
if (err != MDB_NOTFOUND)
return {lmdb::error(err)};
cur.reset(); // return empty set
}
return value_stream<V, D>{std::move(cur)};
}
};
} // lmdb

95
src/lmdb/transaction.h Normal file
View file

@ -0,0 +1,95 @@
// Copyright (c) 2018, The Monero Project
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without modification, are
// permitted provided that the following conditions are met:
//
// 1. Redistributions of source code must retain the above copyright notice, this list of
// conditions and the following disclaimer.
//
// 2. Redistributions in binary form must reproduce the above copyright notice, this list
// of conditions and the following disclaimer in the documentation and/or other
// materials provided with the distribution.
//
// 3. Neither the name of the copyright holder nor the names of its contributors may be
// used to endorse or promote products derived from this software without specific
// prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#pragma once
#include <lmdb.h>
#include <memory>
#include "lmdb/error.h"
//! Uses C++ type system to differentiate between cursors
#define MONERO_CURSOR(name) \
struct close_ ## name : ::lmdb::close_cursor {}; \
using name = std::unique_ptr< MDB_cursor, close_ ## name >;
namespace lmdb
{
struct abort_txn
{
void operator()(MDB_txn* ptr) const noexcept
{
if (ptr)
mdb_txn_abort(ptr);
}
};
/*!
Only valid if used via `create_read_txn()`. Decrements active count in
associated `context`, and aborts a LMDB transaction (`mdb_txn_abort`).
*/
struct release_read_txn
{
void operator()(MDB_txn* ptr) const noexcept;
// implementation in database.cpp
};
/*!
Only valid if used via `create_write_txn()`. Decrements active count in
associated `context`, and aborts a LMDB transaction (`mdb_txn_abort`).
*/
struct abort_write_txn
{
void operator()(MDB_txn* ptr) const noexcept
{
release_read_txn{}(ptr);
}
};
struct close_cursor
{
void operator()(MDB_cursor* ptr) const noexcept
{
if (ptr)
mdb_cursor_close(ptr);
}
};
template<typename D>
inline expect<std::unique_ptr<MDB_cursor, D>>
open_cursor(MDB_txn& txn, MDB_dbi tbl) noexcept
{
MDB_cursor* cur = nullptr;
MONERO_LMDB_CHECK(mdb_cursor_open(&txn, tbl, &cur));
return std::unique_ptr<MDB_cursor, D>{cur};
}
// The below use the C++ type system to designate `MDB_txn` status.
using suspended_txn = std::unique_ptr<MDB_txn, abort_txn>;
using read_txn = std::unique_ptr<MDB_txn, release_read_txn>;
using write_txn = std::unique_ptr<MDB_txn, abort_write_txn>;
} // lmdb

149
src/lmdb/util.h Normal file
View file

@ -0,0 +1,149 @@
// Copyright (c) 2018, The Monero Project
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without modification, are
// permitted provided that the following conditions are met:
//
// 1. Redistributions of source code must retain the above copyright notice, this list of
// conditions and the following disclaimer.
//
// 2. Redistributions in binary form must reproduce the above copyright notice, this list
// of conditions and the following disclaimer in the documentation and/or other
// materials provided with the distribution.
//
// 3. Neither the name of the copyright holder nor the names of its contributors may be
// used to endorse or promote products derived from this software without specific
// prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#pragma once
#include <cstddef>
#include <cstring>
#include <lmdb.h>
#include <type_traits>
#include <utility>
#include "span.h"
/*! Calculates types and offset of struct field. Use in template arguments for
`table::get_value`, `value_iterator::get_value`,
`value_stream::make_iterator`, or `value_stream::make_range`. */
#define MONERO_FIELD(obj, field) \
obj , decltype(std::declval<obj>().field) , offsetof(obj, field)
//! Expands to `lmdb::less` for the value `field` within `obj`.
#define MONERO_SORT_BY(obj, field) \
&::lmdb::less< \
lmdb::native_type<decltype(std::declval<obj>().field)>, \
offsetof(obj, field) \
>
//! Expands to `lmdb::compare` for the value `field` within `obj`.
#define MONERO_COMPARE(obj, field) \
&::lmdb::compare< \
decltype(std::declval<obj>().field), \
offsetof(obj, field) \
>
namespace lmdb
{
//! Prevent instantiation of `std::underlying_type<T>` when `T` is not enum.
template<typename T>
struct identity
{
using type = T;
};
/*!
Get the native type for enums, or return `T` unchanged. Useful for
merging generated machine code for templated functions that use enums
with identical size-widths without relying on aggressive identical
comdat folding (ICF) support in linker. So with enum defintion
`enum class enum_foo : unsigned long {};` will always yield
`assert(&func_foo<unsigned long> == &func_foo<native_type<enum_foo>>)`.
*/
template<typename T>
using native_type = typename std::conditional<
std::is_enum<T>::value, std::underlying_type<T>, identity<T>
>::type::type;
//! \return `value` as its native type.
template<typename T, typename U = typename std::underlying_type<T>::type>
inline constexpr U to_native(T value) noexcept
{
return U(value);
}
//! \return `value` bytes in a LMDB `MDB_val` object.
template<typename T>
inline MDB_val to_val(T&& value) noexcept
{
// lmdb does not touch user data, so const_cast is acceptable
static_assert(!std::is_rvalue_reference<T&&>(), "cannot use temporary value");
void const* const temp = reinterpret_cast<void const*>(std::addressof(value));
return MDB_val{sizeof(value), const_cast<void*>(temp)};
}
//! \return A span over the same chunk of memory as `value`.
inline constexpr epee::span<const std::uint8_t> to_byte_span(MDB_val value) noexcept
{
return {static_cast<const std::uint8_t*>(value.mv_data), value.mv_size};
}
/*!
A LMDB comparison function that uses `operator<`.
\tparam T has a defined `operator<` .
\tparam offset to `T` within the value.
\return -1 if `left < right`, 1 if `right < left`, and 0 otherwise.
*/
template<typename T, std::size_t offset = 0>
inline int less(MDB_val const* left, MDB_val const* right) noexcept
{
if (!left || !right || left->mv_size < sizeof(T) + offset || right->mv_size < sizeof(T) + offset)
{
assert("invalid use of custom comparison" == 0);
return -1;
}
T left_val;
T right_val;
std::memcpy(std::addressof(left_val), static_cast<char*>(left->mv_data) + offset, sizeof(T));
std::memcpy(std::addressof(right_val), static_cast<char*>(right->mv_data) + offset, sizeof(T));
return left_val < right_val ? -1 : bool(right_val < left_val);
}
/*!
A LMDB comparison function that uses `std::memcmp`.
\toaram T is `!epee::has_padding`
\tparam offset to `T` within the value.
\return The result of `std::memcmp` over the value.
*/
template<typename T, std::size_t offset = 0>
inline int compare(MDB_val const* left, MDB_val const* right) noexcept
{
static_assert(!epee::has_padding<T>(), "memcmp will not work");
if (!left || !right || left->mv_size < sizeof(T) + offset || right->mv_size < sizeof(T) + offset)
{
assert("invalid use of custom comparison" == 0);
return -1;
}
return std::memcmp(
static_cast<char*>(left->mv_data) + offset,
static_cast<char*>(right->mv_data) + offset,
sizeof(T)
);
}
} // lmdb

74
src/lmdb/value_stream.cpp Normal file
View file

@ -0,0 +1,74 @@
// Copyright (c) 2018, The Monero Project
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without modification, are
// permitted provided that the following conditions are met:
//
// 1. Redistributions of source code must retain the above copyright notice, this list of
// conditions and the following disclaimer.
//
// 2. Redistributions in binary form must reproduce the above copyright notice, this list
// of conditions and the following disclaimer in the documentation and/or other
// materials provided with the distribution.
//
// 3. Neither the name of the copyright holder nor the names of its contributors may be
// used to endorse or promote products derived from this software without specific
// prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include "value_stream.h"
#include <stdexcept>
#include "common/expect.h"
#include "lmdb/error.h"
#include "lmdb/util.h"
namespace lmdb
{
namespace stream
{
std::size_t count(MDB_cursor* cur)
{
std::size_t out = 0;
if (cur)
{
const int rc = mdb_cursor_count(cur, &out);
if (rc)
MONERO_THROW(lmdb::error(rc), "mdb_cursor_count");
}
return out;
}
std::pair<epee::span<const std::uint8_t>, epee::span<const std::uint8_t>>
get(MDB_cursor& cur, MDB_cursor_op op, std::size_t key, std::size_t value)
{
MDB_val key_bytes{};
MDB_val value_bytes{};
const int rc = mdb_cursor_get(&cur, &key_bytes, &value_bytes, op);
if (rc)
{
if (rc == MDB_NOTFOUND)
return {};
MONERO_THROW(lmdb::error(rc), "mdb_cursor_get");
}
if (key && key != key_bytes.mv_size)
MONERO_THROW(lmdb::error(MDB_BAD_VALSIZE), "mdb_cursor_get key");
if (value && (value_bytes.mv_size % value != 0 || value_bytes.mv_size == 0))
MONERO_THROW(lmdb::error(MDB_BAD_VALSIZE), "mdb_cursor_get value");
return {lmdb::to_byte_span(key_bytes), lmdb::to_byte_span(value_bytes)};
}
}
}

287
src/lmdb/value_stream.h Normal file
View file

@ -0,0 +1,287 @@
// Copyright (c) 2018, The Monero Project
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without modification, are
// permitted provided that the following conditions are met:
//
// 1. Redistributions of source code must retain the above copyright notice, this list of
// conditions and the following disclaimer.
//
// 2. Redistributions in binary form must reproduce the above copyright notice, this list
// of conditions and the following disclaimer in the documentation and/or other
// materials provided with the distribution.
//
// 3. Neither the name of the copyright holder nor the names of its contributors may be
// used to endorse or promote products derived from this software without specific
// prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#pragma once
#include <boost/range/iterator_range.hpp>
#include <cstdint>
#include <cstring>
#include <iterator>
#include <lmdb.h>
#include <utility>
#include "span.h"
namespace lmdb
{
namespace stream
{
/*
\throw std::system_error if unexpected LMDB error.
\return 0 if `cur == nullptr`, otherwise count of values at current key.
*/
std::size_t count(MDB_cursor* cur);
/*!
Calls `mdb_cursor_get` and does some error checking.
\param cur is given to `mdb_cursor_get` without modification.
\param op is passed to `mdb_cursor_get` without modification.
\param key expected key size or 0 to skip key size check.
\param value expected value size or 0 to skip value size check.
\throw std::system_error if `key != 0` and `key_.mv_size != key`.
\throw std::system_error if `value != 0` and `value_.mv_size != value`.
\throw std::system_error if `mdb_cursor_get` returns any error
other than `MDB_NOTFOUND`.
\return {key bytes, value bytes} or two empty spans if `MDB_NOTFOUND`.
*/
std::pair<epee::span<const std::uint8_t>, epee::span<const std::uint8_t>>
get(MDB_cursor& cur, MDB_cursor_op op, std::size_t key, std::size_t value);
}
/*!
An InputIterator for a fixed-sized LMDB value at a specific key.
\tparam T The value type at the specific key.
\tparam F The value type being returned when dereferenced.
\tparam offset to `F` within `T`.
\note This meets requirements for an InputIterator only. The iterator
can only be incremented and dereferenced. All copies of an iterator
share the same LMDB cursor, and therefore incrementing any copy will
change the cursor state for all (incrementing an iterator will
invalidate all prior copies of the iterator). Usage is identical
to `std::istream_iterator`.
*/
template<typename T, typename F = T, std::size_t offset = 0>
class value_iterator
{
MDB_cursor* cur;
epee::span<const std::uint8_t> values;
void increment()
{
values.remove_prefix(sizeof(T));
if (values.empty() && cur)
values = lmdb::stream::get(*cur, MDB_NEXT_DUP, 0, sizeof(T)).second;
}
public:
using value_type = F;
using reference = value_type;
using pointer = void;
using difference_type = std::size_t;
using iterator_category = std::input_iterator_tag;
//! Construct an "end" iterator.
value_iterator() noexcept
: cur(nullptr), values()
{}
/*!
\param cur Iterate over values starting at this cursor position.
\throw std::system_error if unexpected LMDB error. This can happen
if `cur` is invalid.
*/
value_iterator(MDB_cursor* cur)
: cur(cur), values()
{
if (cur)
values = lmdb::stream::get(*cur, MDB_GET_CURRENT, 0, sizeof(T)).second;
}
value_iterator(value_iterator const&) = default;
~value_iterator() = default;
value_iterator& operator=(value_iterator const&) = default;
//! \return True if `this` is one-past the last value.
bool is_end() const noexcept { return values.empty(); }
//! \return True iff `rhs` is referencing `this` value.
bool equal(value_iterator const& rhs) const noexcept
{
return
(values.empty() && rhs.values.empty()) ||
values.data() == rhs.values.data();
}
//! Invalidates all prior copies of the iterator.
value_iterator& operator++()
{
increment();
return *this;
}
//! \return A copy that is already invalidated, ignore
value_iterator operator++(int)
{
value_iterator out{*this};
increment();
return out;
}
/*!
Get a specific field within `F`. Default behavior is to return
the entirety of `U`, despite the filtering logic of `operator*`.
\pre `!is_end()`
\tparam U must match `T`, used for `MONERO_FIELD` sanity checking.
\tparam G field type to extract from the value
\tparam uoffset to `G` type, or `0` when `std::is_same<U, G>()`.
\return The field `G`, at `uoffset` within `U`.
*/
template<typename U, typename G = U, std::size_t uoffset = 0>
G get_value() const noexcept
{
static_assert(std::is_same<U, T>(), "bad MONERO_FIELD usage?");
static_assert(std::is_pod<U>(), "value type must be pod");
static_assert(std::is_pod<G>(), "field type must be pod");
static_assert(sizeof(G) + uoffset <= sizeof(U), "bad field and/or offset");
assert(sizeof(G) + uoffset <= values.size());
assert(!is_end());
G value;
std::memcpy(std::addressof(value), values.data() + uoffset, sizeof(value));
return value;
}
//! \pre `!is_end()` \return The field `F`, at `offset`, within `T`.
value_type operator*() const noexcept { return get_value<T, F, offset>(); }
};
/*!
C++ wrapper for a LMDB read-only cursor on a fixed-sized value `T`.
\tparam T value type being stored by each record.
\tparam D cleanup functor for the cursor; usually unique per db/table.
*/
template<typename T, typename D>
class value_stream
{
std::unique_ptr<MDB_cursor, D> cur;
public:
//! Take ownership of `cur` without changing position. `nullptr` valid.
explicit value_stream(std::unique_ptr<MDB_cursor, D> cur)
: cur(std::move(cur))
{}
value_stream(value_stream&&) = default;
value_stream(value_stream const&) = delete;
~value_stream() = default;
value_stream& operator=(value_stream&&) = default;
value_stream& operator=(value_stream const&) = delete;
/*!
Give up ownership of the cursor. `count()`, `make_iterator()` and
`make_range()` can still be invoked, but return the empty set.
\return Currently owned LMDB cursor.
*/
std::unique_ptr<MDB_cursor, D> give_cursor() noexcept
{
return {std::move(cur)};
}
/*!
Place the stream back at the first value. Newly created iterators
will start at the first value again.
\note Invalidates all current iterators from `this`, including
those created with `make_iterator` or `make_range`.
*/
void reset()
{
if (cur)
lmdb::stream::get(*cur, MDB_FIRST_DUP, 0, 0);
}
/*!
\throw std::system_error if LMDB has unexpected errors.
\return Number of values at this key.
*/
std::size_t count() const
{
return lmdb::stream::count(cur.get());
}
/*!
Return a C++ iterator over database values from current cursor
position that will reach `.is_end()` after the last duplicate key
record. Calling `make_iterator()` will return an iterator whose
`operator*` will return entire value (`T`).
`make_iterator<MONERO_FIELD(account, id)>()` will return an
iterator whose `operator*` will return a `decltype(account.id)`
object - the other fields in the struct `account` are never copied
from the database.
\throw std::system_error if LMDB has unexpected errors.
\return C++ iterator starting at current cursor position.
*/
template<typename U = T, typename F = U, std::size_t offset = 0>
value_iterator<U, F, offset> make_iterator() const
{
static_assert(std::is_same<U, T>(), "was MONERO_FIELD used with wrong type?");
return {cur.get()};
}
/*!
Return a range from current cursor position until last duplicate
key record. Useful in for-each range loops or in templated code
expecting a range of elements. Calling `make_range()` will return
a range of `T` objects. `make_range<MONERO_FIELD(account, id)>()`
will return a range of `decltype(account.id)` objects - the other
fields in the struct `account` are never copied from the database.
\throw std::system_error if LMDB has unexpected errors.
\return An InputIterator range over values at cursor position.
*/
template<typename U = T, typename F = U, std::size_t offset = 0>
boost::iterator_range<value_iterator<U, F, offset>> make_range() const
{
return {make_iterator<U, F, offset>(), value_iterator<U, F, offset>{}};
}
};
template<typename T, typename F, std::size_t offset>
inline
bool operator==(value_iterator<T, F, offset> const& lhs, value_iterator<T, F, offset> const& rhs) noexcept
{
return lhs.equal(rhs);
}
template<typename T, typename F, std::size_t offset>
inline
bool operator!=(value_iterator<T, F, offset> const& lhs, value_iterator<T, F, offset> const& rhs) noexcept
{
return !lhs.equal(rhs);
}
} // lmdb

2
tests/unit_tests/CMakeLists.txt
View file

@ -56,6 +56,7 @@ set(unit_tests_sources
keccak.cpp
logging.cpp
long_term_block_weight.cpp
lmdb.cpp
main.cpp
memwipe.cpp
mlocker.cpp
@ -101,6 +102,7 @@ target_link_libraries(unit_tests
cryptonote_protocol
cryptonote_core
blockchain_db
lmdb_lib
rpc
net
serialization

404
tests/unit_tests/lmdb.cpp Normal file
View file

@ -0,0 +1,404 @@
// Copyright (c) 2014-2018, The Monero Project
//
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without modification, are
// permitted provided that the following conditions are met:
//
// 1. Redistributions of source code must retain the above copyright notice, this list of
// conditions and the following disclaimer.
//
// 2. Redistributions in binary form must reproduce the above copyright notice, this list
// of conditions and the following disclaimer in the documentation and/or other
// materials provided with the distribution.
//
// 3. Neither the name of the copyright holder nor the names of its contributors may be
// used to endorse or promote products derived from this software without specific
// prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include <boost/range/algorithm_ext/iota.hpp>
#include <boost/range/algorithm/equal.hpp>
#include <gtest/gtest.h>
#include "lmdb/database.h"
#include "lmdb/table.h"
#include "lmdb/transaction.h"
#include "lmdb/util.h"
namespace
{
enum class choice : unsigned {};
enum class big_choice : unsigned long {};
struct bytes {
char data[16];
};
MONERO_CURSOR(test_cursor);
template<typename T>
int run_compare(T left, T right, MDB_cmp_func* cmp)
{
MDB_val left_val = lmdb::to_val(left);
MDB_val right_val = lmdb::to_val(right);
return (*cmp)(&left_val, &right_val);
}
}
TEST(LMDB, Traits)
{
EXPECT_TRUE((std::is_same<void, lmdb::identity<void>::type>()));
EXPECT_TRUE((std::is_same<unsigned, lmdb::identity<unsigned>::type>()));
EXPECT_TRUE((std::is_same<void, lmdb::native_type<void>>()));
EXPECT_TRUE((std::is_same<unsigned, lmdb::native_type<unsigned>>()));
EXPECT_TRUE((std::is_same<unsigned, lmdb::native_type<choice>>()));
EXPECT_TRUE((std::is_same<unsigned long, lmdb::native_type<big_choice>>()));
}
TEST(LMDB, ToNative)
{
enum class negative_choice : int {};
EXPECT_TRUE((std::is_same<unsigned, decltype(lmdb::to_native(choice(0)))>()));
EXPECT_TRUE(
(std::is_same<unsigned long, decltype(lmdb::to_native(big_choice(0)))>())
);
EXPECT_TRUE(
(std::is_same<int, decltype(lmdb::to_native(negative_choice(0)))>())
);
EXPECT_EQ(unsigned(0), lmdb::to_native(choice(0)));
EXPECT_EQ(unsigned(0xffffffff), lmdb::to_native(choice(0xffffffff)));
EXPECT_EQ(-1, lmdb::to_native(negative_choice(-1)));
// test constexpr
static_assert(100 == lmdb::to_native(choice(100)), "to_native failed");
static_assert(-100 == lmdb::to_native(negative_choice(-100)), "to_native failed");
}
TEST(LMDB, Conversions)
{
struct one
{
big_choice i;
choice j;
};
const one test{big_choice(100), choice(95)};
one test2{big_choice(1000), choice(950)};
EXPECT_EQ(&test, lmdb::to_val(test).mv_data);
EXPECT_NE(&test2, lmdb::to_val(test).mv_data);
EXPECT_EQ(
&test,
static_cast<const void*>(lmdb::to_byte_span(lmdb::to_val(test)).begin())
);
EXPECT_EQ(sizeof(test), lmdb::to_val(test).mv_size);
EXPECT_EQ(sizeof(test), lmdb::to_byte_span(lmdb::to_val(test)).size());
EXPECT_EQ(&test2, lmdb::to_val(test2).mv_data);
EXPECT_NE(&test, lmdb::to_val(test2).mv_data);
EXPECT_EQ(
&test2,
static_cast<const void*>(lmdb::to_byte_span(lmdb::to_val(test2)).begin())
);
EXPECT_EQ(sizeof(test2), lmdb::to_val(test2).mv_size);
EXPECT_EQ(sizeof(test2), lmdb::to_byte_span(lmdb::to_val(test2)).size());
}
TEST(LMDB, LessSort)
{
struct one
{
unsigned i;
unsigned j;
};
struct two
{
unsigned i;
choice j;
};
EXPECT_EQ(0, run_compare(0u, 0u, &lmdb::less<unsigned>));
EXPECT_EQ(-1, run_compare(0u, 1u, &lmdb::less<unsigned>));
EXPECT_EQ(1, run_compare(1u, 0u, &lmdb::less<unsigned>));
EXPECT_EQ(0, run_compare<one>({0, 1}, {0, 1}, &lmdb::less<unsigned, sizeof(unsigned)>));
EXPECT_EQ(-1, run_compare<one>({0, 0}, {0, 1}, &lmdb::less<unsigned, sizeof(unsigned)>));
EXPECT_EQ(1, run_compare<one>({0, 1}, {0, 0}, &lmdb::less<unsigned, sizeof(unsigned)>));
EXPECT_EQ(0, run_compare<one>({0, 1}, {0, 1}, MONERO_SORT_BY(one, j)));
EXPECT_EQ(-1, run_compare<one>({0, 0}, {0, 1}, MONERO_SORT_BY(one, j)));
EXPECT_EQ(1, run_compare<one>({0, 1}, {0, 0}, MONERO_SORT_BY(one, j)));
EXPECT_EQ(0, run_compare<two>({0, choice(1)}, {0, choice(1)}, MONERO_SORT_BY(two, j)));
EXPECT_EQ(-1, run_compare<two>({0, choice(0)}, {0, choice(1)}, MONERO_SORT_BY(two, j)));
EXPECT_EQ(1, run_compare<two>({0, choice(1)}, {0, choice(0)}, MONERO_SORT_BY(two, j)));
// compare function addresses
EXPECT_EQ((MONERO_SORT_BY(one, i)), (MONERO_SORT_BY(two, i)));
EXPECT_EQ((MONERO_SORT_BY(one, j)), (MONERO_SORT_BY(two, j)));
EXPECT_NE((MONERO_SORT_BY(one, i)), (MONERO_SORT_BY(two, j)));
EXPECT_NE((MONERO_SORT_BY(one, j)), (MONERO_SORT_BY(two, i)));
}
TEST(LMDB, SortCompare)
{
struct one
{
unsigned i;
bytes j;
};
one test{55};
boost::iota(test.j.data, 10);
const one test2 = test;
EXPECT_EQ(0, run_compare(test, test2, MONERO_COMPARE(one, j)));
test.j.data[15] = 1;
EXPECT_GT(0, run_compare(test, test2, MONERO_COMPARE(one, j)));
test.j.data[15] = 100;
EXPECT_LT(0, run_compare(test, test2, MONERO_COMPARE(one, j)));
}
TEST(LMDB, Table)
{
struct one
{
bytes i;
bytes j;
};
constexpr lmdb::basic_table<choice, bytes> test{"foo"};
EXPECT_STREQ("foo", test.name);
static_assert(test.flags == 0, "bad flags");
static_assert(&lmdb::less<unsigned> == test.key_cmp, "bad key_cmp");
static_assert(test.value_cmp == nullptr, "bad value_cmp");
EXPECT_TRUE(test.get_value<bytes>(MDB_val{}).matches(std::errc::invalid_argument));
lmdb::basic_table<big_choice, one> test2{
"foo2", MDB_DUPSORT, &lmdb::compare<one>
};
EXPECT_STREQ("foo2", test2.name);
EXPECT_EQ((MDB_DUPSORT | MDB_DUPFIXED), test2.flags);
EXPECT_EQ(&lmdb::less<unsigned long>, test2.key_cmp);
EXPECT_EQ(&lmdb::compare<one>, test2.value_cmp);
EXPECT_TRUE(test2.get_value<one>(MDB_val{}).matches(std::errc::invalid_argument));
one record{};
boost::iota(record.i.data, 0);
boost::iota(record.i.data, 20);
const one record_copy = MONERO_UNWRAP(test2.get_value<one>(lmdb::to_val(record)));
EXPECT_TRUE(boost::equal(record.i.data, record_copy.i.data));
EXPECT_TRUE(boost::equal(record.j.data, record_copy.j.data));
const bytes j_copy = MONERO_UNWRAP(
test2.get_value<MONERO_FIELD(one, j)>(lmdb::to_val(record))
);
EXPECT_TRUE(boost::equal(record.j.data, j_copy.data));
EXPECT_TRUE(
test.get_key_stream(test_cursor{}).matches(std::errc::invalid_argument)
);
EXPECT_TRUE(
test2.get_key_stream(test_cursor{}).matches(std::errc::invalid_argument)
);
EXPECT_TRUE(
test.get_value_stream(choice(0), test_cursor{}).matches(std::errc::invalid_argument)
);
EXPECT_TRUE(
test2.get_value_stream(big_choice(0), test_cursor{}).matches(std::errc::invalid_argument)
);
}
TEST(LMDB, InvalidDatabase)
{
lmdb::database test{lmdb::environment{}};
EXPECT_TRUE(test.resize().matches(std::errc::invalid_argument));
EXPECT_TRUE(test.create_read_txn().matches(std::errc::invalid_argument));
EXPECT_TRUE(test.reset_txn(lmdb::read_txn{}).matches(std::errc::invalid_argument));
EXPECT_TRUE(test.create_write_txn().matches(std::errc::invalid_argument));
EXPECT_TRUE(test.commit(lmdb::write_txn{}).matches(std::errc::invalid_argument));
EXPECT_TRUE(
test.try_write( [](MDB_txn&) { return success(); } ).matches(std::errc::invalid_argument)
);
}
TEST(LMDB, InvalidValueStream)
{
struct one
{
choice i;
choice j;
bytes k;
};
lmdb::value_stream<one, close_test_cursor> test{test_cursor{}};
EXPECT_TRUE((std::is_same<one, decltype(*(test.make_iterator()))>()));
EXPECT_TRUE((std::is_same<one, decltype(*(test.make_range().begin()))>()));
EXPECT_TRUE(
(std::is_same<bytes, decltype(*(test.make_iterator<MONERO_FIELD(one, k)>()))>())
);
EXPECT_TRUE(
(std::is_same<bytes, decltype(*(test.make_range<MONERO_FIELD(one, k)>().begin()))>())
);
EXPECT_NO_THROW(test.reset());
EXPECT_EQ(0u, test.count());
EXPECT_TRUE(test.make_iterator().is_end());
EXPECT_TRUE(test.make_range().empty());
EXPECT_EQ(nullptr, test.give_cursor());
EXPECT_EQ(0u, test.count());
EXPECT_TRUE(test.make_iterator().is_end());
EXPECT_TRUE(test.make_range().empty());
EXPECT_EQ(nullptr, test.give_cursor());
}
TEST(LMDB, InvalidValueIterator)
{
struct one
{
choice i;
choice j;
bytes k;
};
lmdb::value_iterator<one> test1{};
EXPECT_TRUE((std::is_same<one, decltype(*test1)>()));
EXPECT_TRUE(
(std::is_same<bytes, decltype(test1.get_value<MONERO_FIELD(one, k)>())>())
);
EXPECT_TRUE(test1.is_end());
EXPECT_NO_THROW(++test1);
EXPECT_NO_THROW(test1++);
EXPECT_TRUE(test1.is_end());
lmdb::value_iterator<one> test2{nullptr};
EXPECT_TRUE(test2.is_end());
EXPECT_NO_THROW(++test2);
EXPECT_NO_THROW(test2++);
EXPECT_TRUE(test2.is_end());
EXPECT_TRUE(test1.equal(test2));
EXPECT_TRUE(test2.equal(test1));
EXPECT_TRUE(test1 == test2);
EXPECT_TRUE(test2 == test1);
EXPECT_FALSE(test1 != test2);
EXPECT_FALSE(test2 != test1);
lmdb::value_iterator<MONERO_FIELD(one, k)> test3{};
EXPECT_TRUE((std::is_same<bytes, decltype(*test3)>()));
EXPECT_TRUE((std::is_same<one, decltype(test3.get_value<one>())>()));
EXPECT_TRUE(
(std::is_same<choice, decltype(test1.get_value<MONERO_FIELD(one, j)>())>())
);
EXPECT_TRUE(test3.is_end());
EXPECT_NO_THROW(++test3);
EXPECT_NO_THROW(test3++);
EXPECT_TRUE(test3.is_end());
}
TEST(LMDB, InvalidKeyStream)
{
struct one
{
choice i;
choice j;
bytes k;
};
using record = std::pair<choice, boost::iterator_range<lmdb::value_iterator<one>>>;
lmdb::key_stream<choice, one, close_test_cursor> test{test_cursor{}};
EXPECT_TRUE((std::is_same<record, decltype(*(test.make_iterator()))>()));
EXPECT_TRUE((std::is_same<record, decltype(*(test.make_range().begin()))>()));
EXPECT_NO_THROW(test.reset());
EXPECT_TRUE(test.make_iterator().is_end());
EXPECT_TRUE(test.make_range().empty());
EXPECT_EQ(nullptr, test.give_cursor());
EXPECT_TRUE(test.make_iterator().is_end());
EXPECT_TRUE(test.make_range().empty());
EXPECT_EQ(nullptr, test.give_cursor());
}
TEST(LMDB, InvalidKeyIterator)
{
struct one
{
choice i;
choice j;
bytes k;
};
using record = std::pair<choice, boost::iterator_range<lmdb::value_iterator<one>>>;
lmdb::key_iterator<choice, one> test1{};
EXPECT_TRUE((std::is_same<record, decltype(*test1)>()));
EXPECT_TRUE((std::is_same<choice, decltype(test1.get_key())>()));
EXPECT_TRUE((std::is_same<one, decltype(*(test1.make_value_iterator()))>()));
EXPECT_TRUE((std::is_same<one, decltype(*(test1.make_value_range().begin()))>()));
EXPECT_TRUE(
(std::is_same<bytes, decltype(*(test1.make_value_iterator<MONERO_FIELD(one, k)>()))>())
);
EXPECT_TRUE(
(std::is_same<bytes, decltype(*(test1.make_value_range<MONERO_FIELD(one, k)>().begin()))>())
);
EXPECT_TRUE(test1.is_end());
EXPECT_NO_THROW(++test1);
EXPECT_NO_THROW(test1++);
EXPECT_TRUE(test1.is_end());
EXPECT_TRUE(test1.make_value_iterator().is_end());
EXPECT_TRUE(test1.make_value_range().empty());
lmdb::key_iterator<choice, one> test2{nullptr};
EXPECT_TRUE(test2.is_end());
EXPECT_NO_THROW(++test2);
EXPECT_NO_THROW(test2++);
EXPECT_TRUE(test2.is_end());
EXPECT_TRUE(test2.make_value_iterator().is_end());
EXPECT_TRUE(test2.make_value_range().empty());
EXPECT_TRUE(test1.equal(test2));
EXPECT_TRUE(test2.equal(test1));
EXPECT_TRUE(test1 == test2);
EXPECT_TRUE(test2 == test1);
EXPECT_FALSE(test1 != test2);
EXPECT_FALSE(test2 != test1);
}