Merge pull request #241 from hyc/vmem

Don't use exceptions in low-level mem allocator
This commit is contained in:
hyc 2022-05-09 17:31:56 +01:00 committed by GitHub
commit 84bc0c9e42
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 121 additions and 68 deletions

View File

@ -39,7 +39,7 @@ src/bytecode_machine.cpp
src/cpu.cpp src/cpu.cpp
src/dataset.cpp src/dataset.cpp
src/soft_aes.cpp src/soft_aes.cpp
src/virtual_memory.cpp src/virtual_memory.c
src/vm_interpreted.cpp src/vm_interpreted.cpp
src/allocator.cpp src/allocator.cpp
src/assembly_generator_x86.cpp src/assembly_generator_x86.cpp

View File

@ -29,7 +29,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include <new> #include <new>
#include "allocator.hpp" #include "allocator.hpp"
#include "intrin_portable.h" #include "intrin_portable.h"
#include "virtual_memory.hpp" #include "virtual_memory.h"
#include "common.hpp" #include "common.hpp"
namespace randomx { namespace randomx {
@ -50,11 +50,14 @@ namespace randomx {
template struct AlignedAllocator<CacheLineSize>; template struct AlignedAllocator<CacheLineSize>;
void* LargePageAllocator::allocMemory(size_t count) { void* LargePageAllocator::allocMemory(size_t count) {
return allocLargePagesMemory(count); void *mem = allocLargePagesMemory(count);
if (mem == nullptr)
throw std::bad_alloc();
return mem;
} }
void LargePageAllocator::freeMemory(void* ptr, size_t count) { void LargePageAllocator::freeMemory(void* ptr, size_t count) {
freePagedMemory(ptr, count); freePagedMemory(ptr, count);
}; };
} }

View File

@ -42,7 +42,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include "common.hpp" #include "common.hpp"
#include "dataset.hpp" #include "dataset.hpp"
#include "virtual_memory.hpp" #include "virtual_memory.h"
#include "superscalar.hpp" #include "superscalar.hpp"
#include "blake2_generator.hpp" #include "blake2_generator.hpp"
#include "reciprocal.h" #include "reciprocal.h"

View File

@ -31,7 +31,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include "superscalar.hpp" #include "superscalar.hpp"
#include "program.hpp" #include "program.hpp"
#include "reciprocal.h" #include "reciprocal.h"
#include "virtual_memory.hpp" #include "virtual_memory.h"
namespace ARMV8A { namespace ARMV8A {
@ -93,6 +93,8 @@ JitCompilerA64::JitCompilerA64()
, literalPos(ImulRcpLiteralsEnd) , literalPos(ImulRcpLiteralsEnd)
, num32bitLiterals(0) , num32bitLiterals(0)
{ {
if (code == nullptr)
throw std::runtime_error("allocMemoryPages");
memset(reg_changed_offset, 0, sizeof(reg_changed_offset)); memset(reg_changed_offset, 0, sizeof(reg_changed_offset));
memcpy(code, (void*) randomx_program_aarch64, CodeSize); memcpy(code, (void*) randomx_program_aarch64, CodeSize);

View File

@ -34,7 +34,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include "superscalar.hpp" #include "superscalar.hpp"
#include "program.hpp" #include "program.hpp"
#include "reciprocal.h" #include "reciprocal.h"
#include "virtual_memory.hpp" #include "virtual_memory.h"
namespace randomx { namespace randomx {
/* /*
@ -225,6 +225,8 @@ namespace randomx {
JitCompilerX86::JitCompilerX86() { JitCompilerX86::JitCompilerX86() {
code = (uint8_t*)allocMemoryPages(CodeSize); code = (uint8_t*)allocMemoryPages(CodeSize);
if (code == nullptr)
throw std::runtime_error("allocMemoryPages");
memcpy(code, codePrologue, prologueSize); memcpy(code, codePrologue, prologueSize);
memcpy(code + epilogueOffset, codeEpilogue, epilogueSize); memcpy(code + epilogueOffset, codeEpilogue, epilogueSize);
} }

View File

@ -113,6 +113,10 @@ extern "C" {
cache = nullptr; cache = nullptr;
} }
} }
if (cache && cache->memory == nullptr) {
randomx_release_cache(cache);
cache = nullptr;
}
return cache; return cache;
} }
@ -162,6 +166,10 @@ extern "C" {
dataset = nullptr; dataset = nullptr;
} }
} }
if (dataset && dataset->memory == nullptr) {
randomx_release_dataset(dataset);
dataset = nullptr;
}
return dataset; return dataset;
} }

View File

@ -26,9 +26,7 @@ OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/ */
#include "virtual_memory.hpp" #include "virtual_memory.h"
#include <stdexcept>
#if defined(_WIN32) || defined(__CYGWIN__) #if defined(_WIN32) || defined(__CYGWIN__)
#include <windows.h> #include <windows.h>
@ -40,10 +38,12 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
# if TARGET_OS_OSX # if TARGET_OS_OSX
# define USE_PTHREAD_JIT_WP 1 # define USE_PTHREAD_JIT_WP 1
# include <pthread.h> # include <pthread.h>
# include <sys/utsname.h>
# endif # endif
#endif #endif
#include <sys/types.h> #include <sys/types.h>
#include <sys/mman.h> #include <sys/mman.h>
#include <errno.h>
#ifndef MAP_ANONYMOUS #ifndef MAP_ANONYMOUS
#define MAP_ANONYMOUS MAP_ANON #define MAP_ANONYMOUS MAP_ANON
#endif #endif
@ -53,27 +53,48 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#define PAGE_EXECUTE_READWRITE (PROT_READ | PROT_WRITE | PROT_EXEC) #define PAGE_EXECUTE_READWRITE (PROT_READ | PROT_WRITE | PROT_EXEC)
#endif #endif
#if defined(_WIN32) || defined(__CYGWIN__) #if defined(USE_PTHREAD_JIT_WP) && defined(MAC_OS_VERSION_11_0) \
std::string getErrorMessage(const char* function) { && MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_VERSION_11_0
LPSTR messageBuffer = nullptr; static int MacOSchecked, MacOSver;
size_t size = FormatMessageA(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS, /* This function is used implicitly by clang's __builtin_available() checker.
NULL, GetLastError(), MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), (LPSTR)&messageBuffer, 0, NULL); * When cross-compiling, the library containing this function doesn't exist,
std::string message(messageBuffer, size); * and linking will fail because the symbol is unresolved. The function here
LocalFree(messageBuffer); * is a quick and dirty hack to get close enough to identify MacOSX 11.0.
return std::string(function) + std::string(": ") + message; */
static int32_t __isOSVersionAtLeast(int32_t major, int32_t minor, int32_t subminor) {
if (!MacOSchecked) {
struct utsname ut;
int mmaj, mmin;
uname(&ut);
sscanf(ut.release, "%d.%d", &mmaj, &mmin);
// The utsname release version is 9 greater than the canonical OS version
mmaj -= 9;
MacOSver = (mmaj << 8) | mmin;
MacOSchecked = 1;
}
return MacOSver >= ((major << 8) | minor);
} }
#endif
void setPrivilege(const char* pszPrivilege, BOOL bEnable) {
#if defined(_WIN32) || defined(__CYGWIN__)
#define Fail(func) do {*errfunc = func; return GetLastError();} while(0)
int setPrivilege(const char* pszPrivilege, BOOL bEnable, char **errfunc) {
HANDLE hToken; HANDLE hToken;
TOKEN_PRIVILEGES tp; TOKEN_PRIVILEGES tp;
BOOL status; BOOL status;
DWORD error; DWORD error = 0;
*errfunc = NULL;
if (!OpenProcessToken(GetCurrentProcess(), TOKEN_ADJUST_PRIVILEGES | TOKEN_QUERY, &hToken)) if (!OpenProcessToken(GetCurrentProcess(), TOKEN_ADJUST_PRIVILEGES | TOKEN_QUERY, &hToken))
throw std::runtime_error(getErrorMessage("OpenProcessToken")); Fail("OpenProcessToken");
if (!LookupPrivilegeValue(NULL, pszPrivilege, &tp.Privileges[0].Luid)) if (!LookupPrivilegeValue(NULL, pszPrivilege, &tp.Privileges[0].Luid)) {
throw std::runtime_error(getErrorMessage("LookupPrivilegeValue")); *errfunc = "LookupPrivilegeValue";
error = GetLastError();
goto out;
}
tp.PrivilegeCount = 1; tp.PrivilegeCount = 1;
@ -85,20 +106,28 @@ void setPrivilege(const char* pszPrivilege, BOOL bEnable) {
status = AdjustTokenPrivileges(hToken, FALSE, &tp, 0, (PTOKEN_PRIVILEGES)NULL, 0); status = AdjustTokenPrivileges(hToken, FALSE, &tp, 0, (PTOKEN_PRIVILEGES)NULL, 0);
error = GetLastError(); error = GetLastError();
if (!status || (error != ERROR_SUCCESS)) if (!status || (error != ERROR_SUCCESS)) {
throw std::runtime_error(getErrorMessage("AdjustTokenPrivileges")); *errfunc = "AdjustTokenPrivileges";
goto out;
}
if (!CloseHandle(hToken)) out:
throw std::runtime_error(getErrorMessage("CloseHandle")); if (!CloseHandle(hToken)) {
if (*errfunc == NULL) {
*errfunc = "CloseHandle";
error = GetLastError();
}
}
return error;
} }
#else
#define Fail(func) do {*errfunc = func; return errno;} while(0)
#endif #endif
void* allocMemoryPages(std::size_t bytes) { void* allocMemoryPages(size_t bytes) {
void* mem; void* mem;
#if defined(_WIN32) || defined(__CYGWIN__) #if defined(_WIN32) || defined(__CYGWIN__)
mem = VirtualAlloc(nullptr, bytes, MEM_COMMIT, PAGE_READWRITE); mem = VirtualAlloc(NULL, bytes, MEM_COMMIT, PAGE_READWRITE);
if (mem == nullptr)
throw std::runtime_error(getErrorMessage("allocMemoryPages - VirtualAlloc"));
#else #else
#if defined(__NetBSD__) #if defined(__NetBSD__)
#define RESERVED_FLAGS PROT_MPROTECT(PROT_EXEC) #define RESERVED_FLAGS PROT_MPROTECT(PROT_EXEC)
@ -112,9 +141,7 @@ void* allocMemoryPages(std::size_t bytes) {
#define MEXTRA 0 #define MEXTRA 0
#define PEXTRA 0 #define PEXTRA 0
#endif #endif
mem = mmap(nullptr, bytes, PAGE_READWRITE | RESERVED_FLAGS | PEXTRA, MAP_ANONYMOUS | MAP_PRIVATE | MEXTRA, -1, 0); mem = mmap(NULL, bytes, PAGE_READWRITE | RESERVED_FLAGS | PEXTRA, MAP_ANONYMOUS | MAP_PRIVATE | MEXTRA, -1, 0);
if (mem == MAP_FAILED)
throw std::runtime_error("allocMemoryPages - mmap failed");
#if defined(USE_PTHREAD_JIT_WP) && defined(MAC_OS_VERSION_11_0) \ #if defined(USE_PTHREAD_JIT_WP) && defined(MAC_OS_VERSION_11_0) \
&& MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_VERSION_11_0 && MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_VERSION_11_0
if (__builtin_available(macOS 11.0, *)) { if (__builtin_available(macOS 11.0, *)) {
@ -125,76 +152,81 @@ void* allocMemoryPages(std::size_t bytes) {
return mem; return mem;
} }
static inline void pageProtect(void* ptr, std::size_t bytes, int rules) { static inline int pageProtect(void* ptr, size_t bytes, int rules, char **errfunc) {
#if defined(_WIN32) || defined(__CYGWIN__) #if defined(_WIN32) || defined(__CYGWIN__)
DWORD oldp; DWORD oldp;
if (!VirtualProtect(ptr, bytes, (DWORD)rules, &oldp)) { if (!VirtualProtect(ptr, bytes, (DWORD)rules, &oldp)) {
throw std::runtime_error(getErrorMessage("VirtualProtect")); Fail("VirtualProtect");
} }
#else #else
if (-1 == mprotect(ptr, bytes, rules)) if (-1 == mprotect(ptr, bytes, rules))
throw std::runtime_error("mprotect failed"); Fail("mprotect");
#endif #endif
return 0;
} }
void setPagesRW(void* ptr, std::size_t bytes) { void setPagesRW(void* ptr, size_t bytes) {
char *errfunc;
#if defined(USE_PTHREAD_JIT_WP) && defined(MAC_OS_VERSION_11_0) \ #if defined(USE_PTHREAD_JIT_WP) && defined(MAC_OS_VERSION_11_0) \
&& MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_VERSION_11_0 && MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_VERSION_11_0
if (__builtin_available(macOS 11.0, *)) { if (__builtin_available(macOS 11.0, *)) {
pthread_jit_write_protect_np(false); pthread_jit_write_protect_np(false);
} else { } else {
pageProtect(ptr, bytes, PAGE_READWRITE); pageProtect(ptr, bytes, PAGE_READWRITE, &errfunc);
} }
#else #else
pageProtect(ptr, bytes, PAGE_READWRITE); pageProtect(ptr, bytes, PAGE_READWRITE, &errfunc);
#endif #endif
} }
void setPagesRX(void* ptr, std::size_t bytes) { void setPagesRX(void* ptr, size_t bytes) {
char *errfunc;
#if defined(USE_PTHREAD_JIT_WP) && defined(MAC_OS_VERSION_11_0) \ #if defined(USE_PTHREAD_JIT_WP) && defined(MAC_OS_VERSION_11_0) \
&& MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_VERSION_11_0 && MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_VERSION_11_0
if (__builtin_available(macOS 11.0, *)) { if (__builtin_available(macOS 11.0, *)) {
pthread_jit_write_protect_np(true); pthread_jit_write_protect_np(true);
} else { } else {
pageProtect(ptr, bytes, PAGE_EXECUTE_READ); pageProtect(ptr, bytes, PAGE_EXECUTE_READ, &errfunc);
} }
#else #else
pageProtect(ptr, bytes, PAGE_EXECUTE_READ); pageProtect(ptr, bytes, PAGE_EXECUTE_READ, &errfunc);
#endif #endif
} }
void setPagesRWX(void* ptr, std::size_t bytes) { void setPagesRWX(void* ptr, size_t bytes) {
pageProtect(ptr, bytes, PAGE_EXECUTE_READWRITE); char *errfunc;
pageProtect(ptr, bytes, PAGE_EXECUTE_READWRITE, &errfunc);
} }
void* allocLargePagesMemory(std::size_t bytes) { void* allocLargePagesMemory(size_t bytes) {
void* mem; void* mem;
char *errfunc;
#if defined(_WIN32) || defined(__CYGWIN__) #if defined(_WIN32) || defined(__CYGWIN__)
setPrivilege("SeLockMemoryPrivilege", 1); if (setPrivilege("SeLockMemoryPrivilege", 1, &errfunc))
return NULL;
auto pageMinimum = GetLargePageMinimum(); auto pageMinimum = GetLargePageMinimum();
if (pageMinimum > 0) if (pageMinimum <= 0) {
mem = VirtualAlloc(NULL, alignSize(bytes, pageMinimum), MEM_COMMIT | MEM_RESERVE | MEM_LARGE_PAGES, PAGE_READWRITE); errfunc = "No large pages";
else return NULL;
throw std::runtime_error("allocLargePagesMemory - Large pages are not supported"); }
if (mem == nullptr) mem = VirtualAlloc(NULL, alignSize(bytes, pageMinimum), MEM_COMMIT | MEM_RESERVE | MEM_LARGE_PAGES, PAGE_READWRITE);
throw std::runtime_error(getErrorMessage("allocLargePagesMemory - VirtualAlloc"));
#else #else
#ifdef __APPLE__ #ifdef __APPLE__
mem = mmap(nullptr, bytes, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, VM_FLAGS_SUPERPAGE_SIZE_2MB, 0); mem = mmap(NULL, bytes, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, VM_FLAGS_SUPERPAGE_SIZE_2MB, 0);
#elif defined(__FreeBSD__) #elif defined(__FreeBSD__)
mem = mmap(nullptr, bytes, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_ALIGNED_SUPER, -1, 0); mem = mmap(NULL, bytes, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_ALIGNED_SUPER, -1, 0);
#elif defined(__OpenBSD__) || defined(__NetBSD__) #elif defined(__OpenBSD__) || defined(__NetBSD__)
mem = MAP_FAILED; // OpenBSD does not support huge pages mem = MAP_FAILED; // OpenBSD does not support huge pages
#else #else
mem = mmap(nullptr, bytes, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB | MAP_POPULATE, -1, 0); mem = mmap(NULL, bytes, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB | MAP_POPULATE, -1, 0);
#endif #endif
if (mem == MAP_FAILED) if (mem == MAP_FAILED)
throw std::runtime_error("allocLargePagesMemory - mmap failed"); mem = NULL;
#endif #endif
return mem; return mem;
} }
void freePagedMemory(void* ptr, std::size_t bytes) { void freePagedMemory(void* ptr, size_t bytes) {
#if defined(_WIN32) || defined(__CYGWIN__) #if defined(_WIN32) || defined(__CYGWIN__)
VirtualFree(ptr, 0, MEM_RELEASE); VirtualFree(ptr, 0, MEM_RELEASE);
#else #else

View File

@ -28,15 +28,21 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#pragma once #pragma once
#include <cstddef> #ifdef __cplusplus
extern "C" {
#endif
constexpr std::size_t alignSize(std::size_t pos, std::size_t align) { #include <stddef.h>
return ((pos - 1) / align + 1) * align;
#define alignSize(pos, align) (((pos - 1) / align + 1) * align)
void* allocMemoryPages(size_t);
void setPagesRW(void*, size_t);
void setPagesRX(void*, size_t);
void setPagesRWX(void*, size_t);
void* allocLargePagesMemory(size_t);
void freePagedMemory(void*, size_t);
#ifdef __cplusplus
} }
#endif
void* allocMemoryPages(std::size_t);
void setPagesRW(void*, std::size_t);
void setPagesRX(void*, std::size_t);
void setPagesRWX(void*, std::size_t);
void* allocLargePagesMemory(std::size_t);
void freePagedMemory(void*, std::size_t);