blob: f52be55990b36a704f01900417546d635955033c [file] [log] [blame]
// Copyright 2019 The Marl Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "marl/memory.h"
#include "marl/debug.h"
#include "marl/sanitizers.h"
#include <cstring>
#if defined(__linux__) || defined(__FreeBSD__) || defined(__APPLE__) || defined(__EMSCRIPTEN__)
#include <sys/mman.h>
#include <unistd.h>
namespace {
// This was a static in pageSize(), but due to the following TSAN false-positive
// bug, this has been moved out to a global.
// https://gcc.gnu.org/bugzilla/show_bug.cgi?id=68338
const size_t kPageSize = sysconf(_SC_PAGESIZE);
inline size_t pageSize() {
return kPageSize;
}
inline void* allocatePages(size_t count) {
auto mapping = mmap(nullptr, count * pageSize(), PROT_READ | PROT_WRITE,
MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
MARL_ASSERT(mapping != MAP_FAILED, "Failed to allocate %d pages", int(count));
if (mapping == MAP_FAILED) {
mapping = nullptr;
}
return mapping;
}
inline void freePages(void* ptr, size_t count) {
auto res = munmap(ptr, count * pageSize());
(void)res;
MARL_ASSERT(res == 0, "Failed to free %d pages at %p", int(count), ptr);
}
inline void protectPage(void* addr) {
auto res = mprotect(addr, pageSize(), PROT_NONE);
(void)res;
MARL_ASSERT(res == 0, "Failed to protect page at %p", addr);
}
} // anonymous namespace
#elif defined(__Fuchsia__)
#include <unistd.h>
#include <zircon/process.h>
#include <zircon/syscalls.h>
namespace {
// This was a static in pageSize(), but due to the following TSAN false-positive
// bug, this has been moved out to a global.
// https://gcc.gnu.org/bugzilla/show_bug.cgi?id=68338
const size_t kPageSize = sysconf(_SC_PAGESIZE);
inline size_t pageSize() {
return kPageSize;
}
inline void* allocatePages(size_t count) {
auto length = count * kPageSize;
zx_handle_t vmo;
if (zx_vmo_create(length, 0, &vmo) != ZX_OK) {
return nullptr;
}
zx_vaddr_t reservation;
zx_status_t status =
zx_vmar_map(zx_vmar_root_self(), ZX_VM_PERM_READ | ZX_VM_PERM_WRITE, 0,
vmo, 0, length, &reservation);
zx_handle_close(vmo);
(void)status;
MARL_ASSERT(status == ZX_OK, "Failed to allocate %d pages", int(count));
return reinterpret_cast<void*>(reservation);
}
inline void freePages(void* ptr, size_t count) {
auto length = count * kPageSize;
zx_status_t status = zx_vmar_unmap(zx_vmar_root_self(),
reinterpret_cast<zx_vaddr_t>(ptr), length);
(void)status;
MARL_ASSERT(status == ZX_OK, "Failed to free %d pages at %p", int(count),
ptr);
}
inline void protectPage(void* addr) {
zx_status_t status = zx_vmar_protect(
zx_vmar_root_self(), 0, reinterpret_cast<zx_vaddr_t>(addr), kPageSize);
(void)status;
MARL_ASSERT(status == ZX_OK, "Failed to protect page at %p", addr);
}
} // anonymous namespace
#elif defined(_WIN32)
#define WIN32_LEAN_AND_MEAN 1
#include <Windows.h>
namespace {
inline size_t pageSize() {
static auto size = [] {
SYSTEM_INFO systemInfo = {};
GetSystemInfo(&systemInfo);
return systemInfo.dwPageSize;
}();
return size;
}
inline void* allocatePages(size_t count) {
auto mapping = VirtualAlloc(nullptr, count * pageSize(),
MEM_COMMIT | MEM_RESERVE, PAGE_READWRITE);
MARL_ASSERT(mapping != nullptr, "Failed to allocate %d pages", int(count));
return mapping;
}
inline void freePages(void* ptr, size_t count) {
(void)count;
auto res = VirtualFree(ptr, 0, MEM_RELEASE);
(void)res;
MARL_ASSERT(res != 0, "Failed to free %d pages at %p", int(count), ptr);
}
inline void protectPage(void* addr) {
DWORD oldVal = 0;
auto res = VirtualProtect(addr, pageSize(), PAGE_NOACCESS, &oldVal);
(void)res;
MARL_ASSERT(res != 0, "Failed to protect page at %p", addr);
}
} // anonymous namespace
#else
#error "Page based allocation not implemented for this platform"
#endif
namespace {
// pagedMalloc() allocates size bytes of uninitialized storage with the
// specified minimum byte alignment using OS specific page mapping calls.
// If guardLow is true then reads or writes to the page below the returned
// address will cause a page fault.
// If guardHigh is true then reads or writes to the page above the allocated
// block will cause a page fault.
// The pointer returned must be freed with pagedFree().
void* pagedMalloc(size_t alignment,
size_t size,
bool guardLow,
bool guardHigh) {
(void)alignment;
MARL_ASSERT(alignment < pageSize(),
"alignment (0x%x) must be less than the page size (0x%x)",
int(alignment), int(pageSize()));
auto numRequestedPages = (size + pageSize() - 1) / pageSize();
auto numTotalPages =
numRequestedPages + (guardLow ? 1 : 0) + (guardHigh ? 1 : 0);
auto mem = reinterpret_cast<uint8_t*>(allocatePages(numTotalPages));
if (guardLow) {
protectPage(mem);
mem += pageSize();
}
if (guardHigh) {
protectPage(mem + numRequestedPages * pageSize());
}
return mem;
}
// pagedFree() frees the memory allocated with pagedMalloc().
void pagedFree(void* ptr,
size_t alignment,
size_t size,
bool guardLow,
bool guardHigh) {
(void)alignment;
MARL_ASSERT(alignment < pageSize(),
"alignment (0x%x) must be less than the page size (0x%x)",
int(alignment), int(pageSize()));
auto numRequestedPages = (size + pageSize() - 1) / pageSize();
auto numTotalPages =
numRequestedPages + (guardLow ? 1 : 0) + (guardHigh ? 1 : 0);
if (guardLow) {
ptr = reinterpret_cast<uint8_t*>(ptr) - pageSize();
}
freePages(ptr, numTotalPages);
}
// alignedMalloc() allocates size bytes of uninitialized storage with the
// specified minimum byte alignment. The pointer returned must be freed with
// alignedFree().
inline void* alignedMalloc(size_t alignment, size_t size) {
size_t allocSize = size + alignment + sizeof(void*);
auto allocation = malloc(allocSize);
auto aligned = reinterpret_cast<uint8_t*>(marl::alignUp(
reinterpret_cast<uintptr_t>(allocation), alignment)); // align
memcpy(aligned + size, &allocation, sizeof(void*)); // pointer-to-allocation
return aligned;
}
// alignedFree() frees memory allocated by alignedMalloc.
inline void alignedFree(void* ptr, size_t size) {
void* base;
memcpy(&base, reinterpret_cast<uint8_t*>(ptr) + size, sizeof(void*));
free(base);
}
class DefaultAllocator : public marl::Allocator {
public:
static DefaultAllocator instance;
virtual marl::Allocation allocate(
const marl::Allocation::Request& request) override {
void* ptr = nullptr;
if (request.useGuards) {
ptr = ::pagedMalloc(request.alignment, request.size, true, true);
} else if (request.alignment > 1U) {
ptr = ::alignedMalloc(request.alignment, request.size);
} else {
ptr = ::malloc(request.size);
}
MARL_ASSERT(ptr != nullptr, "Allocation failed");
MARL_ASSERT(reinterpret_cast<uintptr_t>(ptr) % request.alignment == 0,
"Allocation gave incorrect alignment");
marl::Allocation allocation;
allocation.ptr = ptr;
allocation.request = request;
return allocation;
}
virtual void free(const marl::Allocation& allocation) override {
if (allocation.request.useGuards) {
::pagedFree(allocation.ptr, allocation.request.alignment,
allocation.request.size, true, true);
} else if (allocation.request.alignment > 1U) {
::alignedFree(allocation.ptr, allocation.request.size);
} else {
::free(allocation.ptr);
}
}
};
DefaultAllocator DefaultAllocator::instance;
} // anonymous namespace
namespace marl {
Allocator* Allocator::Default = &DefaultAllocator::instance;
size_t pageSize() {
return ::pageSize();
}
} // namespace marl