blob: db5145b046d2a6345a2e83a679bc062d7a96559d [file] [log] [blame]
// Copyright 2016 The SwiftShader Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "ExecutableMemory.hpp"
#include "Debug.hpp"
#if defined(_WIN32)
# ifndef WIN32_LEAN_AND_MEAN
# define WIN32_LEAN_AND_MEAN
# endif
# include <Windows.h>
# include <intrin.h>
#elif defined(__Fuchsia__)
# include <unistd.h>
# include <zircon/process.h>
# include <zircon/syscalls.h>
#else
# include <errno.h>
# include <sys/mman.h>
# include <stdlib.h>
# include <unistd.h>
#endif
#if defined(__ANDROID__) && !defined(ANDROID_HOST_BUILD) && !defined(ANDROID_NDK_BUILD)
# include <sys/prctl.h>
#endif
#include <memory.h>
#undef allocate
#undef deallocate
#if(defined(__i386__) || defined(_M_IX86) || defined(__x86_64__) || defined(_M_X64)) && !defined(__x86__)
# define __x86__
#endif
#define STRINGIFY(x) #x
#define MACRO_STRINGIFY(x) STRINGIFY(x)
// A Clang extension to determine compiler features.
// We use it to detect Sanitizer builds (e.g. -fsanitize=memory).
#ifndef __has_feature
# define __has_feature(x) 0
#endif
namespace rr {
namespace {
struct Allocation
{
// size_t bytes;
unsigned char *block;
};
void *allocateRaw(size_t bytes, size_t alignment)
{
ASSERT((alignment & (alignment - 1)) == 0); // Power of 2 alignment.
#if defined(__linux__) && defined(REACTOR_ANONYMOUS_MMAP_NAME)
if(alignment < sizeof(void *))
{
return malloc(bytes);
}
else
{
void *allocation;
int result = posix_memalign(&allocation, alignment, bytes);
if(result != 0)
{
errno = result;
allocation = nullptr;
}
return allocation;
}
#else
unsigned char *block = new unsigned char[bytes + sizeof(Allocation) + alignment];
unsigned char *aligned = nullptr;
if(block)
{
aligned = (unsigned char *)((uintptr_t)(block + sizeof(Allocation) + alignment - 1) & -(intptr_t)alignment);
Allocation *allocation = (Allocation *)(aligned - sizeof(Allocation));
// allocation->bytes = bytes;
allocation->block = block;
}
return aligned;
#endif
}
#if defined(_WIN32)
DWORD permissionsToProtectMode(int permissions)
{
switch(permissions)
{
case PERMISSION_READ:
return PAGE_READONLY;
case PERMISSION_EXECUTE:
return PAGE_EXECUTE;
case PERMISSION_READ | PERMISSION_WRITE:
return PAGE_READWRITE;
case PERMISSION_READ | PERMISSION_EXECUTE:
return PAGE_EXECUTE_READ;
case PERMISSION_READ | PERMISSION_WRITE | PERMISSION_EXECUTE:
return PAGE_EXECUTE_READWRITE;
}
return PAGE_NOACCESS;
}
#endif
#if !defined(_WIN32) && !defined(__Fuchsia__)
int permissionsToMmapProt(int permissions)
{
int result = 0;
if(permissions & PERMISSION_READ)
{
result |= PROT_READ;
}
if(permissions & PERMISSION_WRITE)
{
result |= PROT_WRITE;
}
if(permissions & PERMISSION_EXECUTE)
{
result |= PROT_EXEC;
}
return result;
}
#endif // !defined(_WIN32) && !defined(__Fuchsia__)
#if defined(__linux__) && defined(REACTOR_ANONYMOUS_MMAP_NAME)
# if !defined(__ANDROID__) || defined(ANDROID_HOST_BUILD) || defined(ANDROID_NDK_BUILD)
// Create a file descriptor for anonymous memory with the given
// name. Returns -1 on failure.
// TODO: remove once libc wrapper exists.
static int memfd_create(const char *name, unsigned int flags)
{
# if __aarch64__
# define __NR_memfd_create 279
# elif __arm__
# define __NR_memfd_create 279
# elif __powerpc64__
# define __NR_memfd_create 360
# elif __i386__
# define __NR_memfd_create 356
# elif __x86_64__
# define __NR_memfd_create 319
# endif /* __NR_memfd_create__ */
# ifdef __NR_memfd_create
// In the event of no system call this returns -1 with errno set
// as ENOSYS.
return syscall(__NR_memfd_create, name, flags);
# else
return -1;
# endif
}
// Returns a file descriptor for use with an anonymous mmap, if
// memfd_create fails, -1 is returned. Note, the mappings should be
// MAP_PRIVATE so that underlying pages aren't shared.
int anonymousFd()
{
static int fd = memfd_create(MACRO_STRINGIFY(REACTOR_ANONYMOUS_MMAP_NAME), 0);
return fd;
}
# else // __ANDROID__ && !ANDROID_HOST_BUILD && !ANDROID_NDK_BUILD
int anonymousFd()
{
return -1;
}
# endif // __ANDROID__ && !ANDROID_HOST_BUILD && !ANDROID_NDK_BUILD
// Ensure there is enough space in the "anonymous" fd for length.
void ensureAnonFileSize(int anonFd, size_t length)
{
static size_t fileSize = 0;
if(length > fileSize)
{
[[maybe_unused]] int result = ftruncate(anonFd, length);
ASSERT(result == 0);
fileSize = length;
}
}
#endif // defined(__linux__) && defined(REACTOR_ANONYMOUS_MMAP_NAME)
#if defined(__Fuchsia__)
zx_vm_option_t permissionsToZxVmOptions(int permissions)
{
zx_vm_option_t result = 0;
if(permissions & PERMISSION_READ)
{
result |= ZX_VM_PERM_READ;
}
if(permissions & PERMISSION_WRITE)
{
result |= ZX_VM_PERM_WRITE;
}
if(permissions & PERMISSION_EXECUTE)
{
result |= ZX_VM_PERM_EXECUTE;
}
return result;
}
#endif // defined(__Fuchsia__)
} // anonymous namespace
size_t memoryPageSize()
{
static int pageSize = [] {
#if defined(_WIN32)
SYSTEM_INFO systemInfo;
GetSystemInfo(&systemInfo);
return systemInfo.dwPageSize;
#else
return sysconf(_SC_PAGESIZE);
#endif
}();
return pageSize;
}
void *allocate(size_t bytes, size_t alignment)
{
void *memory = allocateRaw(bytes, alignment);
// Zero-initialize the memory, for security reasons.
// MemorySanitizer builds skip this so that we can detect when we
// inadvertently rely on this, which would indicate a bug.
if(memory && !__has_feature(memory_sanitizer))
{
memset(memory, 0, bytes);
}
return memory;
}
void deallocate(void *memory)
{
#if defined(__linux__) && defined(REACTOR_ANONYMOUS_MMAP_NAME)
free(memory);
#else
if(memory)
{
unsigned char *aligned = (unsigned char *)memory;
Allocation *allocation = (Allocation *)(aligned - sizeof(Allocation));
delete[] allocation->block;
}
#endif
}
// Rounds |x| up to a multiple of |m|, where |m| is a power of 2.
inline uintptr_t roundUp(uintptr_t x, uintptr_t m)
{
ASSERT(m > 0 && (m & (m - 1)) == 0); // |m| must be a power of 2.
return (x + m - 1) & ~(m - 1);
}
void *allocateMemoryPages(size_t bytes, int permissions, bool need_exec)
{
size_t pageSize = memoryPageSize();
size_t length = roundUp(bytes, pageSize);
void *mapping = nullptr;
#if defined(_WIN32)
return VirtualAlloc(nullptr, length, MEM_COMMIT | MEM_RESERVE,
permissionsToProtectMode(permissions));
#elif defined(__linux__) && defined(REACTOR_ANONYMOUS_MMAP_NAME)
int flags = MAP_PRIVATE;
// Try to name the memory region for the executable code,
// to aid profilers.
int anonFd = anonymousFd();
if(anonFd == -1)
{
flags |= MAP_ANONYMOUS;
}
else
{
ensureAnonFileSize(anonFd, length);
}
mapping = mmap(
nullptr, length, permissionsToMmapProt(permissions), flags, anonFd, 0);
if(mapping == MAP_FAILED)
{
mapping = nullptr;
}
# if defined(__ANDROID__) && !defined(ANDROID_HOST_BUILD) && !defined(ANDROID_NDK_BUILD)
else
{
// On Android, prefer to use a non-standard prctl called
// PR_SET_VMA_ANON_NAME to set the name of a private anonymous
// mapping, as Android restricts EXECUTE permission on
// CoW/shared anonymous mappings with sepolicy neverallows.
prctl(PR_SET_VMA, PR_SET_VMA_ANON_NAME, mapping, length,
MACRO_STRINGIFY(REACTOR_ANONYMOUS_MMAP_NAME));
}
# endif // __ANDROID__ && !ANDROID_HOST_BUILD && !ANDROID_NDK_BUILD
#elif defined(__Fuchsia__)
zx_handle_t vmo;
if(zx_vmo_create(length, 0, &vmo) != ZX_OK)
{
return nullptr;
}
if(need_exec &&
zx_vmo_replace_as_executable(vmo, ZX_HANDLE_INVALID, &vmo) != ZX_OK)
{
return nullptr;
}
zx_vaddr_t reservation;
zx_status_t status = zx_vmar_map(
zx_vmar_root_self(), permissionsToZxVmOptions(permissions), 0, vmo,
0, length, &reservation);
zx_handle_close(vmo);
if(status != ZX_OK)
{
return nullptr;
}
// zx_vmar_map() returns page-aligned address.
ASSERT(roundUp(reservation, pageSize) == reservation);
mapping = reinterpret_cast<void *>(reservation);
#elif defined(__APPLE__)
int prot = permissionsToMmapProt(permissions);
int flags = MAP_PRIVATE | MAP_ANONYMOUS;
// On macOS 10.14 and higher, executables that are code signed with the
// "runtime" option cannot execute writable memory by default. They can opt
// into this capability by specifying the "com.apple.security.cs.allow-jit"
// code signing entitlement and allocating the region with the MAP_JIT flag.
mapping = mmap(nullptr, length, prot, flags | MAP_JIT, -1, 0);
if(mapping == MAP_FAILED)
{
// Retry without MAP_JIT (for older macOS versions).
mapping = mmap(nullptr, length, prot, flags, -1, 0);
}
if(mapping == MAP_FAILED)
{
mapping = nullptr;
}
#else
mapping = allocate(length, pageSize);
protectMemoryPages(mapping, length, permissions);
#endif
return mapping;
}
void protectMemoryPages(void *memory, size_t bytes, int permissions)
{
if(bytes == 0)
{
return;
}
bytes = roundUp(bytes, memoryPageSize());
#if defined(_WIN32)
unsigned long oldProtection;
BOOL result =
VirtualProtect(memory, bytes, permissionsToProtectMode(permissions),
&oldProtection);
ASSERT(result);
#elif defined(__Fuchsia__)
zx_status_t status = zx_vmar_protect(
zx_vmar_root_self(), permissionsToZxVmOptions(permissions),
reinterpret_cast<zx_vaddr_t>(memory), bytes);
ASSERT(status == ZX_OK);
#else
int result =
mprotect(memory, bytes, permissionsToMmapProt(permissions));
ASSERT(result == 0);
#endif
}
void deallocateMemoryPages(void *memory, size_t bytes)
{
#if defined(_WIN32)
unsigned long oldProtection;
BOOL result;
result = VirtualProtect(memory, bytes, PAGE_READWRITE, &oldProtection);
ASSERT(result);
result = VirtualFree(memory, 0, MEM_RELEASE);
ASSERT(result);
#elif defined(__APPLE__) || (defined(__linux__) && defined(REACTOR_ANONYMOUS_MMAP_NAME))
size_t pageSize = memoryPageSize();
size_t length = (bytes + pageSize - 1) & ~(pageSize - 1);
int result = munmap(memory, length);
ASSERT(result == 0);
#elif defined(__Fuchsia__)
size_t pageSize = memoryPageSize();
size_t length = roundUp(bytes, pageSize);
zx_status_t status = zx_vmar_unmap(
zx_vmar_root_self(), reinterpret_cast<zx_vaddr_t>(memory), length);
ASSERT(status == ZX_OK);
#else
int result = mprotect(memory, bytes, PROT_READ | PROT_WRITE);
ASSERT(result == 0);
deallocate(memory);
#endif
}
} // namespace rr