| // Copyright 2016 The SwiftShader Authors. All Rights Reserved. |
| // |
| // Licensed under the Apache License, Version 2.0 (the "License"); |
| // you may not use this file except in compliance with the License. |
| // You may obtain a copy of the License at |
| // |
| // http://www.apache.org/licenses/LICENSE-2.0 |
| // |
| // Unless required by applicable law or agreed to in writing, software |
| // distributed under the License is distributed on an "AS IS" BASIS, |
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| // See the License for the specific language governing permissions and |
| // limitations under the License. |
| |
| #include "ExecutableMemory.hpp" |
| |
| #include "Debug.hpp" |
| |
| #if defined(_WIN32) |
| # ifndef WIN32_LEAN_AND_MEAN |
| # define WIN32_LEAN_AND_MEAN |
| # endif |
| # include <Windows.h> |
| # include <intrin.h> |
| #elif defined(__Fuchsia__) |
| # include <unistd.h> |
| # include <zircon/process.h> |
| # include <zircon/syscalls.h> |
| #else |
| # include <errno.h> |
| # include <sys/mman.h> |
| # include <stdlib.h> |
| # include <unistd.h> |
| #endif |
| |
| #if defined(__ANDROID__) && !defined(ANDROID_HOST_BUILD) && !defined(ANDROID_NDK_BUILD) |
| # include <sys/prctl.h> |
| #endif |
| |
| #include <memory.h> |
| |
| #undef allocate |
| #undef deallocate |
| |
| #if(defined(__i386__) || defined(_M_IX86) || defined(__x86_64__) || defined(_M_X64)) && !defined(__x86__) |
| # define __x86__ |
| #endif |
| |
| #define STRINGIFY(x) #x |
| #define MACRO_STRINGIFY(x) STRINGIFY(x) |
| |
| // A Clang extension to determine compiler features. |
| // We use it to detect Sanitizer builds (e.g. -fsanitize=memory). |
| #ifndef __has_feature |
| # define __has_feature(x) 0 |
| #endif |
| |
| namespace rr { |
| namespace { |
| |
| struct Allocation |
| { |
| // size_t bytes; |
| unsigned char *block; |
| }; |
| |
| void *allocateRaw(size_t bytes, size_t alignment) |
| { |
| ASSERT((alignment & (alignment - 1)) == 0); // Power of 2 alignment. |
| |
| #if defined(__linux__) && defined(REACTOR_ANONYMOUS_MMAP_NAME) |
| if(alignment < sizeof(void *)) |
| { |
| return malloc(bytes); |
| } |
| else |
| { |
| void *allocation; |
| int result = posix_memalign(&allocation, alignment, bytes); |
| if(result != 0) |
| { |
| errno = result; |
| allocation = nullptr; |
| } |
| return allocation; |
| } |
| #else |
| unsigned char *block = new unsigned char[bytes + sizeof(Allocation) + alignment]; |
| unsigned char *aligned = nullptr; |
| |
| if(block) |
| { |
| aligned = (unsigned char *)((uintptr_t)(block + sizeof(Allocation) + alignment - 1) & -(intptr_t)alignment); |
| Allocation *allocation = (Allocation *)(aligned - sizeof(Allocation)); |
| |
| // allocation->bytes = bytes; |
| allocation->block = block; |
| } |
| |
| return aligned; |
| #endif |
| } |
| |
| #if defined(_WIN32) |
| DWORD permissionsToProtectMode(int permissions) |
| { |
| switch(permissions) |
| { |
| case PERMISSION_READ: |
| return PAGE_READONLY; |
| case PERMISSION_EXECUTE: |
| return PAGE_EXECUTE; |
| case PERMISSION_READ | PERMISSION_WRITE: |
| return PAGE_READWRITE; |
| case PERMISSION_READ | PERMISSION_EXECUTE: |
| return PAGE_EXECUTE_READ; |
| case PERMISSION_READ | PERMISSION_WRITE | PERMISSION_EXECUTE: |
| return PAGE_EXECUTE_READWRITE; |
| } |
| return PAGE_NOACCESS; |
| } |
| #endif |
| |
| #if !defined(_WIN32) && !defined(__Fuchsia__) |
| int permissionsToMmapProt(int permissions) |
| { |
| int result = 0; |
| if(permissions & PERMISSION_READ) |
| { |
| result |= PROT_READ; |
| } |
| if(permissions & PERMISSION_WRITE) |
| { |
| result |= PROT_WRITE; |
| } |
| if(permissions & PERMISSION_EXECUTE) |
| { |
| result |= PROT_EXEC; |
| } |
| return result; |
| } |
| #endif // !defined(_WIN32) && !defined(__Fuchsia__) |
| |
| #if defined(__linux__) && defined(REACTOR_ANONYMOUS_MMAP_NAME) |
| # if !defined(__ANDROID__) || defined(ANDROID_HOST_BUILD) || defined(ANDROID_NDK_BUILD) |
| // Create a file descriptor for anonymous memory with the given |
| // name. Returns -1 on failure. |
| // TODO: remove once libc wrapper exists. |
| static int memfd_create(const char *name, unsigned int flags) |
| { |
| # if __aarch64__ |
| # define __NR_memfd_create 279 |
| # elif __arm__ |
| # define __NR_memfd_create 279 |
| # elif __powerpc64__ |
| # define __NR_memfd_create 360 |
| # elif __i386__ |
| # define __NR_memfd_create 356 |
| # elif __x86_64__ |
| # define __NR_memfd_create 319 |
| # endif /* __NR_memfd_create__ */ |
| # ifdef __NR_memfd_create |
| // In the event of no system call this returns -1 with errno set |
| // as ENOSYS. |
| return syscall(__NR_memfd_create, name, flags); |
| # else |
| return -1; |
| # endif |
| } |
| |
| // Returns a file descriptor for use with an anonymous mmap, if |
| // memfd_create fails, -1 is returned. Note, the mappings should be |
| // MAP_PRIVATE so that underlying pages aren't shared. |
| int anonymousFd() |
| { |
| static int fd = memfd_create(MACRO_STRINGIFY(REACTOR_ANONYMOUS_MMAP_NAME), 0); |
| return fd; |
| } |
| # else // __ANDROID__ && !ANDROID_HOST_BUILD && !ANDROID_NDK_BUILD |
| int anonymousFd() |
| { |
| return -1; |
| } |
| # endif // __ANDROID__ && !ANDROID_HOST_BUILD && !ANDROID_NDK_BUILD |
| |
| // Ensure there is enough space in the "anonymous" fd for length. |
| void ensureAnonFileSize(int anonFd, size_t length) |
| { |
| static size_t fileSize = 0; |
| if(length > fileSize) |
| { |
| [[maybe_unused]] int result = ftruncate(anonFd, length); |
| ASSERT(result == 0); |
| fileSize = length; |
| } |
| } |
| #endif // defined(__linux__) && defined(REACTOR_ANONYMOUS_MMAP_NAME) |
| |
| #if defined(__Fuchsia__) |
| zx_vm_option_t permissionsToZxVmOptions(int permissions) |
| { |
| zx_vm_option_t result = 0; |
| if(permissions & PERMISSION_READ) |
| { |
| result |= ZX_VM_PERM_READ; |
| } |
| if(permissions & PERMISSION_WRITE) |
| { |
| result |= ZX_VM_PERM_WRITE; |
| } |
| if(permissions & PERMISSION_EXECUTE) |
| { |
| result |= ZX_VM_PERM_EXECUTE; |
| } |
| return result; |
| } |
| #endif // defined(__Fuchsia__) |
| |
| } // anonymous namespace |
| |
| size_t memoryPageSize() |
| { |
| static int pageSize = [] { |
| #if defined(_WIN32) |
| SYSTEM_INFO systemInfo; |
| GetSystemInfo(&systemInfo); |
| return systemInfo.dwPageSize; |
| #else |
| return sysconf(_SC_PAGESIZE); |
| #endif |
| }(); |
| |
| return pageSize; |
| } |
| |
| void *allocate(size_t bytes, size_t alignment) |
| { |
| void *memory = allocateRaw(bytes, alignment); |
| |
| // Zero-initialize the memory, for security reasons. |
| // MemorySanitizer builds skip this so that we can detect when we |
| // inadvertently rely on this, which would indicate a bug. |
| if(memory && !__has_feature(memory_sanitizer)) |
| { |
| memset(memory, 0, bytes); |
| } |
| |
| return memory; |
| } |
| |
| void deallocate(void *memory) |
| { |
| #if defined(__linux__) && defined(REACTOR_ANONYMOUS_MMAP_NAME) |
| free(memory); |
| #else |
| if(memory) |
| { |
| unsigned char *aligned = (unsigned char *)memory; |
| Allocation *allocation = (Allocation *)(aligned - sizeof(Allocation)); |
| |
| delete[] allocation->block; |
| } |
| #endif |
| } |
| |
| // Rounds |x| up to a multiple of |m|, where |m| is a power of 2. |
| inline uintptr_t roundUp(uintptr_t x, uintptr_t m) |
| { |
| ASSERT(m > 0 && (m & (m - 1)) == 0); // |m| must be a power of 2. |
| return (x + m - 1) & ~(m - 1); |
| } |
| |
| void *allocateMemoryPages(size_t bytes, int permissions, bool need_exec) |
| { |
| size_t pageSize = memoryPageSize(); |
| size_t length = roundUp(bytes, pageSize); |
| void *mapping = nullptr; |
| |
| #if defined(__linux__) && defined(REACTOR_ANONYMOUS_MMAP_NAME) |
| int flags = MAP_PRIVATE; |
| |
| // Try to name the memory region for the executable code, |
| // to aid profilers. |
| int anonFd = anonymousFd(); |
| if(anonFd == -1) |
| { |
| flags |= MAP_ANONYMOUS; |
| } |
| else |
| { |
| ensureAnonFileSize(anonFd, length); |
| } |
| |
| mapping = mmap( |
| nullptr, length, permissionsToMmapProt(permissions), flags, anonFd, 0); |
| |
| if(mapping == MAP_FAILED) |
| { |
| mapping = nullptr; |
| } |
| # if defined(__ANDROID__) && !defined(ANDROID_HOST_BUILD) && !defined(ANDROID_NDK_BUILD) |
| else |
| { |
| // On Android, prefer to use a non-standard prctl called |
| // PR_SET_VMA_ANON_NAME to set the name of a private anonymous |
| // mapping, as Android restricts EXECUTE permission on |
| // CoW/shared anonymous mappings with sepolicy neverallows. |
| prctl(PR_SET_VMA, PR_SET_VMA_ANON_NAME, mapping, length, |
| MACRO_STRINGIFY(REACTOR_ANONYMOUS_MMAP_NAME)); |
| } |
| # endif // __ANDROID__ && !ANDROID_HOST_BUILD && !ANDROID_NDK_BUILD |
| #elif defined(__Fuchsia__) |
| zx_handle_t vmo; |
| if(zx_vmo_create(length, 0, &vmo) != ZX_OK) |
| { |
| return nullptr; |
| } |
| if(need_exec && |
| zx_vmo_replace_as_executable(vmo, ZX_HANDLE_INVALID, &vmo) != ZX_OK) |
| { |
| return nullptr; |
| } |
| zx_vaddr_t reservation; |
| zx_status_t status = zx_vmar_map( |
| zx_vmar_root_self(), permissionsToZxVmOptions(permissions), 0, vmo, |
| 0, length, &reservation); |
| zx_handle_close(vmo); |
| if(status != ZX_OK) |
| { |
| return nullptr; |
| } |
| |
| // zx_vmar_map() returns page-aligned address. |
| ASSERT(roundUp(reservation, pageSize) == reservation); |
| |
| mapping = reinterpret_cast<void *>(reservation); |
| #elif defined(__APPLE__) |
| int prot = permissionsToMmapProt(permissions); |
| int flags = MAP_PRIVATE | MAP_ANONYMOUS; |
| // On macOS 10.14 and higher, executables that are code signed with the |
| // "runtime" option cannot execute writable memory by default. They can opt |
| // into this capability by specifying the "com.apple.security.cs.allow-jit" |
| // code signing entitlement and allocating the region with the MAP_JIT flag. |
| mapping = mmap(nullptr, length, prot, flags | MAP_JIT, -1, 0); |
| |
| if(mapping == MAP_FAILED) |
| { |
| // Retry without MAP_JIT (for older macOS versions). |
| mapping = mmap(nullptr, length, prot, flags, -1, 0); |
| } |
| |
| if(mapping == MAP_FAILED) |
| { |
| mapping = nullptr; |
| } |
| #else |
| mapping = allocate(length, pageSize); |
| protectMemoryPages(mapping, length, permissions); |
| #endif |
| |
| return mapping; |
| } |
| |
| void protectMemoryPages(void *memory, size_t bytes, int permissions) |
| { |
| if(bytes == 0) |
| { |
| return; |
| } |
| |
| bytes = roundUp(bytes, memoryPageSize()); |
| |
| #if defined(_WIN32) |
| unsigned long oldProtection; |
| BOOL result = |
| VirtualProtect(memory, bytes, permissionsToProtectMode(permissions), |
| &oldProtection); |
| ASSERT(result); |
| #elif defined(__Fuchsia__) |
| zx_status_t status = zx_vmar_protect( |
| zx_vmar_root_self(), permissionsToZxVmOptions(permissions), |
| reinterpret_cast<zx_vaddr_t>(memory), bytes); |
| ASSERT(status == ZX_OK); |
| #else |
| int result = |
| mprotect(memory, bytes, permissionsToMmapProt(permissions)); |
| ASSERT(result == 0); |
| #endif |
| } |
| |
| void deallocateMemoryPages(void *memory, size_t bytes) |
| { |
| #if defined(_WIN32) |
| unsigned long oldProtection; |
| BOOL result = |
| VirtualProtect(memory, bytes, PAGE_READWRITE, &oldProtection); |
| ASSERT(result); |
| deallocate(memory); |
| #elif defined(__APPLE__) || (defined(__linux__) && defined(REACTOR_ANONYMOUS_MMAP_NAME)) |
| size_t pageSize = memoryPageSize(); |
| size_t length = (bytes + pageSize - 1) & ~(pageSize - 1); |
| int result = munmap(memory, length); |
| ASSERT(result == 0); |
| #elif defined(__Fuchsia__) |
| size_t pageSize = memoryPageSize(); |
| size_t length = roundUp(bytes, pageSize); |
| zx_status_t status = zx_vmar_unmap( |
| zx_vmar_root_self(), reinterpret_cast<zx_vaddr_t>(memory), length); |
| ASSERT(status == ZX_OK); |
| #else |
| int result = mprotect(memory, bytes, PROT_READ | PROT_WRITE); |
| ASSERT(result == 0); |
| deallocate(memory); |
| #endif |
| } |
| |
| } // namespace rr |