Allocate executable memory backed by named mmaps on Linux.
Executable heap memory can confuse profiling tools. Use memfd_create
on Linux, if possible, to create a named anonymous memory region.
Only enabled if LINUX_ENABLE_NAMED_MMAP is defined.
Bug b/73721724
Change-Id: I420711e4f64725ae834ab54264038683e4c445fe
Reviewed-on: https://swiftshader-review.googlesource.com/17208
Reviewed-by: Alexis Hétu <sugoi@google.com>
Reviewed-by: Ian Rogers <irogers@google.com>
Tested-by: Nicolas Capens <nicolascapens@google.com>
diff --git a/src/Common/Memory.cpp b/src/Common/Memory.cpp
index f946d2f..938223e 100644
--- a/src/Common/Memory.cpp
+++ b/src/Common/Memory.cpp
@@ -24,7 +24,9 @@
#include <windows.h>
#include <intrin.h>
#else
+ #include <errno.h>
#include <sys/mman.h>
+ #include <stdlib.h>
#include <unistd.h>
#endif
@@ -39,6 +41,93 @@
namespace sw
{
+namespace
+{
+struct Allocation
+{
+// size_t bytes;
+ unsigned char *block;
+};
+
+void *allocateRaw(size_t bytes, size_t alignment)
+{
+ ASSERT((alignment & (alignment - 1)) == 0); // Power of 2 alignment.
+
+ #if defined(LINUX_ENABLE_NAMED_MMAP)
+ void *allocation;
+ int result = posix_memalign(&allocation, alignment, bytes);
+ if(result != 0)
+ {
+ errno = result;
+ allocation = nullptr;
+ }
+ return allocation;
+ #else
+ unsigned char *block = new unsigned char[bytes + sizeof(Allocation) + alignment];
+ unsigned char *aligned = nullptr;
+
+ if(block)
+ {
+ aligned = (unsigned char*)((uintptr_t)(block + sizeof(Allocation) + alignment - 1) & -(intptr_t)alignment);
+ Allocation *allocation = (Allocation*)(aligned - sizeof(Allocation));
+
+ // allocation->bytes = bytes;
+ allocation->block = block;
+ }
+
+ return aligned;
+ #endif
+}
+
+#if defined(LINUX_ENABLE_NAMED_MMAP)
+// Create a file descriptor for anonymous memory with the given
+// name. Returns -1 on failure.
+// TODO: remove once libc wrapper exists.
+int memfd_create(const char* name, unsigned int flags)
+{
+ #if __aarch64__
+ #define __NR_memfd_create 279
+ #elif __arm__
+ #define __NR_memfd_create 279
+ #elif __powerpc64__
+ #define __NR_memfd_create 360
+ #elif __i386__
+ #define __NR_memfd_create 356
+ #elif __x86_64__
+ #define __NR_memfd_create 319
+ #endif /* __NR_memfd_create__ */
+ #ifdef __NR_memfd_create
+ // In the event of no system call this returns -1 with errno set
+ // as ENOSYS.
+ return syscall(__NR_memfd_create, name, flags);
+ #else
+ return -1;
+ #endif
+}
+
+// Returns a file descriptor for use with an anonymous mmap, if
+// memfd_create fails, -1 is returned. Note, the mappings should be
+// MAP_PRIVATE so that underlying pages aren't shared.
+int anonymousFd()
+{
+ static int fd = memfd_create("SwiftShader JIT", 0);
+ return fd;
+}
+
+// Ensure there is enough space in the "anonymous" fd for length.
+void ensureAnonFileSize(int anonFd, size_t length)
+{
+ static size_t fileSize = 0;
+ if(length > fileSize)
+ {
+ ftruncate(anonFd, length);
+ fileSize = length;
+ }
+}
+#endif // defined(LINUX_ENABLE_NAMED_MMAP)
+
+} // anonymous namespace
+
size_t memoryPageSize()
{
static int pageSize = 0;
@@ -57,29 +146,6 @@
return pageSize;
}
-struct Allocation
-{
-// size_t bytes;
- unsigned char *block;
-};
-
-inline void *allocateRaw(size_t bytes, size_t alignment)
-{
- unsigned char *block = new unsigned char[bytes + sizeof(Allocation) + alignment];
- unsigned char *aligned = nullptr;
-
- if(block)
- {
- aligned = (unsigned char*)((uintptr_t)(block + sizeof(Allocation) + alignment - 1) & -(intptr_t)alignment);
- Allocation *allocation = (Allocation*)(aligned - sizeof(Allocation));
-
- // allocation->bytes = bytes;
- allocation->block = block;
- }
-
- return aligned;
-}
-
void *allocate(size_t bytes, size_t alignment)
{
void *memory = allocateRaw(bytes, alignment);
@@ -94,20 +160,50 @@
void deallocate(void *memory)
{
- if(memory)
- {
- unsigned char *aligned = (unsigned char*)memory;
- Allocation *allocation = (Allocation*)(aligned - sizeof(Allocation));
+ #if defined(LINUX_ENABLE_NAMED_MMAP)
+ free(memory);
+ #else
+ if(memory)
+ {
+ unsigned char *aligned = (unsigned char*)memory;
+ Allocation *allocation = (Allocation*)(aligned - sizeof(Allocation));
- delete[] allocation->block;
- }
+ delete[] allocation->block;
+ }
+ #endif
}
void *allocateExecutable(size_t bytes)
{
size_t pageSize = memoryPageSize();
+ size_t length = (bytes + pageSize - 1) & ~(pageSize - 1);
+ void *mapping;
- return allocate((bytes + pageSize - 1) & ~(pageSize - 1), pageSize);
+ #if defined(LINUX_ENABLE_NAMED_MMAP)
+ // Try to name the memory region for the executable code,
+ // to aid profilers.
+ int anonFd = anonymousFd();
+ if(anonFd == -1)
+ {
+ mapping = mmap(nullptr, length, PROT_READ | PROT_WRITE,
+ MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+ }
+ else
+ {
+ ensureAnonFileSize(anonFd, length);
+ mapping = mmap(nullptr, length, PROT_READ | PROT_WRITE,
+ MAP_PRIVATE, anonFd, 0);
+ }
+
+ if(mapping == MAP_FAILED)
+ {
+ mapping = nullptr;
+ }
+ #else
+ mapping = allocate(length, pageSize);
+ #endif
+
+ return mapping;
}
void markExecutable(void *memory, size_t bytes)
@@ -125,11 +221,15 @@
#if defined(_WIN32)
unsigned long oldProtection;
VirtualProtect(memory, bytes, PAGE_READWRITE, &oldProtection);
+ deallocate(memory);
+ #elif defined(LINUX_ENABLE_NAMED_MMAP)
+ size_t pageSize = memoryPageSize();
+ size_t length = (bytes + pageSize - 1) & ~(pageSize - 1);
+ munmap(memory, length);
#else
mprotect(memory, bytes, PROT_READ | PROT_WRITE);
+ deallocate(memory);
#endif
-
- deallocate(memory);
}
void clear(uint16_t *memory, uint16_t element, size_t count)
