Merge changes I2b7adc3c,I5873dfa8 * changes: Update Marl to ca8408f68 Squashed 'third_party/marl/' changes from 64d123947..ca8408f68
diff --git a/src/Pipeline/ShaderCore.cpp b/src/Pipeline/ShaderCore.cpp index fff67ad..83f8126 100644 --- a/src/Pipeline/ShaderCore.cpp +++ b/src/Pipeline/ShaderCore.cpp
@@ -611,7 +611,8 @@ UInt r11g11b10Pack(const Float4 &value) { - auto halfBits = floatToHalfBits(As<UInt4>(value), true); + // 10 and 11 bit floats are unsigned, so their minimal value is 0 + auto halfBits = floatToHalfBits(As<UInt4>(Max(value, Float4(0.0f))), true); // Truncates instead of rounding. See b/147900455 UInt4 truncBits = halfBits & UInt4(0x7FF00000, 0x7FF00000, 0x7FE00000, 0); return (UInt(truncBits.x) >> 20) | (UInt(truncBits.y) >> 9) | (UInt(truncBits.z) << 1);
diff --git a/src/Reactor/SubzeroReactor.cpp b/src/Reactor/SubzeroReactor.cpp index 9a83795..df72f66 100644 --- a/src/Reactor/SubzeroReactor.cpp +++ b/src/Reactor/SubzeroReactor.cpp
@@ -58,37 +58,6 @@ // These functions only accept and return Subzero (Ice) types, and do not access any globals. namespace { namespace sz { -void replaceEntryNode(Ice::Cfg *function, Ice::CfgNode *newEntryNode) -{ - ASSERT_MSG(function->getEntryNode() != nullptr, "Function should have an entry node"); - - if(function->getEntryNode() == newEntryNode) - { - return; - } - - // Make this the new entry node - function->setEntryNode(newEntryNode); - - // Reorder nodes so that new entry block comes first. This is required - // by Cfg::renumberInstructions, which expects the first node in the list - // to be the entry node. - { - auto nodes = function->getNodes(); - - // TODO(amaiorano): Fast path if newEntryNode is last? Can avoid linear search. - - auto iter = std::find(nodes.begin(), nodes.end(), newEntryNode); - ASSERT_MSG(iter != nodes.end(), "New node should be in the function's node list"); - - nodes.erase(iter); - nodes.insert(nodes.begin(), newEntryNode); - - // swapNodes replaces its nodes with the input one, and renumbers them, - // so our new entry node will be 0, and the previous will be 1. - function->swapNodes(nodes); - } -} Ice::Cfg *createFunction(Ice::GlobalContext *context, Ice::Type returnType, const std::vector<Ice::Type> ¶mTypes) { @@ -194,6 +163,8 @@ template<typename Return, typename... CArgs, typename... RArgs> Ice::Variable *Call(Ice::Cfg *function, Ice::CfgNode *basicBlock, Return(fptr)(CArgs...), RArgs &&... args) { + static_assert(sizeof...(CArgs) == sizeof...(RArgs), "Expected number of args don't match"); + Ice::Type retTy = T(rr::CToReactorT<Return>::getType()); std::vector<Ice::Operand *> iceArgs{ std::forward<RArgs>(args)... }; return Call(function, basicBlock, retTy, reinterpret_cast<void const *>(fptr), iceArgs, false); @@ -256,6 +227,8 @@ Ice::GlobalContext *context = nullptr; Ice::Cfg *function = nullptr; +Ice::CfgNode *entryBlock = nullptr; +Ice::CfgNode *basicBlockTop = nullptr; Ice::CfgNode *basicBlock = nullptr; Ice::CfgLocalAllocatorScope *allocator = nullptr; rr::ELFMemoryStreamer *routine = nullptr; @@ -489,12 +462,17 @@ return Ice::typeWidthInBytes(T(type)); } -static void createRetVoidIfNoRet() +static void finalizeFunction() { + // Create a return if none was added if(::basicBlock->getInsts().empty() || ::basicBlock->getInsts().back().getKind() != Ice::Inst::Ret) { Nucleus::createRetVoid(); } + + // Connect the entry block to the top of the initial basic block + auto br = Ice::InstBr::create(::function, ::basicBlockTop); + ::entryBlock->appendInst(br); } using ElfHeader = std::conditional<sizeof(void *) == 8, Elf64_Ehdr, Elf32_Ehdr>::type; @@ -656,13 +634,23 @@ return symbolValue; } -void *loadImage(uint8_t *const elfImage, size_t &codeSize, const char *functionName = nullptr) +struct EntryPoint { + const void *entry; + size_t codeSize = 0; +}; + +std::vector<EntryPoint> loadImage(uint8_t *const elfImage, const std::vector<const char *> &functionNames) +{ + ASSERT(functionNames.size() > 0); + std::vector<EntryPoint> entryPoints(functionNames.size()); + ElfHeader *elfHeader = (ElfHeader *)elfImage; + // TODO: assert? if(!elfHeader->checkMagic()) { - return nullptr; + return {}; } // Expect ELF bitness to match platform @@ -682,7 +670,6 @@ #endif SectionHeader *sectionHeader = (SectionHeader *)(elfImage + elfHeader->e_shoff); - void *entry = nullptr; for(int i = 0; i < elfHeader->e_shnum; i++) { @@ -690,17 +677,25 @@ { if(sectionHeader[i].sh_flags & SHF_EXECINSTR) { - auto getCurrSectionName = [&]() { + auto findSectionNameEntryIndex = [&]() -> size_t { auto sectionNameOffset = sectionHeader[elfHeader->e_shstrndx].sh_offset + sectionHeader[i].sh_name; - return reinterpret_cast<const char *>(elfImage + sectionNameOffset); - }; - if(functionName && strstr(getCurrSectionName(), functionName) == nullptr) - { - continue; - } + const char *sectionName = reinterpret_cast<const char *>(elfImage + sectionNameOffset); - entry = elfImage + sectionHeader[i].sh_offset; - codeSize = sectionHeader[i].sh_size; + for(size_t j = 0; j < functionNames.size(); ++j) + { + if(strstr(sectionName, functionNames[j]) != nullptr) + { + return j; + } + } + + UNREACHABLE("Failed to find executable section that matches input function names"); + return static_cast<size_t>(-1); + }; + + size_t index = findSectionNameEntryIndex(); + entryPoints[index].entry = elfImage + sectionHeader[i].sh_offset; + entryPoints[index].codeSize = sectionHeader[i].sh_size; } } else if(sectionHeader[i].sh_type == SHT_REL) @@ -725,7 +720,7 @@ } } - return entry; + return entryPoints; } template<typename T> @@ -796,18 +791,20 @@ void seek(uint64_t Off) override { position = Off; } - const void *getEntryByName(const char *name) + std::vector<EntryPoint> loadImageAndGetEntryPoints(const std::vector<const char *> &functionNames) { - size_t codeSize = 0; - const void *entry = loadImage(&buffer[0], codeSize, name); + auto entryPoints = loadImage(&buffer[0], functionNames); #if defined(_WIN32) FlushInstructionCache(GetCurrentProcess(), NULL, 0); #else - __builtin___clear_cache((char *)entry, (char *)entry + codeSize); + for(auto &entryPoint : entryPoints) + { + __builtin___clear_cache((char *)entryPoint.entry, (char *)entryPoint.entry + entryPoint.codeSize); + } #endif - return entry; + return entryPoints; } void finalize() @@ -926,7 +923,9 @@ delete ::out; ::out = nullptr; + ::entryBlock = nullptr; ::basicBlock = nullptr; + ::basicBlockTop = nullptr; ::codegenMutex.unlock(); } @@ -1026,10 +1025,11 @@ objectWriter->writeNonUserSections(); // Done compiling functions, get entry pointers to each of them - for(size_t i = 0; i < Count; ++i) + auto entryPoints = ::routine->loadImageAndGetEntryPoints({ names, names + Count }); + ASSERT(entryPoints.size() == Count); + for(size_t i = 0; i < entryPoints.size(); ++i) { - const void *entry = ::routine->getEntryByName(names[i]); - ::routine->setEntry(i, entry); + ::routine->setEntry(i, entryPoints[i].entry); } ::routine->finalize(); @@ -1042,7 +1042,7 @@ std::shared_ptr<Routine> Nucleus::acquireRoutine(const char *name, const Config::Edit &cfgEdit /* = Config::Edit::None */) { - createRetVoidIfNoRet(); + finalizeFunction(); return rr::acquireRoutine({ ::function }, { name }, cfgEdit); } @@ -1083,7 +1083,9 @@ { ASSERT(::function == nullptr); ASSERT(::allocator == nullptr); + ASSERT(::entryBlock == nullptr); ASSERT(::basicBlock == nullptr); + ASSERT(::basicBlockTop == nullptr); ::function = sz::createFunction(::context, T(returnType), T(paramTypes)); @@ -1093,7 +1095,9 @@ // TODO: Get rid of this as a global, and create scoped allocs in every Nucleus function instead. ::allocator = new Ice::CfgLocalAllocatorScope(::function); - ::basicBlock = ::function->getEntryNode(); + ::entryBlock = ::function->getEntryNode(); + ::basicBlock = ::function->makeNode(); + ::basicBlockTop = ::basicBlock; } Value *Nucleus::getArgument(unsigned int index) @@ -4595,29 +4599,13 @@ // ... <REACTOR CODE> ... // - // Save original entry block and current block, and create a new entry block and make it current. - // This new block will be used to inject code above the begin routine's existing code. We make - // this block branch to the original entry block as the last instruction. - auto origEntryBB = ::function->getEntryNode(); - auto origCurrBB = ::basicBlock; - auto newBB = ::function->makeNode(); - sz::replaceEntryNode(::function, newBB); - ::basicBlock = newBB; - // this->handle = coro::getHandleParam(); - this->handle = sz::Call(::function, ::basicBlock, coro::getHandleParam); + this->handle = sz::Call(::function, ::entryBlock, coro::getHandleParam); // YieldType promise; // coro::setPromisePtr(handle, &promise); // For await this->promise = sz::allocateStackVariable(::function, T(::coroYieldType)); - sz::Call(::function, ::basicBlock, coro::setPromisePtr, this->handle, this->promise); - - // Branch to original entry block - auto br = Ice::InstBr::create(::function, origEntryBB); - ::basicBlock->appendInst(br); - - // Restore current block for future instructions - ::basicBlock = origCurrBB; + sz::Call(::function, ::entryBlock, coro::setPromisePtr, this->handle, this->promise); } // Adds instructions for Yield() calls at the current location of the main coroutine function. @@ -4719,7 +4707,7 @@ // <resumeBlock> // } Ice::CfgNode *bb = awaitFunc->getEntryNode(); - Ice::Variable *done = sz::Call(awaitFunc, bb, coro::isDone); + Ice::Variable *done = sz::Call(awaitFunc, bb, coro::isDone, handle); auto br = Ice::InstBr::create(awaitFunc, done, doneBlock, resumeBlock); bb->appendInst(br); @@ -4831,7 +4819,7 @@ // Finish generating coroutine functions { Ice::CfgLocalAllocatorScope scopedAlloc{ ::function }; - createRetVoidIfNoRet(); + finalizeFunction(); } auto awaitFunc = ::coroGen->generateAwaitFunction(); @@ -4851,7 +4839,7 @@ { { Ice::CfgLocalAllocatorScope scopedAlloc{ ::function }; - createRetVoidIfNoRet(); + finalizeFunction(); } ::coroYieldType = nullptr;
diff --git a/src/Vulkan/BUILD.gn b/src/Vulkan/BUILD.gn index 9ea2d28..19b7e79 100644 --- a/src/Vulkan/BUILD.gn +++ b/src/Vulkan/BUILD.gn
@@ -101,8 +101,6 @@ ] } else if (is_fuchsia) { sources += [ "VkSemaphoreExternalFuchsia.hpp" ] - } else { - sources += [ "VkSemaphoreExternalNone.hpp" ] } }
diff --git a/src/Vulkan/VkSemaphore.cpp b/src/Vulkan/VkSemaphore.cpp index 72bee8b..8daef84 100644 --- a/src/Vulkan/VkSemaphore.cpp +++ b/src/Vulkan/VkSemaphore.cpp
@@ -17,18 +17,6 @@ #include "VkConfig.h" #include "VkStringify.hpp" -#if SWIFTSHADER_EXTERNAL_SEMAPHORE_OPAQUE_FD -# if defined(__linux__) || defined(__ANDROID__) -# include "VkSemaphoreExternalLinux.hpp" -# else -# error "Missing VK_KHR_external_semaphore_fd implementation for this platform!" -# endif -#elif VK_USE_PLATFORM_FUCHSIA -# include "VkSemaphoreExternalFuchsia.hpp" -#else -# include "VkSemaphoreExternalNone.hpp" -#endif - #include "marl/blockingcall.h" #include "marl/conditionvariable.h" @@ -38,11 +26,76 @@ namespace vk { +// This is a base abstract class for all external semaphore implementations +// used in this source file. +class Semaphore::External +{ +public: + virtual ~External() = default; + + // Initialize new instance with a given initial state. + virtual VkResult init(bool initialState) = 0; + + virtual bool tryWait() = 0; + virtual void wait() = 0; + virtual void signal() = 0; + + // For VK_KHR_external_semaphore_fd + virtual VkResult importOpaqueFd(int fd) + { + return VK_ERROR_INVALID_EXTERNAL_HANDLE; + } + + virtual VkResult exportOpaqueFd(int *pFd) + { + return VK_ERROR_INVALID_EXTERNAL_HANDLE; + } + +#if VK_USE_PLATFORM_FUCHSIA + // For VK_FUCHSIA_external_semaphore + virtual VkResult importHandle(zx_handle_t handle) + { + return VK_ERROR_INVALID_EXTERNAL_HANDLE; + } + virtual VkResult exportHandle(zx_handle_t *pHandle) + { + return VK_ERROR_INVALID_EXTERNAL_HANDLE; + } +#endif + // Pointer to previous temporary external instanc,e used for |tempExternal| only. + External *previous = nullptr; +}; + +} // namespace vk + +#if SWIFTSHADER_EXTERNAL_SEMAPHORE_OPAQUE_FD +# if defined(__linux__) || defined(__ANDROID__) +# include "VkSemaphoreExternalLinux.hpp" +# else +# error "Missing VK_KHR_external_semaphore_fd implementation for this platform!" +# endif +#elif VK_USE_PLATFORM_FUCHSIA +# include "VkSemaphoreExternalFuchsia.hpp" +#endif + +namespace vk { + +// The bitmask of all external semaphore handle types supported by this source file. +static const VkExternalSemaphoreHandleTypeFlags kSupportedTypes = +#if SWIFTSHADER_EXTERNAL_SEMAPHORE_OPAQUE_FD + VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_FD_BIT | +#endif +#if VK_USE_PLATFORM_FUCHSIA + VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_TEMP_ZIRCON_EVENT_BIT_FUCHSIA | +#endif + 0; + namespace { struct SemaphoreCreateInfo { bool exportSemaphore = false; + VkExternalSemaphoreHandleTypeFlags exportHandleTypes = 0; // Create a new instance. The external instance will be allocated only // the pCreateInfo->pNext chain indicates it needs to be exported. @@ -57,9 +110,12 @@ { const auto *exportInfo = reinterpret_cast<const VkExportSemaphoreCreateInfo *>(nextInfo); exportSemaphore = true; - if(exportInfo->handleTypes != Semaphore::External::kExternalSemaphoreHandleType) + exportHandleTypes = exportInfo->handleTypes; + if((exportHandleTypes & ~kSupportedTypes) != 0) { - UNSUPPORTED("exportInfo->handleTypes %d", int(exportInfo->handleTypes)); + UNSUPPORTED("exportInfo->handleTypes 0x%X (supports 0x%X)", + int(exportHandleTypes), + int(kSupportedTypes)); } } break; @@ -74,27 +130,30 @@ void Semaphore::wait() { - if(external) + std::unique_lock<std::mutex> lock(mutex); + External *ext = tempExternal ? tempExternal : external; + if(ext) { - if(!external->tryWait()) + if(!ext->tryWait()) { - // Dispatch the external wait to a background thread. + // Dispatch the ext wait to a background thread. // Even if this creates a new thread on each // call, it is assumed that this is negligible // compared with the actual semaphore wait() // operation. - marl::blocking_call([this]() { - external->wait(); + marl::blocking_call([ext, &lock]() { + lock.unlock(); + ext->wait(); + lock.lock(); }); } - // If the import was temporary, reset the semaphore to its - // permanent state by getting rid of |external|. + // If the import was temporary, reset the semaphore to its previous state. // See "6.4.5. Importing Semaphore Payloads" in Vulkan 1.1 spec. - if(temporaryImport) + if(ext == tempExternal) { - deallocateExternal(); - temporaryImport = false; + tempExternal = ext->previous; + deallocateExternal(ext); } } else @@ -105,11 +164,13 @@ void Semaphore::signal() { - if(external) + std::unique_lock<std::mutex> lock(mutex); + External *ext = tempExternal ? tempExternal : external; + if(ext) { // Assumes that signalling an external semaphore is non-blocking, // so it can be performed directly either from a fiber or thread. - external->signal(); + ext->signal(); } else { @@ -121,16 +182,22 @@ : allocator(pAllocator) { SemaphoreCreateInfo info(pCreateInfo); - if(info.exportSemaphore) - { - allocateExternal(); - external->init(); - } + exportableHandleTypes = info.exportHandleTypes; } void Semaphore::destroy(const VkAllocationCallbacks *pAllocator) { - deallocateExternal(); + while(tempExternal) + { + External *ext = tempExternal; + tempExternal = ext->previous; + deallocateExternal(ext); + } + if(external) + { + deallocateExternal(external); + external = nullptr; + } } size_t Semaphore::ComputeRequiredAllocationSize(const VkSemaphoreCreateInfo *pCreateInfo) @@ -139,78 +206,140 @@ return 0; } -void Semaphore::allocateExternal() +template<class EXTERNAL> +Semaphore::External *Semaphore::allocateExternal() { - ASSERT(external == nullptr); - external = reinterpret_cast<Semaphore::External *>( - vk::allocate(sizeof(Semaphore::External), vk::REQUIRED_MEMORY_ALIGNMENT, allocator)); - new(external) Semaphore::External(); + auto *ext = reinterpret_cast<Semaphore::External *>( + vk::allocate(sizeof(EXTERNAL), alignof(EXTERNAL), allocator)); + new(ext) EXTERNAL(); + return ext; } -void Semaphore::deallocateExternal() +void Semaphore::deallocateExternal(Semaphore::External *ext) { - if(external) + ext->~External(); + vk::deallocate(ext, allocator); +} + +template<typename ALLOC_FUNC, typename IMPORT_FUNC> +VkResult Semaphore::importPayload(bool temporaryImport, + ALLOC_FUNC alloc_func, + IMPORT_FUNC import_func) +{ + std::unique_lock<std::mutex> lock(mutex); + + // Create new External instance if needed. + External *ext = external; + if(temporaryImport || !ext) { - vk::deallocate(external, allocator); - external = nullptr; + ext = alloc_func(); } + VkResult result = import_func(ext); + if(result != VK_SUCCESS) + { + if(temporaryImport || !external) + { + deallocateExternal(ext); + } + return result; + } + + if(temporaryImport) + { + ext->previous = tempExternal; + tempExternal = ext; + } + else if(!external) + { + external = ext; + } + return VK_SUCCESS; +} + +template<typename ALLOC_FUNC, typename EXPORT_FUNC> +VkResult Semaphore::exportPayload(ALLOC_FUNC alloc_func, EXPORT_FUNC export_func) +{ + std::unique_lock<std::mutex> lock(mutex); + // Sanity check, do not try to export a semaphore that has a temporary import. + if(tempExternal != nullptr) + { + TRACE("Cannot export semaphore with a temporary import!"); + return VK_ERROR_INVALID_EXTERNAL_HANDLE; + } + // Allocate |external| if it doesn't exist yet. + if(!external) + { + External *ext = alloc_func(); + VkResult result = ext->init(internal.isSignalled()); + if(result != VK_SUCCESS) + { + deallocateExternal(ext); + return result; + } + external = ext; + } + return export_func(external); } #if SWIFTSHADER_EXTERNAL_SEMAPHORE_OPAQUE_FD -VkResult Semaphore::importFd(int fd, bool tempImport) +VkResult Semaphore::importFd(int fd, bool temporaryImport) { - std::unique_lock<std::mutex> lock(mutex); - if(!external) - { - allocateExternal(); - } - VkResult result = external->importFd(fd); - if(result != VK_SUCCESS) - { - deallocateExternal(); - } - else - { - temporaryImport = tempImport; - } - return result; + return importPayload( + temporaryImport, + [this]() { + return allocateExternal<OpaqueFdExternalSemaphore>(); + }, + [fd](External *ext) { + return ext->importOpaqueFd(fd); + }); } VkResult Semaphore::exportFd(int *pFd) { - std::unique_lock<std::mutex> lock(mutex); - if(!external) + if((exportableHandleTypes & VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_FD_BIT) == 0) { - TRACE("Cannot export non-external semaphore"); + TRACE("Cannot export semaphore as opaque FD (exportableHandleType = 0x%X, want 0x%X)", + exportableHandleTypes, + VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_FD_BIT); + return VK_ERROR_INVALID_EXTERNAL_HANDLE; } - return external->exportFd(pFd); + + return exportPayload([this]() { return allocateExternal<OpaqueFdExternalSemaphore>(); }, + [pFd](External *ext) { + return ext->exportOpaqueFd(pFd); + }); } #endif // SWIFTSHADER_EXTERNAL_SEMAPHORE_OPAQUE_FD #if VK_USE_PLATFORM_FUCHSIA -VkResult Semaphore::importHandle(zx_handle_t handle, bool tempImport) +VkResult Semaphore::importHandle(zx_handle_t handle, bool temporaryImport) { - std::unique_lock<std::mutex> lock(mutex); - if(!external) - { - allocateExternal(); - } - // NOTE: Imports are just moving a handle so cannot fail. - external->importHandle(handle); - temporaryImport = tempImport; - return VK_SUCCESS; + return importPayload( + temporaryImport, + [this]() { + return allocateExternal<ZirconEventExternalSemaphore>(); + }, + [handle](External *ext) { + return ext->importHandle(handle); + }); } VkResult Semaphore::exportHandle(zx_handle_t *pHandle) { - std::unique_lock<std::mutex> lock(mutex); - if(!external) + if((exportableHandleTypes & VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_TEMP_ZIRCON_EVENT_BIT_FUCHSIA) == 0) { - TRACE("Cannot export non-external semaphore"); + TRACE("Cannot export semaphore as Zircon handle (exportableHandleType = 0x%X, want 0x%X)", + exportableHandleTypes, + VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_TEMP_ZIRCON_EVENT_BIT_FUCHSIA); + return VK_ERROR_INVALID_EXTERNAL_HANDLE; } - return external->exportHandle(pHandle); + + return exportPayload([this]() { return allocateExternal<ZirconEventExternalSemaphore>(); }, + [pHandle](External *ext) { + return ext->exportHandle(pHandle); + }); } #endif // VK_USE_PLATFORM_FUCHSIA
diff --git a/src/Vulkan/VkSemaphore.hpp b/src/Vulkan/VkSemaphore.hpp index 54004d8..b67d4d9 100644 --- a/src/Vulkan/VkSemaphore.hpp +++ b/src/Vulkan/VkSemaphore.hpp
@@ -58,14 +58,99 @@ class External; private: - void allocateExternal(); - void deallocateExternal(); + // Small technical note on how semaphores are imported/exported with Vulkan: + // + // - A Vulkan Semaphore objects has a "payload", corresponding to a + // simple atomic boolean flag. + // + // - A Vulkan Semaphore object can be "exported": this creates a + // platform-specific handle / descriptor (which can be passed to other + // processes), and is linked in some way to the original semaphore's + // payload. + // + // - Similarly, said handle / descriptor can be "imported" into a Vulkan + // Semaphore object. By default, that semaphore loses its payload, and + // instead uses the one referenced / shared through the descriptor. + // + // Hence if semaphore A exports its payload through a descriptor that + // is later imported into semaphore B, then both A and B will use/share + // the same payload (i.e. signal flag), making cross-process + // synchronization possible. + // + // - There are also "temporary imports", where the target semaphore's + // payload is not lost, but is simply hidden/stashed. But the next wait() + // operation on the same semaphore should remove the temporary import, + // and restore the previous payload. + // + // - There are many handle / descriptor types, which are listed through + // the VkExternalSemaphoreHandleTypeFlagBits. A given Vulkan + // implementation might support onle one or several at the same time + // (e.g. on Linux or Android, it could support both OPAQUE_FD_BIT and + // SYNC_FD_BIT, while on Windows, it would be OPAQUE_WIN32_BIT + + // OPAQUE_WIN32_KMT_BIT + D3D12_FENCE_BIT). + // + // - To be able to export a semaphore, VkCreateSemaphore() must be called + // with a VkSemaphoreCreateInfo that lists the types of all possible + // platform-specific handles the semaphore could be exported to + // (e.g. on Linux, it is possible to specify that a semaphore might be + // exported as an opaque FD, or as a Linux Sync FD). + // + // However, which exact type is however only determined later by the + // export operation itself (e.g. vkGetSemaphoreFdKHR() could be called to export + // either a VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_FD_BIT or a + // VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_SYNC_FD_BIT). + // + // Once a semaphore has been exported as one type, it is not possible + // to export the same payload with a different type (though the spec + // doesn't seem to be explicit about this, it's simply impossible in + // general). + // + // This leads to the following design: + // + // - |internal| is a simple marl::Event that represents the semaphore's + // payload when it is not exported, or imported non-temporarily. + // + // - |external| points to an external semaphore payload. It is created + // on demand if the semaphore is exported or imported non-temporarily. + // Note that once |external| is created, |internal| is ignored. + // + // - |tempExternal| points to a linked-list of temporary external + // semaphore payloads. The list head corresponds to the most recent + // temporary import. + // + + // Internal template to allocate a new External implementation. + template<class EXTERNAL> + External *allocateExternal(); + + void deallocateExternal(External *ext); + + // Used internally to import an external payload. + // |temporaryImport| is true iff the import is temporary. + // |alloc_func| is callable that allocates a new External instance of the + // appropriate type. + // |import_func| is callable that takes a single parameter, which + // corresponds to the external handle/descriptor, and returns a VkResult + // values. + template<typename ALLOC_FUNC, typename IMPORT_FUNC> + VkResult importPayload(bool temporaryImport, + ALLOC_FUNC alloc_func, + IMPORT_FUNC import_func); + + // Used internally to export a given payload. + // |alloc_func| is a callable that allocates a new External instance of + // the appropriate type. + // |export_func| is a callable that takes a pointer to an External instance, + // and a pointer to a handle/descriptor, and returns a VkResult. + template<typename ALLOC_FUNC, typename EXPORT_FUNC> + VkResult exportPayload(ALLOC_FUNC alloc_func, EXPORT_FUNC export_func); const VkAllocationCallbacks *allocator = nullptr; - marl::Event internal; + VkExternalSemaphoreHandleTypeFlags exportableHandleTypes = (VkExternalSemaphoreHandleTypeFlags)0; std::mutex mutex; + marl::Event internal; External *external = nullptr; - bool temporaryImport = false; + External *tempExternal = nullptr; }; static inline Semaphore *Cast(VkSemaphore object)
diff --git a/src/Vulkan/VkSemaphoreExternalFuchsia.hpp b/src/Vulkan/VkSemaphoreExternalFuchsia.hpp index a45f0b8..90f68d4 100644 --- a/src/Vulkan/VkSemaphoreExternalFuchsia.hpp +++ b/src/Vulkan/VkSemaphoreExternalFuchsia.hpp
@@ -26,38 +26,44 @@ namespace vk { -class Semaphore::External +class ZirconEventExternalSemaphore : public Semaphore::External { public: - // The type of external semaphore handle types supported by this implementation. - static const VkExternalSemaphoreHandleTypeFlags kExternalSemaphoreHandleType = - VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_TEMP_ZIRCON_EVENT_BIT_FUCHSIA; - - // Default constructor. Note that one should call either init() or - // importFd() before any call to wait() or signal(). - External() = default; - - ~External() + ~ZirconEventExternalSemaphore() { zx_handle_close(handle); } - void init() + VkResult init(bool initialValue) override { zx_status_t status = zx_event_create(0, &handle); if(status != ZX_OK) { - ABORT("zx_event_create() returned %d", status); + TRACE("zx_event_create() returned %d", status); + return VK_ERROR_INITIALIZATION_FAILED; } + if(initialValue) + { + status = zx_object_signal(handle, 0, ZX_EVENT_SIGNALED); + if(status != ZX_OK) + { + TRACE("zx_object_signal() returned %d", status); + zx_handle_close(handle); + handle = ZX_HANDLE_INVALID; + return VK_ERROR_INITIALIZATION_FAILED; + } + } + return VK_SUCCESS; } - void importHandle(zx_handle_t new_handle) + VkResult importHandle(zx_handle_t new_handle) override { zx_handle_close(handle); handle = new_handle; + return VK_SUCCESS; } - VkResult exportHandle(zx_handle_t *pHandle) const + VkResult exportHandle(zx_handle_t *pHandle) override { zx_handle_t new_handle = ZX_HANDLE_INVALID; zx_status_t status = zx_handle_duplicate(handle, ZX_RIGHT_SAME_RIGHTS, &new_handle); @@ -70,7 +76,7 @@ return VK_SUCCESS; } - void wait() + void wait() override { zx_signals_t observed = 0; zx_status_t status = zx_object_wait_one( @@ -91,7 +97,7 @@ } } - bool tryWait() + bool tryWait() override { zx_signals_t observed = 0; zx_status_t status = zx_object_wait_one( @@ -113,7 +119,7 @@ return true; } - void signal() + void signal() override { zx_status_t status = zx_object_signal(handle, 0, ZX_EVENT_SIGNALED); if(status != ZX_OK)
diff --git a/src/Vulkan/VkSemaphoreExternalLinux.hpp b/src/Vulkan/VkSemaphoreExternalLinux.hpp index 36bac26..d4e4cd2 100644 --- a/src/Vulkan/VkSemaphoreExternalLinux.hpp +++ b/src/Vulkan/VkSemaphoreExternalLinux.hpp
@@ -44,7 +44,8 @@ class SharedSemaphore { public: - SharedSemaphore() + SharedSemaphore(bool initialValue) + : signaled(initialValue) { pthread_mutexattr_t mattr; pthread_mutexattr_init(&mattr); @@ -129,20 +130,13 @@ namespace vk { -class Semaphore::External +class OpaqueFdExternalSemaphore : public Semaphore::External { public: - // The type of external semaphore handle types supported by this implementation. - static const VkExternalSemaphoreHandleTypeFlags kExternalSemaphoreHandleType = VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_FD_BIT; - - // Default constructor. Note that one should call either init() or - // importFd() before any call to wait() or signal(). - External() = default; - - ~External() { close(); } + ~OpaqueFdExternalSemaphore() { unmapRegion(); } // Initialize instance by creating a new shared memory region. - void init() + VkResult init(bool initialState) override { // Allocate or import the region's file descriptor. const size_t size = sw::memoryPageSize(); @@ -153,24 +147,30 @@ snprintf(name, sizeof(name), "SwiftShader.Semaphore.%d", ++counter); if(!memfd.allocate(name, size)) { - ABORT("memfd.allocate() returned %s", strerror(errno)); + TRACE("memfd.allocate() returned %s", strerror(errno)); + return VK_ERROR_INITIALIZATION_FAILED; } - mapRegion(size, true); + if(!mapRegion(size, true, initialState)) + return VK_ERROR_INITIALIZATION_FAILED; + + return VK_SUCCESS; } // Import an existing semaphore through its file descriptor. - VkResult importFd(int fd) + VkResult importOpaqueFd(int fd) override { - close(); + unmapRegion(); memfd.importFd(fd); - mapRegion(sw::memoryPageSize(), false); + if(!mapRegion(sw::memoryPageSize(), false, false)) + return VK_ERROR_INITIALIZATION_FAILED; + return VK_SUCCESS; } // Export the current semaphore as a duplicated file descriptor to the same // region. This can be consumed by importFd() running in a different // process. - VkResult exportFd(int *pFd) const + VkResult exportOpaqueFd(int *pFd) override { int fd = memfd.exportFd(); if(fd < 0) @@ -181,24 +181,23 @@ return VK_SUCCESS; } - void wait() + void wait() override { semaphore->wait(); } - bool tryWait() + bool tryWait() override { return semaphore->tryWait(); } - void signal() + void signal() override { semaphore->signal(); } private: - // Unmap the semaphore if needed and close its file descriptor. - void close() + void unmapRegion() { if(semaphore) { @@ -213,23 +212,25 @@ } // Remap the shared region and setup the semaphore or increment its reference count. - void mapRegion(size_t size, bool needInitialization) + bool mapRegion(size_t size, bool needsInitialization, bool initialValue) { // Map the region into memory and point the semaphore to it. void *addr = memfd.mapReadWrite(0, size); if(!addr) { - ABORT("mmap() failed: %s", strerror(errno)); + TRACE("mmap() failed: %s", strerror(errno)); + return false; } semaphore = reinterpret_cast<SharedSemaphore *>(addr); - if(needInitialization) + if(needsInitialization) { - new(semaphore) SharedSemaphore(); + new(semaphore) SharedSemaphore(initialValue); } else { semaphore->addRef(); } + return true; } LinuxMemFd memfd;
diff --git a/src/Vulkan/VkSemaphoreExternalNone.hpp b/src/Vulkan/VkSemaphoreExternalNone.hpp deleted file mode 100644 index 9592e3f..0000000 --- a/src/Vulkan/VkSemaphoreExternalNone.hpp +++ /dev/null
@@ -1,41 +0,0 @@ -// Copyright 2019 The SwiftShader Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef VK_SEMAPHORE_EXTERNAL_NONE_H_ -#define VK_SEMAPHORE_EXTERNAL_NONE_H_ - -namespace vk { - -// Empty external sempahore implementation. -class Semaphore::External -{ -public: - // The type of external semaphore handle types supported by this implementation. - static const VkExternalSemaphoreHandleTypeFlags kExternalSemaphoreHandleType = 0; - - void init() {} - - void wait() {} - - bool tryWait() { return true; } - - void signal() {} - -private: - int dummy; -}; - -} // namespace vk - -#endif // VK_SEMAPHORE_EXTERNAL_NONE_H_
diff --git a/third_party/llvm-subzero/include/llvm/Support/Compiler.h b/third_party/llvm-subzero/include/llvm/Support/Compiler.h index 55148a4..803fd48 100644 --- a/third_party/llvm-subzero/include/llvm/Support/Compiler.h +++ b/third_party/llvm-subzero/include/llvm/Support/Compiler.h
@@ -243,6 +243,15 @@ #define LLVM_FALLTHROUGH #endif +/// LLVM_REQUIRE_CONSTANT_INITIALIZATION - Apply this to globals to ensure that +/// they are constant initialized. +#if __has_cpp_attribute(clang::require_constant_initialization) +#define LLVM_REQUIRE_CONSTANT_INITIALIZATION \ + [[clang::require_constant_initialization]] +#else +#define LLVM_REQUIRE_CONSTANT_INITIALIZATION +#endif + /// LLVM_EXTENSION - Support compilers where we have a keyword to suppress /// pedantic diagnostics. #ifdef __GNUC__
diff --git a/third_party/llvm-subzero/include/llvm/Support/ManagedStatic.h b/third_party/llvm-subzero/include/llvm/Support/ManagedStatic.h index 7ce86ee..e4ebd7c 100644 --- a/third_party/llvm-subzero/include/llvm/Support/ManagedStatic.h +++ b/third_party/llvm-subzero/include/llvm/Support/ManagedStatic.h
@@ -36,18 +36,37 @@ static void call(void *Ptr) { delete[](T *)Ptr; } }; +// If the current compiler is MSVC 2017 or earlier, then we have to work around +// a bug where MSVC emits code to perform dynamic initialization even if the +// class has a constexpr constructor. Instead, fall back to the C++98 strategy +// where there are no constructors or member initializers. We can remove this +// when MSVC 2019 (19.20+) is our minimum supported version. +#if !defined(__clang__) && defined(_MSC_VER) && _MSC_VER < 1920 +#define LLVM_AVOID_CONSTEXPR_CTOR +#endif + /// ManagedStaticBase - Common base class for ManagedStatic instances. class ManagedStaticBase { protected: +#ifndef LLVM_AVOID_CONSTEXPR_CTOR + mutable std::atomic<void *> Ptr{}; + mutable void (*DeleterFn)(void *) = nullptr; + mutable const ManagedStaticBase *Next = nullptr; +#else // This should only be used as a static variable, which guarantees that this // will be zero initialized. mutable std::atomic<void *> Ptr; - mutable void (*DeleterFn)(void*); + mutable void (*DeleterFn)(void *); mutable const ManagedStaticBase *Next; +#endif void RegisterManagedStatic(void *(*creator)(), void (*deleter)(void*)) const; public: +#ifndef LLVM_AVOID_CONSTEXPR_CTOR + constexpr ManagedStaticBase() = default; +#endif + /// isConstructed - Return true if this object has not been created yet. bool isConstructed() const { return Ptr != nullptr; }
diff --git a/third_party/llvm-subzero/lib/Support/CommandLine.cpp b/third_party/llvm-subzero/lib/Support/CommandLine.cpp index fa1782c..3e77cd0 100644 --- a/third_party/llvm-subzero/lib/Support/CommandLine.cpp +++ b/third_party/llvm-subzero/lib/Support/CommandLine.cpp
@@ -383,11 +383,16 @@ GlobalParser->registerCategory(this); } -// A special subcommand representing no subcommand -ManagedStatic<SubCommand> llvm::cl::TopLevelSubCommand; +// A special subcommand representing no subcommand. It is particularly important +// that this ManagedStatic uses constant initailization and not dynamic +// initialization because it is referenced from cl::opt constructors, which run +// dynamically in an arbitrary order. +LLVM_REQUIRE_CONSTANT_INITIALIZATION ManagedStatic<SubCommand> +llvm::cl::TopLevelSubCommand; // A special subcommand that can be used to put an option into all subcommands. -ManagedStatic<SubCommand> llvm::cl::AllSubCommands; +LLVM_REQUIRE_CONSTANT_INITIALIZATION ManagedStatic<SubCommand> +llvm::cl::AllSubCommands; void SubCommand::registerSubCommand() { GlobalParser->registerSubCommand(this);
diff --git a/third_party/marl/src/osfiber_x86.c b/third_party/marl/src/osfiber_x86.c index 6c486aa..cac72cb 100644 --- a/third_party/marl/src/osfiber_x86.c +++ b/third_party/marl/src/osfiber_x86.c
@@ -25,12 +25,19 @@ uint32_t stack_size, void (*target)(void*), void* arg) { + // The stack pointer needs to be 16-byte aligned when making a 'call'. + // The 'call' instruction automatically pushes the return instruction to the + // stack (4-bytes), before making the jump. + // The marl_fiber_swap() assembly function does not use 'call', instead it + // uses 'jmp', so we need to offset the ESP pointer by 4 bytes so that the + // stack is still 16-byte aligned when the return target is stack-popped by + // the callee. uintptr_t* stack_top = (uintptr_t*)((uint8_t*)(stack) + stack_size); ctx->EIP = (uintptr_t)&marl_fiber_trampoline; - ctx->ESP = (uintptr_t)&stack_top[-3]; - stack_top[-1] = (uintptr_t)arg; - stack_top[-2] = (uintptr_t)target; - stack_top[-3] = 0; // No return target. + ctx->ESP = (uintptr_t)&stack_top[-5]; + stack_top[-3] = (uintptr_t)arg; + stack_top[-4] = (uintptr_t)target; + stack_top[-5] = 0; // No return target. } #endif // defined(__i386__)