Vulkan: Add debugging of the command buffer. Add a debugger context and server to vk::Device, along with a getter for acquiring the debugger context. Generate a synthetic file containing all the Vulkan commands in the command buffer, and allow the debugger to single line step over these. All of this is no-op unless ENABLE_VK_DEBUGGER is defined at compile time, and the VK_DEBUGGER_PORT env var is set at run time. Bug: b/145351270 Change-Id: I8bea398a6c08d4cb23d76172dbca2a08ae109a6d Reviewed-on: https://swiftshader-review.googlesource.com/c/SwiftShader/+/38913 Kokoro-Presubmit: kokoro <noreply+kokoro@google.com> Reviewed-by: Nicolas Capens <nicolascapens@google.com> Tested-by: Ben Clayton <bclayton@google.com>
diff --git a/src/Vulkan/Debug/Value.hpp b/src/Vulkan/Debug/Value.hpp index 9e85ff2..132f818 100644 --- a/src/Vulkan/Debug/Value.hpp +++ b/src/Vulkan/Debug/Value.hpp
@@ -15,14 +15,14 @@ #ifndef VK_DEBUG_VALUE_HPP_ #define VK_DEBUG_VALUE_HPP_ +#include "Type.hpp" + #include <memory> #include <string> namespace vk { namespace dbg { -class Type; - // FormatFlags holds settings used to serialize a Value to a string. struct FormatFlags {
diff --git a/src/Vulkan/VkCommandBuffer.cpp b/src/Vulkan/VkCommandBuffer.cpp index 60e968e..186b1ec 100644 --- a/src/Vulkan/VkCommandBuffer.cpp +++ b/src/Vulkan/VkCommandBuffer.cpp
@@ -25,6 +25,12 @@ #include "VkRenderPass.hpp" #include "Device/Renderer.hpp" +#include "./Debug/Context.hpp" +#include "./Debug/File.hpp" +#include "./Debug/Thread.hpp" + +#include "marl/defer.h" + #include <cstring> class vk::CommandBuffer::Command @@ -1281,8 +1287,9 @@ namespace vk { -CommandBuffer::CommandBuffer(VkCommandBufferLevel pLevel) - : level(pLevel) +CommandBuffer::CommandBuffer(Device *device, VkCommandBufferLevel pLevel) + : device(device) + , level(pLevel) { // FIXME (b/119409619): replace this vector by an allocator so we can control all memory allocations commands = new std::vector<std::unique_ptr<Command>>(); @@ -1329,6 +1336,19 @@ state = EXECUTABLE; +#ifdef ENABLE_VK_DEBUGGER + auto debuggerContext = device->getDebuggerContext(); + if(debuggerContext) + { + std::string source; + for(auto &command : *commands) + { + source += command->description() + "\n"; + } + debuggerFile = debuggerContext->lock().createVirtualFile("VkCommandBuffer", source.c_str()); + } +#endif // ENABLE_VK_DEBUGGER + return VK_SUCCESS; } @@ -1745,8 +1765,30 @@ // Perform recorded work state = PENDING; +#ifdef ENABLE_VK_DEBUGGER + std::shared_ptr<vk::dbg::Thread> debuggerThread; + auto debuggerContext = device->getDebuggerContext(); + if(debuggerContext) + { + auto lock = debuggerContext->lock(); + debuggerThread = lock.currentThread(); + debuggerThread->setName("vkQueue processor"); + debuggerThread->enter(lock, debuggerFile, "vkCommandBuffer::submit"); + lock.unlock(); + } + defer(if(debuggerThread) { debuggerThread->exit(); }); + int line = 1; +#endif // ENABLE_VK_DEBUGGER + for(auto &command : *commands) { +#ifdef ENABLE_VK_DEBUGGER + if(debuggerThread) + { + debuggerThread->update({ line++, debuggerFile }); + } +#endif // ENABLE_VK_DEBUGGER + command->play(executionState); }
diff --git a/src/Vulkan/VkCommandBuffer.hpp b/src/Vulkan/VkCommandBuffer.hpp index 497e8d1..0e03fc7 100644 --- a/src/Vulkan/VkCommandBuffer.hpp +++ b/src/Vulkan/VkCommandBuffer.hpp
@@ -20,6 +20,7 @@ #include "VkObject.hpp" #include "Device/Color.hpp" #include "Device/Context.hpp" + #include <memory> #include <vector> @@ -33,6 +34,11 @@ namespace vk { +namespace dbg { +class File; +} // namespace dbg + +class Device; class Buffer; class Event; class Framebuffer; @@ -47,7 +53,7 @@ public: static constexpr VkSystemAllocationScope GetAllocationScope() { return VK_SYSTEM_ALLOCATION_SCOPE_OBJECT; } - CommandBuffer(VkCommandBufferLevel pLevel); + CommandBuffer(Device *device, VkCommandBufferLevel pLevel); static inline CommandBuffer *Cast(VkCommandBuffer object) { @@ -201,11 +207,17 @@ PENDING, INVALID }; + + Device *const device; State state = INITIAL; VkCommandBufferLevel level = VK_COMMAND_BUFFER_LEVEL_PRIMARY; // FIXME (b/119409619): replace this vector by an allocator so we can control all memory allocations std::vector<std::unique_ptr<Command>> *commands; + +#ifdef ENABLE_VK_DEBUGGER + std::shared_ptr<vk::dbg::File> debuggerFile; +#endif // ENABLE_VK_DEBUGGER }; using DispatchableCommandBuffer = DispatchableObject<CommandBuffer, VkCommandBuffer>;
diff --git a/src/Vulkan/VkCommandPool.cpp b/src/Vulkan/VkCommandPool.cpp index 35ec438..fe467e0 100644 --- a/src/Vulkan/VkCommandPool.cpp +++ b/src/Vulkan/VkCommandPool.cpp
@@ -46,7 +46,7 @@ return 0; } -VkResult CommandPool::allocateCommandBuffers(VkCommandBufferLevel level, uint32_t commandBufferCount, VkCommandBuffer *pCommandBuffers) +VkResult CommandPool::allocateCommandBuffers(Device *device, VkCommandBufferLevel level, uint32_t commandBufferCount, VkCommandBuffer *pCommandBuffers) { for(uint32_t i = 0; i < commandBufferCount; i++) { @@ -54,7 +54,7 @@ void *deviceMemory = vk::allocate(sizeof(DispatchableCommandBuffer), REQUIRED_MEMORY_ALIGNMENT, DEVICE_MEMORY, DispatchableCommandBuffer::GetAllocationScope()); ASSERT(deviceMemory); - DispatchableCommandBuffer *commandBuffer = new(deviceMemory) DispatchableCommandBuffer(level); + DispatchableCommandBuffer *commandBuffer = new(deviceMemory) DispatchableCommandBuffer(device, level); if(commandBuffer) { pCommandBuffers[i] = *commandBuffer;
diff --git a/src/Vulkan/VkCommandPool.hpp b/src/Vulkan/VkCommandPool.hpp index f4f99c2..0aec423 100644 --- a/src/Vulkan/VkCommandPool.hpp +++ b/src/Vulkan/VkCommandPool.hpp
@@ -16,10 +16,13 @@ #define VK_COMMAND_POOL_HPP_ #include "VkObject.hpp" + #include <set> namespace vk { +class Device; + class CommandPool : public Object<CommandPool, VkCommandPool> { public: @@ -28,7 +31,7 @@ static size_t ComputeRequiredAllocationSize(const VkCommandPoolCreateInfo *pCreateInfo); - VkResult allocateCommandBuffers(VkCommandBufferLevel level, uint32_t commandBufferCount, VkCommandBuffer *pCommandBuffers); + VkResult allocateCommandBuffers(Device *device, VkCommandBufferLevel level, uint32_t commandBufferCount, VkCommandBuffer *pCommandBuffers); void freeCommandBuffers(uint32_t commandBufferCount, const VkCommandBuffer *pCommandBuffers); VkResult reset(VkCommandPoolResetFlags flags); void trim(VkCommandPoolTrimFlags flags);
diff --git a/src/Vulkan/VkDevice.cpp b/src/Vulkan/VkDevice.cpp index cbc399c..8a92ac3 100644 --- a/src/Vulkan/VkDevice.cpp +++ b/src/Vulkan/VkDevice.cpp
@@ -19,6 +19,8 @@ #include "VkDescriptorSetLayout.hpp" #include "VkFence.hpp" #include "VkQueue.hpp" +#include "Debug/Context.hpp" +#include "Debug/Server.hpp" #include "Device/Blitter.hpp" #include <chrono> @@ -97,6 +99,18 @@ // FIXME (b/119409619): use an allocator here so we can control all memory allocations blitter.reset(new sw::Blitter()); samplingRoutineCache.reset(new SamplingRoutineCache()); + +#ifdef ENABLE_VK_DEBUGGER + static auto port = getenv("VK_DEBUGGER_PORT"); + if(port) + { + // Construct the debugger context and server - this may block for a + // debugger connection, allowing breakpoints to be set before they're + // executed. + debugger.context = vk::dbg::Context::create(); + debugger.server = vk::dbg::Server::create(debugger.context, atoi(port)); + } +#endif // ENABLE_VK_DEBUGGER } void Device::destroy(const VkAllocationCallbacks *pAllocator)
diff --git a/src/Vulkan/VkDevice.hpp b/src/Vulkan/VkDevice.hpp index 94fbba6..d83e8e8 100644 --- a/src/Vulkan/VkDevice.hpp +++ b/src/Vulkan/VkDevice.hpp
@@ -33,6 +33,11 @@ class PhysicalDevice; class Queue; +namespace dbg { +class Context; +class Server; +} // namespace dbg + class Device { public: @@ -93,6 +98,13 @@ rr::Routine *findInConstCache(const SamplingRoutineCache::Key &key) const; void updateSamplingRoutineConstCache(); +#ifdef ENABLE_VK_DEBUGGER + std::shared_ptr<vk::dbg::Context> getDebuggerContext() const + { + return debugger.context; + } +#endif // ENABLE_VK_DEBUGGER + private: PhysicalDevice *const physicalDevice = nullptr; Queue *const queues = nullptr; @@ -105,6 +117,14 @@ ExtensionName *extensions = nullptr; const VkPhysicalDeviceFeatures enabledFeatures = {}; std::shared_ptr<marl::Scheduler> scheduler; + +#ifdef ENABLE_VK_DEBUGGER + struct + { + std::shared_ptr<vk::dbg::Context> context; + std::shared_ptr<vk::dbg::Server> server; + } debugger; +#endif // ENABLE_VK_DEBUGGER }; using DispatchableDevice = DispatchableObject<Device, VkDevice>;
diff --git a/src/Vulkan/libVulkan.cpp b/src/Vulkan/libVulkan.cpp index 8c5484d..6af8bd9 100644 --- a/src/Vulkan/libVulkan.cpp +++ b/src/Vulkan/libVulkan.cpp
@@ -1964,7 +1964,7 @@ nextInfo = nextInfo->pNext; } - return vk::Cast(pAllocateInfo->commandPool)->allocateCommandBuffers(pAllocateInfo->level, pAllocateInfo->commandBufferCount, pCommandBuffers); + return vk::Cast(pAllocateInfo->commandPool)->allocateCommandBuffers(vk::Cast(device), pAllocateInfo->level, pAllocateInfo->commandBufferCount, pCommandBuffers); } VKAPI_ATTR void VKAPI_CALL vkFreeCommandBuffers(VkDevice device, VkCommandPool commandPool, uint32_t commandBufferCount, const VkCommandBuffer *pCommandBuffers)