Vulkan: Add debugging of the command buffer.
Add a debugger context and server to vk::Device, along with a getter for acquiring the debugger context.
Generate a synthetic file containing all the Vulkan commands in the command buffer, and allow the debugger to single line step over these.
All of this is no-op unless ENABLE_VK_DEBUGGER is defined at compile time, and the VK_DEBUGGER_PORT env var is set at run time.
Bug: b/145351270
Change-Id: I8bea398a6c08d4cb23d76172dbca2a08ae109a6d
Reviewed-on: https://swiftshader-review.googlesource.com/c/SwiftShader/+/38913
Kokoro-Presubmit: kokoro <noreply+kokoro@google.com>
Reviewed-by: Nicolas Capens <nicolascapens@google.com>
Tested-by: Ben Clayton <bclayton@google.com>
diff --git a/src/Vulkan/Debug/Value.hpp b/src/Vulkan/Debug/Value.hpp
index 9e85ff2..132f818 100644
--- a/src/Vulkan/Debug/Value.hpp
+++ b/src/Vulkan/Debug/Value.hpp
@@ -15,14 +15,14 @@
#ifndef VK_DEBUG_VALUE_HPP_
#define VK_DEBUG_VALUE_HPP_
+#include "Type.hpp"
+
#include <memory>
#include <string>
namespace vk {
namespace dbg {
-class Type;
-
// FormatFlags holds settings used to serialize a Value to a string.
struct FormatFlags
{
diff --git a/src/Vulkan/VkCommandBuffer.cpp b/src/Vulkan/VkCommandBuffer.cpp
index 60e968e..186b1ec 100644
--- a/src/Vulkan/VkCommandBuffer.cpp
+++ b/src/Vulkan/VkCommandBuffer.cpp
@@ -25,6 +25,12 @@
#include "VkRenderPass.hpp"
#include "Device/Renderer.hpp"
+#include "./Debug/Context.hpp"
+#include "./Debug/File.hpp"
+#include "./Debug/Thread.hpp"
+
+#include "marl/defer.h"
+
#include <cstring>
class vk::CommandBuffer::Command
@@ -1281,8 +1287,9 @@
namespace vk {
-CommandBuffer::CommandBuffer(VkCommandBufferLevel pLevel)
- : level(pLevel)
+CommandBuffer::CommandBuffer(Device *device, VkCommandBufferLevel pLevel)
+ : device(device)
+ , level(pLevel)
{
// FIXME (b/119409619): replace this vector by an allocator so we can control all memory allocations
commands = new std::vector<std::unique_ptr<Command>>();
@@ -1329,6 +1336,19 @@
state = EXECUTABLE;
+#ifdef ENABLE_VK_DEBUGGER
+ auto debuggerContext = device->getDebuggerContext();
+ if(debuggerContext)
+ {
+ std::string source;
+ for(auto &command : *commands)
+ {
+ source += command->description() + "\n";
+ }
+ debuggerFile = debuggerContext->lock().createVirtualFile("VkCommandBuffer", source.c_str());
+ }
+#endif // ENABLE_VK_DEBUGGER
+
return VK_SUCCESS;
}
@@ -1745,8 +1765,30 @@
// Perform recorded work
state = PENDING;
+#ifdef ENABLE_VK_DEBUGGER
+ std::shared_ptr<vk::dbg::Thread> debuggerThread;
+ auto debuggerContext = device->getDebuggerContext();
+ if(debuggerContext)
+ {
+ auto lock = debuggerContext->lock();
+ debuggerThread = lock.currentThread();
+ debuggerThread->setName("vkQueue processor");
+ debuggerThread->enter(lock, debuggerFile, "vkCommandBuffer::submit");
+ lock.unlock();
+ }
+ defer(if(debuggerThread) { debuggerThread->exit(); });
+ int line = 1;
+#endif // ENABLE_VK_DEBUGGER
+
for(auto &command : *commands)
{
+#ifdef ENABLE_VK_DEBUGGER
+ if(debuggerThread)
+ {
+ debuggerThread->update({ line++, debuggerFile });
+ }
+#endif // ENABLE_VK_DEBUGGER
+
command->play(executionState);
}
diff --git a/src/Vulkan/VkCommandBuffer.hpp b/src/Vulkan/VkCommandBuffer.hpp
index 497e8d1..0e03fc7 100644
--- a/src/Vulkan/VkCommandBuffer.hpp
+++ b/src/Vulkan/VkCommandBuffer.hpp
@@ -20,6 +20,7 @@
#include "VkObject.hpp"
#include "Device/Color.hpp"
#include "Device/Context.hpp"
+
#include <memory>
#include <vector>
@@ -33,6 +34,11 @@
namespace vk {
+namespace dbg {
+class File;
+} // namespace dbg
+
+class Device;
class Buffer;
class Event;
class Framebuffer;
@@ -47,7 +53,7 @@
public:
static constexpr VkSystemAllocationScope GetAllocationScope() { return VK_SYSTEM_ALLOCATION_SCOPE_OBJECT; }
- CommandBuffer(VkCommandBufferLevel pLevel);
+ CommandBuffer(Device *device, VkCommandBufferLevel pLevel);
static inline CommandBuffer *Cast(VkCommandBuffer object)
{
@@ -201,11 +207,17 @@
PENDING,
INVALID
};
+
+ Device *const device;
State state = INITIAL;
VkCommandBufferLevel level = VK_COMMAND_BUFFER_LEVEL_PRIMARY;
// FIXME (b/119409619): replace this vector by an allocator so we can control all memory allocations
std::vector<std::unique_ptr<Command>> *commands;
+
+#ifdef ENABLE_VK_DEBUGGER
+ std::shared_ptr<vk::dbg::File> debuggerFile;
+#endif // ENABLE_VK_DEBUGGER
};
using DispatchableCommandBuffer = DispatchableObject<CommandBuffer, VkCommandBuffer>;
diff --git a/src/Vulkan/VkCommandPool.cpp b/src/Vulkan/VkCommandPool.cpp
index 35ec438..fe467e0 100644
--- a/src/Vulkan/VkCommandPool.cpp
+++ b/src/Vulkan/VkCommandPool.cpp
@@ -46,7 +46,7 @@
return 0;
}
-VkResult CommandPool::allocateCommandBuffers(VkCommandBufferLevel level, uint32_t commandBufferCount, VkCommandBuffer *pCommandBuffers)
+VkResult CommandPool::allocateCommandBuffers(Device *device, VkCommandBufferLevel level, uint32_t commandBufferCount, VkCommandBuffer *pCommandBuffers)
{
for(uint32_t i = 0; i < commandBufferCount; i++)
{
@@ -54,7 +54,7 @@
void *deviceMemory = vk::allocate(sizeof(DispatchableCommandBuffer), REQUIRED_MEMORY_ALIGNMENT,
DEVICE_MEMORY, DispatchableCommandBuffer::GetAllocationScope());
ASSERT(deviceMemory);
- DispatchableCommandBuffer *commandBuffer = new(deviceMemory) DispatchableCommandBuffer(level);
+ DispatchableCommandBuffer *commandBuffer = new(deviceMemory) DispatchableCommandBuffer(device, level);
if(commandBuffer)
{
pCommandBuffers[i] = *commandBuffer;
diff --git a/src/Vulkan/VkCommandPool.hpp b/src/Vulkan/VkCommandPool.hpp
index f4f99c2..0aec423 100644
--- a/src/Vulkan/VkCommandPool.hpp
+++ b/src/Vulkan/VkCommandPool.hpp
@@ -16,10 +16,13 @@
#define VK_COMMAND_POOL_HPP_
#include "VkObject.hpp"
+
#include <set>
namespace vk {
+class Device;
+
class CommandPool : public Object<CommandPool, VkCommandPool>
{
public:
@@ -28,7 +31,7 @@
static size_t ComputeRequiredAllocationSize(const VkCommandPoolCreateInfo *pCreateInfo);
- VkResult allocateCommandBuffers(VkCommandBufferLevel level, uint32_t commandBufferCount, VkCommandBuffer *pCommandBuffers);
+ VkResult allocateCommandBuffers(Device *device, VkCommandBufferLevel level, uint32_t commandBufferCount, VkCommandBuffer *pCommandBuffers);
void freeCommandBuffers(uint32_t commandBufferCount, const VkCommandBuffer *pCommandBuffers);
VkResult reset(VkCommandPoolResetFlags flags);
void trim(VkCommandPoolTrimFlags flags);
diff --git a/src/Vulkan/VkDevice.cpp b/src/Vulkan/VkDevice.cpp
index cbc399c..8a92ac3 100644
--- a/src/Vulkan/VkDevice.cpp
+++ b/src/Vulkan/VkDevice.cpp
@@ -19,6 +19,8 @@
#include "VkDescriptorSetLayout.hpp"
#include "VkFence.hpp"
#include "VkQueue.hpp"
+#include "Debug/Context.hpp"
+#include "Debug/Server.hpp"
#include "Device/Blitter.hpp"
#include <chrono>
@@ -97,6 +99,18 @@
// FIXME (b/119409619): use an allocator here so we can control all memory allocations
blitter.reset(new sw::Blitter());
samplingRoutineCache.reset(new SamplingRoutineCache());
+
+#ifdef ENABLE_VK_DEBUGGER
+ static auto port = getenv("VK_DEBUGGER_PORT");
+ if(port)
+ {
+ // Construct the debugger context and server - this may block for a
+ // debugger connection, allowing breakpoints to be set before they're
+ // executed.
+ debugger.context = vk::dbg::Context::create();
+ debugger.server = vk::dbg::Server::create(debugger.context, atoi(port));
+ }
+#endif // ENABLE_VK_DEBUGGER
}
void Device::destroy(const VkAllocationCallbacks *pAllocator)
diff --git a/src/Vulkan/VkDevice.hpp b/src/Vulkan/VkDevice.hpp
index 94fbba6..d83e8e8 100644
--- a/src/Vulkan/VkDevice.hpp
+++ b/src/Vulkan/VkDevice.hpp
@@ -33,6 +33,11 @@
class PhysicalDevice;
class Queue;
+namespace dbg {
+class Context;
+class Server;
+} // namespace dbg
+
class Device
{
public:
@@ -93,6 +98,13 @@
rr::Routine *findInConstCache(const SamplingRoutineCache::Key &key) const;
void updateSamplingRoutineConstCache();
+#ifdef ENABLE_VK_DEBUGGER
+ std::shared_ptr<vk::dbg::Context> getDebuggerContext() const
+ {
+ return debugger.context;
+ }
+#endif // ENABLE_VK_DEBUGGER
+
private:
PhysicalDevice *const physicalDevice = nullptr;
Queue *const queues = nullptr;
@@ -105,6 +117,14 @@
ExtensionName *extensions = nullptr;
const VkPhysicalDeviceFeatures enabledFeatures = {};
std::shared_ptr<marl::Scheduler> scheduler;
+
+#ifdef ENABLE_VK_DEBUGGER
+ struct
+ {
+ std::shared_ptr<vk::dbg::Context> context;
+ std::shared_ptr<vk::dbg::Server> server;
+ } debugger;
+#endif // ENABLE_VK_DEBUGGER
};
using DispatchableDevice = DispatchableObject<Device, VkDevice>;
diff --git a/src/Vulkan/libVulkan.cpp b/src/Vulkan/libVulkan.cpp
index 8c5484d..6af8bd9 100644
--- a/src/Vulkan/libVulkan.cpp
+++ b/src/Vulkan/libVulkan.cpp
@@ -1964,7 +1964,7 @@
nextInfo = nextInfo->pNext;
}
- return vk::Cast(pAllocateInfo->commandPool)->allocateCommandBuffers(pAllocateInfo->level, pAllocateInfo->commandBufferCount, pCommandBuffers);
+ return vk::Cast(pAllocateInfo->commandPool)->allocateCommandBuffers(vk::Cast(device), pAllocateInfo->level, pAllocateInfo->commandBufferCount, pCommandBuffers);
}
VKAPI_ATTR void VKAPI_CALL vkFreeCommandBuffers(VkDevice device, VkCommandPool commandPool, uint32_t commandBufferCount, const VkCommandBuffer *pCommandBuffers)
