VulkanBenchmarks: add validation layers and load via ICD In DEBUG builds, enable validation layers: VK_LAYER_KHRONOS_validation and VK_LAYER_LUNARG_standard_validation. Also register the debug utils messenger ext callback for when validation layers report a problem. We can force validation layers in non-DEBUG by defining ENABLE_VALIDATION_LAYERS to 1. In order to be able to use validation layers, load the Vulkan driver via ICD, rather than directly. To load SwiftShader, generate a temp icd.json file pointing at the driver path, and temporarily set the VK_ICD_FILENAMES env var to point at it when we load the ICD. This also allows us to load the native GPU driver by defining LOAD_NATIVE_DRIVER to 1 (default is 0). Fixed errors reported by enabling validation layers: * TriangleSampleTexture had a mismatched binding number for the sampler2D * Correctly set memoryTypeIndex for allocations instead of 0 Bug: b/176981107 Change-Id: I3c791086acea048b73d3568d6d7a45d8e0100c17 Reviewed-on: https://swiftshader-review.googlesource.com/c/SwiftShader/+/52749 Kokoro-Result: kokoro <noreply+kokoro@google.com> Reviewed-by: Alexis Hétu <sugoi@google.com> Tested-by: Antonio Maiorano <amaiorano@google.com> Commit-Queue: Antonio Maiorano <amaiorano@google.com>
diff --git a/tests/VulkanBenchmarks/ClearImageBenchmarks.cpp b/tests/VulkanBenchmarks/ClearImageBenchmarks.cpp index 8852903..72a6db2 100644 --- a/tests/VulkanBenchmarks/ClearImageBenchmarks.cpp +++ b/tests/VulkanBenchmarks/ClearImageBenchmarks.cpp
@@ -12,11 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include "Util.hpp" #include "VulkanTester.hpp" #include "benchmark/benchmark.h" #include <cassert> - class ClearImageBenchmark { public: @@ -24,6 +24,7 @@ { tester.initialize(); auto &device = tester.getDevice(); + auto &physicalDevice = tester.getPhysicalDevice(); vk::ImageCreateInfo imageInfo; imageInfo.imageType = vk::ImageType::e2D; @@ -42,7 +43,7 @@ vk::MemoryAllocateInfo allocateInfo; allocateInfo.allocationSize = memoryRequirements.size; - allocateInfo.memoryTypeIndex = 0; + allocateInfo.memoryTypeIndex = Util::getMemoryTypeIndex(physicalDevice, memoryRequirements.memoryTypeBits); memory = device.allocateMemory(allocateInfo);
diff --git a/tests/VulkanBenchmarks/TriangleBenchmarks.cpp b/tests/VulkanBenchmarks/TriangleBenchmarks.cpp index 215d3f4..9050010 100644 --- a/tests/VulkanBenchmarks/TriangleBenchmarks.cpp +++ b/tests/VulkanBenchmarks/TriangleBenchmarks.cpp
@@ -198,7 +198,7 @@ layout(location = 0) out vec4 outColor; - layout(binding = 0) uniform sampler2D texSampler; + layout(binding = 1) uniform sampler2D texSampler; void main() { @@ -221,9 +221,10 @@ tester.onUpdateDescriptorSet([](DrawTester &tester, vk::CommandPool &commandPool, vk::DescriptorSet &descriptorSet) { auto &device = tester.getDevice(); + auto &physicalDevice = tester.getPhysicalDevice(); auto &queue = tester.getQueue(); - auto &texture = tester.addImage(device, 16, 16, vk::Format::eR8G8B8A8Unorm).obj; + auto &texture = tester.addImage(device, physicalDevice, 16, 16, vk::Format::eR8G8B8A8Unorm).obj; // Fill texture with white vk::DeviceSize bufferSize = 16 * 16 * 4;
diff --git a/tests/VulkanWrapper/DrawTester.cpp b/tests/VulkanWrapper/DrawTester.cpp index 2e2e29e..356c892 100644 --- a/tests/VulkanWrapper/DrawTester.cpp +++ b/tests/VulkanWrapper/DrawTester.cpp
@@ -193,7 +193,7 @@ for(size_t i = 0; i < framebuffers.size(); i++) { - framebuffers[i].reset(new Framebuffer(device, swapchain->getImageView(i), swapchain->colorFormat, renderPass, swapchain->getExtent(), multisample)); + framebuffers[i].reset(new Framebuffer(device, physicalDevice, swapchain->getImageView(i), swapchain->colorFormat, renderPass, swapchain->getExtent(), multisample)); } } @@ -392,10 +392,13 @@ } // Draw - commandBuffers[i].bindPipeline(vk::PipelineBindPoint::eGraphics, pipeline); - VULKAN_HPP_NAMESPACE::DeviceSize offset = 0; - commandBuffers[i].bindVertexBuffers(0, 1, &vertices.buffer, &offset); - commandBuffers[i].draw(vertices.numVertices, 1, 0, 0); + if(vertices.numVertices > 0) + { + commandBuffers[i].bindPipeline(vk::PipelineBindPoint::eGraphics, pipeline); + VULKAN_HPP_NAMESPACE::DeviceSize offset = 0; + commandBuffers[i].bindVertexBuffers(0, 1, &vertices.buffer, &offset); + commandBuffers[i].draw(vertices.numVertices, 1, 0, 0); + } commandBuffers[i].endRenderPass(); commandBuffers[i].end();
diff --git a/tests/VulkanWrapper/DrawTester.hpp b/tests/VulkanWrapper/DrawTester.hpp index 431a6c5..e380ef2 100644 --- a/tests/VulkanWrapper/DrawTester.hpp +++ b/tests/VulkanWrapper/DrawTester.hpp
@@ -148,7 +148,7 @@ std::vector<vk::VertexInputAttributeDescription> inputAttributes; vk::PipelineVertexInputStateCreateInfo inputState; - uint32_t numVertices; + uint32_t numVertices = 0; } vertices; vk::DescriptorSetLayout descriptorSetLayout; // Owning handle
diff --git a/tests/VulkanWrapper/Framebuffer.cpp b/tests/VulkanWrapper/Framebuffer.cpp index ab62a10..dd9298a 100644 --- a/tests/VulkanWrapper/Framebuffer.cpp +++ b/tests/VulkanWrapper/Framebuffer.cpp
@@ -14,14 +14,14 @@ #include "Framebuffer.hpp" -Framebuffer::Framebuffer(vk::Device device, vk::ImageView attachment, vk::Format colorFormat, vk::RenderPass renderPass, vk::Extent2D extent, bool multisample) +Framebuffer::Framebuffer(vk::Device device, vk::PhysicalDevice physicalDevice, vk::ImageView attachment, vk::Format colorFormat, vk::RenderPass renderPass, vk::Extent2D extent, bool multisample) : device(device) { std::vector<vk::ImageView> attachments(multisample ? 2 : 1); if(multisample) { - multisampleImage.reset(new Image(device, extent.width, extent.height, colorFormat, vk::SampleCountFlagBits::e4)); + multisampleImage.reset(new Image(device, physicalDevice, extent.width, extent.height, colorFormat, vk::SampleCountFlagBits::e4)); // We'll be rendering to attachment location 0 attachments[0] = multisampleImage->getImageView();
diff --git a/tests/VulkanWrapper/Framebuffer.hpp b/tests/VulkanWrapper/Framebuffer.hpp index b3d1620..505eea1 100644 --- a/tests/VulkanWrapper/Framebuffer.hpp +++ b/tests/VulkanWrapper/Framebuffer.hpp
@@ -21,7 +21,7 @@ class Framebuffer { public: - Framebuffer(vk::Device device, vk::ImageView attachment, vk::Format colorFormat, vk::RenderPass renderPass, vk::Extent2D extent, bool multisample); + Framebuffer(vk::Device device, vk::PhysicalDevice physicalDevice, vk::ImageView attachment, vk::Format colorFormat, vk::RenderPass renderPass, vk::Extent2D extent, bool multisample); ~Framebuffer(); vk::Framebuffer getFramebuffer()
diff --git a/tests/VulkanWrapper/Image.cpp b/tests/VulkanWrapper/Image.cpp index 0ffc69b..c5cbad6 100644 --- a/tests/VulkanWrapper/Image.cpp +++ b/tests/VulkanWrapper/Image.cpp
@@ -13,8 +13,9 @@ // limitations under the License. #include "Image.hpp" +#include "Util.hpp" -Image::Image(vk::Device device, uint32_t width, uint32_t height, vk::Format format, vk::SampleCountFlagBits sampleCount /*= vk::SampleCountFlagBits::e1*/) +Image::Image(vk::Device device, vk::PhysicalDevice physicalDevice, uint32_t width, uint32_t height, vk::Format format, vk::SampleCountFlagBits sampleCount /*= vk::SampleCountFlagBits::e1*/) : device(device) { vk::ImageCreateInfo imageInfo; @@ -34,7 +35,7 @@ vk::MemoryAllocateInfo allocateInfo; allocateInfo.allocationSize = memoryRequirements.size; - allocateInfo.memoryTypeIndex = 0; //getMemoryTypeIndex(memoryRequirements.memoryTypeBits, vk::MemoryPropertyFlagBits::eDeviceLocal); + allocateInfo.memoryTypeIndex = Util::getMemoryTypeIndex(physicalDevice, memoryRequirements.memoryTypeBits); imageMemory = device.allocateMemory(allocateInfo);
diff --git a/tests/VulkanWrapper/Image.hpp b/tests/VulkanWrapper/Image.hpp index 3526ff2..5bfa863 100644 --- a/tests/VulkanWrapper/Image.hpp +++ b/tests/VulkanWrapper/Image.hpp
@@ -20,7 +20,7 @@ class Image { public: - Image(vk::Device device, uint32_t width, uint32_t height, vk::Format format, vk::SampleCountFlagBits sampleCount = vk::SampleCountFlagBits::e1); + Image(vk::Device device, vk::PhysicalDevice physicalDevice, uint32_t width, uint32_t height, vk::Format format, vk::SampleCountFlagBits sampleCount = vk::SampleCountFlagBits::e1); ~Image(); vk::Image getImage()
diff --git a/tests/VulkanWrapper/Util.hpp b/tests/VulkanWrapper/Util.hpp index adfdedb..980c984 100644 --- a/tests/VulkanWrapper/Util.hpp +++ b/tests/VulkanWrapper/Util.hpp
@@ -22,7 +22,7 @@ namespace Util { -uint32_t getMemoryTypeIndex(vk::PhysicalDevice physicalDevice, uint32_t typeBits, vk::MemoryPropertyFlags properties); +uint32_t getMemoryTypeIndex(vk::PhysicalDevice physicalDevice, uint32_t typeBits, vk::MemoryPropertyFlags properties = {}); vk::CommandBuffer beginSingleTimeCommands(vk::Device device, vk::CommandPool commandPool);
diff --git a/tests/VulkanWrapper/VulkanTester.cpp b/tests/VulkanWrapper/VulkanTester.cpp index 90f2ca0..e4c05e5 100644 --- a/tests/VulkanWrapper/VulkanTester.cpp +++ b/tests/VulkanWrapper/VulkanTester.cpp
@@ -13,7 +13,30 @@ // limitations under the License. #include "VulkanTester.hpp" +#include <cstdlib> +#include <filesystem> #include <fstream> +#include <iostream> + +namespace fs = std::filesystem; + +// By default, load SwiftShader via loader +#ifndef LOAD_NATIVE_DRIVER +# define LOAD_NATIVE_DRIVER 0 +#endif + +#ifndef LOAD_SWIFTSHADER_DIRECTLY +# define LOAD_SWIFTSHADER_DIRECTLY 0 +#endif + +#if LOAD_NATIVE_DRIVER && LOAD_SWIFTSHADER_DIRECTLY +# error Enable only one of LOAD_NATIVE_DRIVER and LOAD_SWIFTSHADER_DIRECTLY +#endif + +// By default, enable validation layers in DEBUG builds +#if !defined(ENABLE_VALIDATION_LAYERS) && !defined(NDEBUG) +# define ENABLE_VALIDATION_LAYERS 1 +#endif #if defined(_WIN32) # define OS_WINDOWS 1 @@ -33,7 +56,107 @@ # error Unimplemented platform #endif +// TODO: move to its own header/cpp +// Wraps a single environment variable, allowing it to be set +// and automatically restored on destruction. +class ScopedSetEnvVar +{ +public: + ScopedSetEnvVar(std::string name) + : name(name) + { + assert(!name.empty()); + } + + ScopedSetEnvVar(std::string name, std::string value) + : name(name) + { + set(value); + } + + ~ScopedSetEnvVar() + { + restore(); + } + + void set(std::string value) + { + restore(); + if(auto ov = getEnv(name.data())) + { + oldValue = ov; + } + putEnv((name + std::string("=") + value).c_str()); + } + + void restore() + { + if(!oldValue.empty()) + { + putEnv((name + std::string("=") + oldValue).c_str()); + oldValue.clear(); + } + } + +private: + void putEnv(const char *env) + { + // POSIX putenv needs 'env' to live beyond the call + envCopy = env; +#if OS_WINDOWS + [[maybe_unused]] auto r = ::_putenv(envCopy.c_str()); + assert(r == 0); +#else + [[maybe_unused]] auto r = ::putenv(const_cast<char *>(envCopy.c_str())); + assert(r == 0); +#endif + } + + const char *getEnv(const char *name) + { + return ::getenv(name); + } + + std::string name; + std::string oldValue; + std::string envCopy; +}; + +// Generates a temporary icd.json file that sets library_path at the input driverPath, +// and sets VK_ICD_FILENAMES environment variable to this file, restoring the env var +// and deleting the temp file on destruction. +class ScopedSetIcdFilenames +{ +public: + ScopedSetIcdFilenames() = default; + ScopedSetIcdFilenames(const char *driverPath) + { + std::ofstream fout(icdFileName); + assert(fout && "Failed to create generated icd file"); + fout << R"raw({ "file_format_version": "1.0.0", "ICD": { "library_path": ")raw" << driverPath << R"raw(", "api_version": "1.0.5" } } )raw"; + fout.close(); + + setEnvVar.set(icdFileName); + } + + ~ScopedSetIcdFilenames() + { + //TODO(b/180494886): fix C++17 filesystem issues on macOS +#if !OS_MAC + if(fs::exists("vk_swiftshader_generated_icd.json")) + { + fs::remove("vk_swiftshader_generated_icd.json"); + } +#endif + } + +private: + static constexpr const char *icdFileName = "vk_swiftshader_generated_icd.json"; + ScopedSetEnvVar setEnvVar{ "VK_ICD_FILENAMES" }; +}; + namespace { + std::vector<const char *> getDriverPaths() { #if OS_WINDOWS @@ -91,24 +214,25 @@ return f.good(); } -std::unique_ptr<vk::DynamicLoader> loadDriver() +std::string findDriverPath() { - for(auto &p : getDriverPaths()) + for(auto &path : getDriverPaths()) { - if(!fileExists(p)) - continue; - return std::make_unique<vk::DynamicLoader>(p); + if(fileExists(path)) + return path; } -#if(OS_MAC || OS_LINUX || OS_ANDROID || OS_FUCHSIA) +#if(OS_LINUX || OS_ANDROID || OS_FUCHSIA) // On Linux-based OSes, the lib path may be resolved by dlopen - for(auto &p : getDriverPaths()) + for(auto &path : getDriverPaths()) { - auto lib = dlopen(p, RTLD_LAZY | RTLD_LOCAL); + auto lib = dlopen(path, RTLD_LAZY | RTLD_LOCAL); if(lib) { + char libPath[2048] = { '\0' }; + dlinfo(lib, RTLD_DI_ORIGIN, libPath); dlclose(lib); - return std::make_unique<vk::DynamicLoader>(p); + return std::string{ libPath } + "/" + path; } } #endif @@ -118,10 +242,13 @@ } // namespace +VulkanTester::VulkanTester() = default; + VulkanTester::~VulkanTester() { device.waitIdle(); device.destroy(nullptr); + if(debugReport) instance.destroy(debugReport); instance.destroy(nullptr); } @@ -133,9 +260,75 @@ PFN_vkGetInstanceProcAddr vkGetInstanceProcAddr = dl->getProcAddress<PFN_vkGetInstanceProcAddr>("vkGetInstanceProcAddr"); VULKAN_HPP_DEFAULT_DISPATCHER.init(vkGetInstanceProcAddr); - instance = vk::createInstance({}, nullptr); + vk::InstanceCreateInfo instanceCreateInfo; + std::vector<const char *> extensionNames + { + VK_KHR_SURFACE_EXTENSION_NAME, +#if defined(USE_HEADLESS_SURFACE) + VK_EXT_HEADLESS_SURFACE_EXTENSION_NAME, +#endif +#if defined(VK_USE_PLATFORM_WIN32_KHR) + VK_KHR_WIN32_SURFACE_EXTENSION_NAME, +#endif + }; +#if ENABLE_VALIDATION_LAYERS + extensionNames.push_back(VK_EXT_DEBUG_UTILS_EXTENSION_NAME); +#endif + + auto addLayerIfAvailable = [](std::vector<const char *> &layers, const char *layer) { + static auto layerProperties = vk::enumerateInstanceLayerProperties(); + if(std::find_if(layerProperties.begin(), layerProperties.end(), [layer](auto &lp) { + return strcmp(layer, lp.layerName) == 0; + }) != layerProperties.end()) + { + //std::cout << "Enabled layer: " << layer << std::endl; + layers.push_back(layer); + } + }; + + std::vector<const char *> layerNames; +#if ENABLE_VALIDATION_LAYERS + addLayerIfAvailable(layerNames, "VK_LAYER_KHRONOS_validation"); + addLayerIfAvailable(layerNames, "VK_LAYER_LUNARG_standard_validation"); +#endif + + instanceCreateInfo.ppEnabledExtensionNames = extensionNames.data(); + instanceCreateInfo.enabledExtensionCount = static_cast<uint32_t>(extensionNames.size()); + instanceCreateInfo.ppEnabledLayerNames = layerNames.data(); + instanceCreateInfo.enabledLayerCount = static_cast<uint32_t>(layerNames.size()); + + instance = vk::createInstance(instanceCreateInfo, nullptr); VULKAN_HPP_DEFAULT_DISPATCHER.init(instance); +#if ENABLE_VALIDATION_LAYERS + if(VULKAN_HPP_DEFAULT_DISPATCHER.vkCreateDebugUtilsMessengerEXT) + { + vk::DebugUtilsMessengerCreateInfoEXT debugInfo; + debugInfo.messageSeverity = + // vk::DebugUtilsMessageSeverityFlagBitsEXT::eVerbose | + vk::DebugUtilsMessageSeverityFlagBitsEXT::eError | + vk::DebugUtilsMessageSeverityFlagBitsEXT::eWarning; + + debugInfo.messageType = vk::DebugUtilsMessageTypeFlagBitsEXT::eGeneral | + vk::DebugUtilsMessageTypeFlagBitsEXT::eValidation | + vk::DebugUtilsMessageTypeFlagBitsEXT::ePerformance; + + PFN_vkDebugUtilsMessengerCallbackEXT debugInfoCallback = + []( + VkDebugUtilsMessageSeverityFlagBitsEXT messageSeverity, + VkDebugUtilsMessageTypeFlagsEXT messageTypes, + const VkDebugUtilsMessengerCallbackDataEXT *pCallbackData, + void *pUserData) -> VkBool32 { + //assert(false); + std::cerr << "[DebugInfoCallback] " << pCallbackData->pMessage << std::endl; + return VK_FALSE; + }; + + debugInfo.pfnUserCallback = debugInfoCallback; + debugReport = instance.createDebugUtilsMessengerEXT(debugInfo); + } +#endif + std::vector<vk::PhysicalDevice> physicalDevices = instance.enumeratePhysicalDevices(); assert(!physicalDevices.empty()); physicalDevice = physicalDevices[0]; @@ -146,11 +339,56 @@ queueCreateInfo.queueCount = 1; queueCreateInfo.pQueuePriorities = &defaultQueuePriority; + std::vector<const char *> deviceExtensions = { + VK_KHR_SWAPCHAIN_EXTENSION_NAME, + }; + vk::DeviceCreateInfo deviceCreateInfo; deviceCreateInfo.queueCreateInfoCount = 1; deviceCreateInfo.pQueueCreateInfos = &queueCreateInfo; + deviceCreateInfo.ppEnabledExtensionNames = deviceExtensions.data(); + deviceCreateInfo.enabledExtensionCount = static_cast<uint32_t>(deviceExtensions.size()); device = physicalDevice.createDevice(deviceCreateInfo, nullptr); queue = device.getQueue(queueFamilyIndex, 0); } + +std::unique_ptr<vk::DynamicLoader> VulkanTester::loadDriver() +{ + if(LOAD_NATIVE_DRIVER) + { + return std::make_unique<vk::DynamicLoader>(); + } + + auto driverPath = findDriverPath(); + assert(!driverPath.empty()); + + if(LOAD_SWIFTSHADER_DIRECTLY) + { + return std::make_unique<vk::DynamicLoader>(driverPath); + } + + // Load SwiftShader via loader + + // Set VK_ICD_FILENAMES env var so it gets picked up by the loading of the ICD driver + setIcdFilenames = std::make_unique<ScopedSetIcdFilenames>(driverPath.c_str()); + + std::unique_ptr<vk::DynamicLoader> dl; +#ifndef VULKAN_HPP_NO_EXCEPTIONS + try + { + dl = std::make_unique<vk::DynamicLoader>(); + } + catch(std::exception &ex) + { + std::cerr << "vk::DynamicLoader exception: " << ex.what() << std::endl; + std::cerr << "Falling back to loading SwiftShader directly (i.e. no validation layers)" << std::endl; + dl = std::make_unique<vk::DynamicLoader>(driverPath); + } +#else + dl = std::make_unique<vk::DynamicLoader>(); +#endif + + return dl; +}
diff --git a/tests/VulkanWrapper/VulkanTester.hpp b/tests/VulkanWrapper/VulkanTester.hpp index 89b7d05..044bffc 100644 --- a/tests/VulkanWrapper/VulkanTester.hpp +++ b/tests/VulkanWrapper/VulkanTester.hpp
@@ -20,19 +20,24 @@ class VulkanTester { public: - VulkanTester() = default; + VulkanTester(); virtual ~VulkanTester(); // Call once after construction so that virtual functions may be called during init void initialize(); const vk::DynamicLoader &dynamicLoader() const { return *dl; } - vk::Device &getDevice() { return this->device; } - vk::Queue &getQueue() { return this->queue; } + vk::PhysicalDevice &getPhysicalDevice() { return physicalDevice; } + vk::Device &getDevice() { return device; } + vk::Queue &getQueue() { return queue; } uint32_t getQueueFamilyIndex() const { return queueFamilyIndex; } private: + std::unique_ptr<vk::DynamicLoader> loadDriver(); + + std::unique_ptr<class ScopedSetIcdFilenames> setIcdFilenames; std::unique_ptr<vk::DynamicLoader> dl; + vk::DebugUtilsMessengerEXT debugReport; protected: const uint32_t queueFamilyIndex = 0;
diff --git a/tests/VulkanWrapper/Window.cpp b/tests/VulkanWrapper/Window.cpp index 1cfdbe6..ad29cfc 100644 --- a/tests/VulkanWrapper/Window.cpp +++ b/tests/VulkanWrapper/Window.cpp
@@ -17,6 +17,7 @@ #if USE_HEADLESS_SURFACE Window::Window(vk::Instance instance, vk::Extent2D windowSize) + : instance(instance) { vk::HeadlessSurfaceCreateInfoEXT surfaceCreateInfo; surface = instance.createHeadlessSurfaceEXT(surfaceCreateInfo); @@ -40,6 +41,7 @@ #elif defined(_WIN32) Window::Window(vk::Instance instance, vk::Extent2D windowSize) + : instance(instance) { windowClass.cbSize = sizeof(WNDCLASSEX); windowClass.style = CS_HREDRAW | CS_VREDRAW;