Revert "Migrate from yarn to marl." This reverts commit 1359b66e24006fccc811634f422fcda48b30d195. Reason for revert: still breaking chrome autorollers. Change-Id: I35d389f542beb3cfe31b1fa3143dfad4ab7e4a7a Reviewed-on: https://swiftshader-review.googlesource.com/c/SwiftShader/+/36110 Reviewed-by: Ben Clayton <bclayton@google.com> Tested-by: Ben Clayton <bclayton@google.com>
diff --git a/CMakeLists.txt b/CMakeLists.txt index 9fc93a5..370ba96 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt
@@ -106,8 +106,6 @@ option(REACTOR_EMIT_PRINT_LOCATION "Emit printing of location info for JIT functions" 0) option(REACTOR_VERIFY_LLVM_IR "Check reactor-generated LLVM IR is valid even in release builds" 0) -set(BUILD_MARL ${BUILD_VULKAN}) - set(DEFAULT_REACTOR_BACKEND "LLVM") set(REACTOR_BACKEND ${DEFAULT_REACTOR_BACKEND} CACHE STRING "JIT compiler back-end used by Reactor") @@ -267,6 +265,7 @@ ########################################################### set(SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/src) +set(YARN_DIR ${SOURCE_DIR}/Yarn) set(OPENGL_DIR ${SOURCE_DIR}/OpenGL) set(OPENGL_COMPILER_DIR ${OPENGL_DIR}/compiler) set(VULKAN_DIR ${SOURCE_DIR}/Vulkan) @@ -275,7 +274,6 @@ set(LIBBACKTRACE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/third_party/libbacktrace/src) set(LIBBACKTRACE_CONFIG_DIR ${CMAKE_CURRENT_SOURCE_DIR}/third_party/libbacktrace/config) set(LIBBACKTRACE_INCLUDE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/third_party/libbacktrace/src) -set(MARL_INCLUDE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/third_party/marl/include) set(SUBZERO_DIR ${CMAKE_CURRENT_SOURCE_DIR}/third_party/subzero) set(SUBZERO_LLVM_DIR ${CMAKE_CURRENT_SOURCE_DIR}/third_party/llvm-subzero) set(TESTS_DIR ${CMAKE_CURRENT_SOURCE_DIR}/tests) @@ -1599,7 +1597,6 @@ set(VULKAN_INCLUDE_DIR ${COMMON_INCLUDE_DIR} - ${MARL_INCLUDE_DIR} ) ########################################################### @@ -1763,6 +1760,22 @@ ${CMAKE_CURRENT_SOURCE_DIR}/include/vulkan/*.h} ) +file(GLOB YARN_FULL_LIST + ${YARN_DIR}/*.cpp + ${YARN_DIR}/*.hpp + ${YARN_DIR}/*.c +) + +if (NOT MSVC) + file(GLOB YARN_ASSEMBLY_LIST ${YARN_DIR}/*.S) + list(APPEND YARN_FULL_LIST ${YARN_ASSEMBLY_LIST}) +endif(NOT MSVC) + +set(YARN_LIST ${YARN_FULL_LIST}) +set(YARN_TEST_LIST ${YARN_FULL_LIST}) +list(FILTER YARN_LIST EXCLUDE REGEX ".*_test\\..*") +list(FILTER YARN_TEST_LIST INCLUDE REGEX ".*_test\\..*") + ########################################################### # Append OS specific files to lists ########################################################### @@ -1861,6 +1874,15 @@ # SwiftShader Targets ########################################################### +add_library(Yarn STATIC ${YARN_LIST}) +set_target_properties(Yarn PROPERTIES + POSITION_INDEPENDENT_CODE 1 + FOLDER "Core" + COMPILE_OPTIONS "${SWIFTSHADER_COMPILE_OPTIONS}" + COMPILE_DEFINITIONS "NO_SANITIZE_FUNCTION=;" +) +target_link_libraries(Yarn ${OS_LIBS}) + add_library(SwiftShader STATIC ${SWIFTSHADER_LIST}) set_target_properties(SwiftShader PROPERTIES INCLUDE_DIRECTORIES "${COMMON_INCLUDE_DIR}" @@ -2008,11 +2030,6 @@ ) endif() -if(BUILD_MARL) - set(MARL_BUILD_TESTS OFF) - add_subdirectory(third_party/marl) -endif(BUILD_MARL) - if(BUILD_VULKAN) if (NOT TARGET SPIRV-Tools) # This variable is also used by SPIRV-Tools to locate SPIRV-Headers @@ -2049,7 +2066,7 @@ endif() set_shared_library_export_map(vk_swiftshader ${SOURCE_DIR}/Vulkan) - target_link_libraries(vk_swiftshader ${Reactor} marl ${OS_LIBS} SPIRV-Tools SPIRV-Tools-opt) + target_link_libraries(vk_swiftshader ${Reactor} Yarn ${OS_LIBS} SPIRV-Tools SPIRV-Tools-opt) add_custom_command( TARGET vk_swiftshader POST_BUILD @@ -2170,6 +2187,29 @@ endif(BUILD_TESTS) if(BUILD_TESTS) + # Yarn unit tests + file(GLOB YARN_TEST_LIST + ${YARN_DIR}/*_test.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/third_party/googletest/googletest/src/gtest-all.cc + ) + + set(YARN_TEST_INCLUDE_DIR + ${CMAKE_CURRENT_SOURCE_DIR}/third_party/googletest/googletest/include/ + ${CMAKE_CURRENT_SOURCE_DIR}/third_party/googletest/googlemock/include/ + ${CMAKE_CURRENT_SOURCE_DIR}/third_party/googletest/googletest/ + ${CMAKE_CURRENT_SOURCE_DIR}/src + ) + + add_executable(yarn-unittests ${YARN_TEST_LIST}) + set_target_properties(yarn-unittests PROPERTIES + INCLUDE_DIRECTORIES "${YARN_TEST_INCLUDE_DIR}" + FOLDER "Tests" + COMPILE_OPTIONS "${SWIFTSHADER_COMPILE_OPTIONS}" + COMPILE_DEFINITIONS "STANDALONE" + ) + + target_link_libraries(yarn-unittests Yarn ${OS_LIBS}) + # Math unit tests set(MATH_UNITTESTS_LIST ${CMAKE_CURRENT_SOURCE_DIR}/tests/MathUnitTests/main.cpp
diff --git a/src/Android.bp b/src/Android.bp index b74589b..88d2a6f 100644 --- a/src/Android.bp +++ b/src/Android.bp
@@ -492,6 +492,31 @@ static_libs: [ "libswiftshader_compiler_debug" ], } +// Yarn, fibers + scheduler library + +cc_library_static { + name: "libyarn", + vendor: true, + + srcs: [ + "Yarn/Debug.cpp", + "Yarn/Scheduler.cpp", + "Yarn/Thread.cpp", + "Yarn/Trace.cpp", + ], + + arch: { + arm: { srcs: ["Yarn/OSFiber_arm.c", "Yarn/OSFiber_asm_arm.S"], }, + arm64: { srcs: ["Yarn/OSFiber_aarch64.c", "Yarn/OSFiber_asm_aarch64.S"], }, + x86: { srcs: ["Yarn/OSFiber_x86.c", "Yarn/OSFiber_asm_x86.S"], }, + x86_64: { srcs: ["Yarn/OSFiber_x64.c", "Yarn/OSFiber_asm_x64.S"], }, + }, + + cflags: [ + "-Wno-unused-parameter", + ], +} + // Vulkan @@ -567,7 +592,6 @@ ], static_libs: [ - "swiftshader_marl", "swiftshader_spirv-tools", ], @@ -583,6 +607,7 @@ static_libs: [ "libswiftshadervk_llvm_debug", "libLLVM7_swiftshader", + "libyarn", ], defaults: [ @@ -597,6 +622,7 @@ static_libs: [ "libswiftshadervk_llvm_debug", "libLLVM7_swiftshader", + "libyarn", ], cflags: [
diff --git a/src/Device/BUILD.gn b/src/Device/BUILD.gn index c412c71..9d99aac 100644 --- a/src/Device/BUILD.gn +++ b/src/Device/BUILD.gn
@@ -61,10 +61,10 @@ ] deps = [ - "../../third_party/marl:Marl_headers", "../Pipeline:Pipeline_headers", "../System:System_headers", "../Vulkan:swiftshader_libvulkan_headers", + "../Yarn:Yarn_headers", ] public_deps = [
diff --git a/src/Device/Renderer.cpp b/src/Device/Renderer.cpp index a7e5042..51b03f1 100644 --- a/src/Device/Renderer.cpp +++ b/src/Device/Renderer.cpp
@@ -32,9 +32,9 @@ #include "Pipeline/SpirvShader.hpp" #include "Vertex.hpp" -#include "marl/containers.h" -#include "marl/defer.h" -#include "marl/trace.h" +#include "Yarn/Containers.hpp" +#include "Yarn/Defer.hpp" +#include "Yarn/Trace.hpp" #undef max @@ -166,7 +166,7 @@ if(count == 0) { return; } auto id = nextDrawID++; - MARL_SCOPED_EVENT("draw %d", id); + YARN_SCOPED_EVENT("draw %d", id); #ifndef NDEBUG { @@ -186,16 +186,16 @@ return; } - marl::Pool<sw::DrawCall>::Loan draw; + yarn::Pool<sw::DrawCall>::Loan draw; { - MARL_SCOPED_EVENT("drawCallPool.borrow()"); + YARN_SCOPED_EVENT("drawCallPool.borrow()"); draw = drawCallPool.borrow(); } draw->id = id; if(update) { - MARL_SCOPED_EVENT("update"); + YARN_SCOPED_EVENT("update"); vertexState = VertexProcessor::update(context); setupState = SetupProcessor::update(context); pixelState = PixelProcessor::update(context); @@ -416,7 +416,7 @@ pixelRoutine.reset(); } - void DrawCall::run(const marl::Loan<DrawCall>& draw, marl::Ticket::Queue* tickets, marl::Ticket::Queue clusterQueues[MaxClusterCount]) + void DrawCall::run(const yarn::Loan<DrawCall>& draw, yarn::Ticket::Queue* tickets, yarn::Ticket::Queue clusterQueues[MaxClusterCount]) { draw->setup(); @@ -425,8 +425,8 @@ auto const numBatches = draw->numBatches; auto ticket = tickets->take(); - auto finally = marl::make_shared_finally([draw, ticket] { - MARL_SCOPED_EVENT("FINISH draw %d", draw->id); + auto finally = yarn::make_shared_finally([draw, ticket] { + YARN_SCOPED_EVENT("FINISH draw %d", draw->id); draw->teardown(); ticket.done(); }); @@ -443,7 +443,7 @@ batch->clusterTickets[cluster] = std::move(clusterQueues[cluster].take()); } - marl::schedule([draw, batch, finally] { + yarn::schedule([draw, batch, finally] { processVertices(draw.get(), batch.get()); @@ -468,11 +468,11 @@ void DrawCall::processVertices(DrawCall* draw, BatchData* batch) { - MARL_SCOPED_EVENT("VERTEX draw %d, batch %d", draw->id, batch->id); + YARN_SCOPED_EVENT("VERTEX draw %d, batch %d", draw->id, batch->id); unsigned int triangleIndices[MaxBatchSize + 1][3]; // One extra for SIMD width overrun. TODO: Adjust to dynamic batch size. { - MARL_SCOPED_EVENT("processPrimitiveVertices"); + YARN_SCOPED_EVENT("processPrimitiveVertices"); processPrimitiveVertices( triangleIndices, draw->data->indices, @@ -496,21 +496,21 @@ void DrawCall::processPrimitives(DrawCall* draw, BatchData* batch) { - MARL_SCOPED_EVENT("PRIMITIVES draw %d batch %d", draw->id, batch->id); + YARN_SCOPED_EVENT("PRIMITIVES draw %d batch %d", draw->id, batch->id); auto triangles = &batch->triangles[0]; auto primitives = &batch->primitives[0]; batch->numVisible = draw->setupPrimitives(triangles, primitives, draw, batch->numPrimitives); } - void DrawCall::processPixels(const marl::Loan<DrawCall>& draw, const marl::Loan<BatchData>& batch, const std::shared_ptr<marl::Finally>& finally) + void DrawCall::processPixels(const yarn::Loan<DrawCall>& draw, const yarn::Loan<BatchData>& batch, const std::shared_ptr<yarn::Finally>& finally) { struct Data { - Data(const marl::Loan<DrawCall>& draw, const marl::Loan<BatchData>& batch, const std::shared_ptr<marl::Finally>& finally) + Data(const yarn::Loan<DrawCall>& draw, const yarn::Loan<BatchData>& batch, const std::shared_ptr<yarn::Finally>& finally) : draw(draw), batch(batch), finally(finally) {} - marl::Loan<DrawCall> draw; - marl::Loan<BatchData> batch; - std::shared_ptr<marl::Finally> finally; + yarn::Loan<DrawCall> draw; + yarn::Loan<BatchData> batch; + std::shared_ptr<yarn::Finally> finally; }; auto data = std::make_shared<Data>(draw, batch, finally); for (int cluster = 0; cluster < MaxClusterCount; cluster++) @@ -519,7 +519,7 @@ { auto& draw = data->draw; auto& batch = data->batch; - MARL_SCOPED_EVENT("PIXEL draw %d, batch %d, cluster %d", draw->id, batch->id, cluster); + YARN_SCOPED_EVENT("PIXEL draw %d, batch %d, cluster %d", draw->id, batch->id, cluster); draw->pixelPointer(&batch->primitives.front(), batch->numVisible, cluster, MaxClusterCount, draw->data); batch->clusterTickets[cluster].done(); }); @@ -528,7 +528,7 @@ void Renderer::synchronize() { - MARL_SCOPED_EVENT("synchronize"); + YARN_SCOPED_EVENT("synchronize"); auto ticket = drawTickets.take(); ticket.wait(); device->updateSamplingRoutineConstCache();
diff --git a/src/Device/Renderer.hpp b/src/Device/Renderer.hpp index 042f932..4b96471 100644 --- a/src/Device/Renderer.hpp +++ b/src/Device/Renderer.hpp
@@ -24,9 +24,9 @@ #include "Device/Config.hpp" #include "Vulkan/VkDescriptorSet.hpp" -#include "marl/pool.h" -#include "marl/finally.h" -#include "marl/ticket.h" +#include "Yarn/Pool.hpp" +#include "Yarn/Finally.hpp" +#include "Yarn/Ticket.hpp" #include <atomic> #include <list> @@ -116,7 +116,7 @@ { struct BatchData { - using Pool = marl::BoundedPool<BatchData, MaxBatchCount, marl::PoolPolicy::Preserve>; + using Pool = yarn::BoundedPool<BatchData, MaxBatchCount, yarn::PoolPolicy::Preserve>; TriangleBatch triangles; PrimitiveBatch primitives; @@ -125,19 +125,19 @@ unsigned int firstPrimitive; unsigned int numPrimitives; int numVisible; - marl::Ticket clusterTickets[MaxClusterCount]; + yarn::Ticket clusterTickets[MaxClusterCount]; }; - using Pool = marl::BoundedPool<DrawCall, MaxDrawCount, marl::PoolPolicy::Preserve>; + using Pool = yarn::BoundedPool<DrawCall, MaxDrawCount, yarn::PoolPolicy::Preserve>; using SetupFunction = int(*)(Triangle *triangles, Primitive *primitives, const DrawCall *drawCall, int count); DrawCall(); ~DrawCall(); - static void run(const marl::Loan<DrawCall>& draw, marl::Ticket::Queue* tickets, marl::Ticket::Queue clusterQueues[MaxClusterCount]); + static void run(const yarn::Loan<DrawCall>& draw, yarn::Ticket::Queue* tickets, yarn::Ticket::Queue clusterQueues[MaxClusterCount]); static void processVertices(DrawCall* draw, BatchData* batch); static void processPrimitives(DrawCall* draw, BatchData* batch); - static void processPixels(const marl::Loan<DrawCall>& draw, const marl::Loan<BatchData>& batch, const std::shared_ptr<marl::Finally>& finally); + static void processPixels(const yarn::Loan<DrawCall>& draw, const yarn::Loan<BatchData>& batch, const std::shared_ptr<yarn::Finally>& finally); void setup(); void teardown(); @@ -223,8 +223,8 @@ std::atomic<int> nextDrawID = {0}; vk::Query *occlusionQuery = nullptr; - marl::Ticket::Queue drawTickets; - marl::Ticket::Queue clusterQueues[MaxClusterCount]; + yarn::Ticket::Queue drawTickets; + yarn::Ticket::Queue clusterQueues[MaxClusterCount]; VertexProcessor::State vertexState; SetupProcessor::State setupState;
diff --git a/src/Pipeline/BUILD.gn b/src/Pipeline/BUILD.gn index e17084a..72c6a3d 100644 --- a/src/Pipeline/BUILD.gn +++ b/src/Pipeline/BUILD.gn
@@ -52,10 +52,10 @@ ] deps = [ - "../../third_party/marl:Marl_headers", "../Device:Device_headers", "../System:System_headers", "../Vulkan:swiftshader_libvulkan_headers", + "../Yarn:Yarn_headers", ] public_deps = [
diff --git a/src/Pipeline/ComputeProgram.cpp b/src/Pipeline/ComputeProgram.cpp index d0f57be..2a86cc4 100644 --- a/src/Pipeline/ComputeProgram.cpp +++ b/src/Pipeline/ComputeProgram.cpp
@@ -18,9 +18,9 @@ #include "Vulkan/VkDebug.hpp" #include "Vulkan/VkPipelineLayout.hpp" -#include "marl/defer.h" -#include "marl/trace.h" -#include "marl/waitgroup.h" +#include "Yarn/Defer.hpp" +#include "Yarn/Trace.hpp" +#include "Yarn/WaitGroup.hpp" #include <queue> @@ -44,7 +44,7 @@ void ComputeProgram::generate() { - MARL_SCOPED_EVENT("ComputeProgram::generate"); + YARN_SCOPED_EVENT("ComputeProgram::generate"); SpirvRoutine routine(pipelineLayout); shader->emitProlog(&routine); @@ -222,7 +222,7 @@ data.pushConstants = pushConstants; data.constants = &sw::constants; - marl::WaitGroup wg; + yarn::WaitGroup wg; const uint32_t batchCount = 16; auto groupCount = groupCountX * groupCountY * groupCountZ; @@ -230,7 +230,7 @@ for (uint32_t batchID = 0; batchID < batchCount && batchID < groupCount; batchID++) { wg.add(1); - marl::schedule([=, &data] + yarn::schedule([=, &data] { defer(wg.done()); std::vector<uint8_t> workgroupMemory(shader->workgroupMemory.size()); @@ -247,7 +247,7 @@ auto groupZ = baseGroupZ + groupOffsetZ; auto groupY = baseGroupY + groupOffsetY; auto groupX = baseGroupX + groupOffsetX; - MARL_SCOPED_EVENT("groupX: %d, groupY: %d, groupZ: %d", groupX, groupY, groupZ); + YARN_SCOPED_EVENT("groupX: %d, groupY: %d, groupZ: %d", groupX, groupY, groupZ); using Coroutine = std::unique_ptr<rr::Stream<SpirvShader::YieldResult>>; std::queue<Coroutine> coroutines;
diff --git a/src/Vulkan/BUILD.gn b/src/Vulkan/BUILD.gn index 3e33cd7..673bf8d 100644 --- a/src/Vulkan/BUILD.gn +++ b/src/Vulkan/BUILD.gn
@@ -148,13 +148,13 @@ } deps = [ - "../../third_party/marl:Marl", "${swiftshader_spirv_tools_dir}:SPIRV-Tools", "../Device", "../Pipeline", "../Reactor:swiftshader_llvm_reactor", "../System", "../WSI", + "../Yarn", ] include_dirs = [
diff --git a/src/Vulkan/VkCommandBuffer.cpp b/src/Vulkan/VkCommandBuffer.cpp index c6cbcbc..34e17b0 100644 --- a/src/Vulkan/VkCommandBuffer.cpp +++ b/src/Vulkan/VkCommandBuffer.cpp
@@ -1108,7 +1108,7 @@ // Everything else is deferred to the Renderer; we will treat those stages all as if they were // `bottom of pipe`. // - // FIXME(chrisforbes): once Marl is integrated, do this in a task so we don't have to stall here. + // FIXME(chrisforbes): once Yarn is integrated, do this in a task so we don't have to stall here. executionState.renderer->synchronize(); }
diff --git a/src/Vulkan/VkDevice.cpp b/src/Vulkan/VkDevice.cpp index b817b31..4ef1ee5 100644 --- a/src/Vulkan/VkDevice.cpp +++ b/src/Vulkan/VkDevice.cpp
@@ -57,7 +57,7 @@ cache.updateConstCache(); } -Device::Device(const VkDeviceCreateInfo* pCreateInfo, void* mem, PhysicalDevice *physicalDevice, const VkPhysicalDeviceFeatures *enabledFeatures, marl::Scheduler *scheduler) +Device::Device(const VkDeviceCreateInfo* pCreateInfo, void* mem, PhysicalDevice *physicalDevice, const VkPhysicalDeviceFeatures *enabledFeatures, yarn::Scheduler *scheduler) : physicalDevice(physicalDevice), queues(reinterpret_cast<Queue*>(mem)), enabledExtensionCount(pCreateInfo->enabledExtensionCount),
diff --git a/src/Vulkan/VkDevice.hpp b/src/Vulkan/VkDevice.hpp index 83ce1e9..24ed2c2 100644 --- a/src/Vulkan/VkDevice.hpp +++ b/src/Vulkan/VkDevice.hpp
@@ -21,7 +21,7 @@ #include <memory> #include <mutex> -namespace marl +namespace yarn { class Scheduler; } @@ -42,7 +42,7 @@ public: static constexpr VkSystemAllocationScope GetAllocationScope() { return VK_SYSTEM_ALLOCATION_SCOPE_DEVICE; } - Device(const VkDeviceCreateInfo* pCreateInfo, void* mem, PhysicalDevice *physicalDevice, const VkPhysicalDeviceFeatures *enabledFeatures, marl::Scheduler *scheduler); + Device(const VkDeviceCreateInfo* pCreateInfo, void* mem, PhysicalDevice *physicalDevice, const VkPhysicalDeviceFeatures *enabledFeatures, yarn::Scheduler *scheduler); void destroy(const VkAllocationCallbacks* pAllocator); static size_t ComputeRequiredAllocationSize(const VkDeviceCreateInfo* pCreateInfo);
diff --git a/src/Vulkan/VkPipeline.cpp b/src/Vulkan/VkPipeline.cpp index 86ea342..c5b72de 100644 --- a/src/Vulkan/VkPipeline.cpp +++ b/src/Vulkan/VkPipeline.cpp
@@ -22,7 +22,7 @@ #include "Pipeline/ComputeProgram.hpp" #include "Pipeline/SpirvShader.hpp" -#include "marl/trace.h" +#include "Yarn/Trace.hpp" #include "spirv-tools/optimizer.hpp" @@ -259,7 +259,7 @@ std::shared_ptr<sw::ComputeProgram> createProgram(const vk::PipelineCache::ComputeProgramKey& key) { - MARL_SCOPED_EVENT("createProgram"); + YARN_SCOPED_EVENT("createProgram"); vk::DescriptorSet::Bindings descriptorSets; // FIXME(b/129523279): Delay code generation until invoke time. // TODO(b/119409619): use allocator.
diff --git a/src/Vulkan/VkQueue.cpp b/src/Vulkan/VkQueue.cpp index 8e41bbb..329847d 100644 --- a/src/Vulkan/VkQueue.cpp +++ b/src/Vulkan/VkQueue.cpp
@@ -19,10 +19,10 @@ #include "WSI/VkSwapchainKHR.hpp" #include "Device/Renderer.hpp" -#include "marl/defer.h" -#include "marl/scheduler.h" -#include "marl/thread.h" -#include "marl/trace.h" +#include "Yarn/Defer.hpp" +#include "Yarn/Scheduler.hpp" +#include "Yarn/Thread.hpp" +#include "Yarn/Trace.hpp" #include <cstring> @@ -79,7 +79,7 @@ namespace vk { -Queue::Queue(Device* device, marl::Scheduler *scheduler) : device(device) +Queue::Queue(Device* device, yarn::Scheduler *scheduler) : device(device) { queueThread = std::thread(&Queue::taskLoop, this, scheduler); } @@ -160,9 +160,9 @@ } } -void Queue::taskLoop(marl::Scheduler* scheduler) +void Queue::taskLoop(yarn::Scheduler* scheduler) { - marl::Thread::setName("Queue<%p>", this); + yarn::Thread::setName("Queue<%p>", this); scheduler->bind(); defer(scheduler->unbind());
diff --git a/src/Vulkan/VkQueue.hpp b/src/Vulkan/VkQueue.hpp index 0bf3d00..c137d15 100644 --- a/src/Vulkan/VkQueue.hpp +++ b/src/Vulkan/VkQueue.hpp
@@ -22,7 +22,7 @@ #include "System/Synchronization.hpp" -namespace marl +namespace yarn { class Scheduler; } @@ -44,7 +44,7 @@ VK_LOADER_DATA loaderData = { ICD_LOADER_MAGIC }; public: - Queue(Device* device, marl::Scheduler *scheduler); + Queue(Device* device, yarn::Scheduler *scheduler); ~Queue(); operator VkQueue() @@ -69,7 +69,7 @@ Type type = SUBMIT_QUEUE; }; - void taskLoop(marl::Scheduler* scheduler); + void taskLoop(yarn::Scheduler* scheduler); void garbageCollect(); void submitQueue(const Task& task);
diff --git a/src/Vulkan/libVulkan.cpp b/src/Vulkan/libVulkan.cpp index c300382..aa4e68c 100644 --- a/src/Vulkan/libVulkan.cpp +++ b/src/Vulkan/libVulkan.cpp
@@ -68,8 +68,8 @@ #include "Reactor/Nucleus.hpp" -#include "marl/scheduler.h" -#include "marl/thread.h" +#include "Yarn/Scheduler.hpp" +#include "Yarn/Thread.hpp" #include "System/CPUID.hpp" @@ -120,14 +120,14 @@ sw::CPUID::setEnableSSE(true); } -marl::Scheduler* getOrCreateScheduler() +yarn::Scheduler* getOrCreateScheduler() { - static auto scheduler = std::unique_ptr<marl::Scheduler>(new marl::Scheduler()); + static auto scheduler = std::unique_ptr<yarn::Scheduler>(new yarn::Scheduler()); scheduler->setThreadInitializer([] { sw::CPUID::setFlushToZero(true); sw::CPUID::setDenormalsAreZero(true); }); - scheduler->setWorkerThreadCount(std::min<size_t>(marl::Thread::numLogicalCPUs(), 16)); + scheduler->setWorkerThreadCount(std::min<size_t>(yarn::Thread::numLogicalCPUs(), 16)); return scheduler.get(); }
diff --git a/src/WSI/BUILD.gn b/src/WSI/BUILD.gn index 2ea3ad2..c13463e 100644 --- a/src/WSI/BUILD.gn +++ b/src/WSI/BUILD.gn
@@ -45,7 +45,6 @@ ] deps = [ - "../../third_party/marl:Marl_headers", "../Vulkan:swiftshader_libvulkan_headers", ] }
diff --git a/src/Yarn/BUILD.gn b/src/Yarn/BUILD.gn new file mode 100644 index 0000000..2718ad1 --- /dev/null +++ b/src/Yarn/BUILD.gn
@@ -0,0 +1,110 @@ +# Copyright 2019 The SwiftShader Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import("../swiftshader.gni") + +import("//testing/test.gni") + +config("yarn_config") { + include_dirs = [ + "..", + ] +} + +swiftshader_source_set("Yarn_headers") { + sources = [ + "ConditionVariable.hpp", + "Containers.hpp", + "Debug.hpp", + "Defer.hpp", + "Finally.hpp", + "OSFiber.hpp", + "Pool.hpp", + "SAL.hpp", + "Scheduler.hpp", + "Thread.hpp", + "Ticket.hpp", + "Trace.hpp", + "Utils.hpp", + "WaitGroup.hpp", + ] + + if (is_win) { + sources += [ "OSFiber_windows.hpp" ] + } else { + sources += [ + "OSFiber_asm.hpp", + "OSFiber_asm_aarch64.h", + "OSFiber_asm_arm.h", + "OSFiber_asm_x64.h", + "OSFiber_asm_x86.h", + ] + } + + public_configs = [ + ":yarn_config", + ] +} + +swiftshader_source_set("Yarn") { + sources = [ + "Debug.cpp", + "Scheduler.cpp", + "Thread.cpp", + "Trace.cpp", + ] + + if (!is_win) { + sources += [ + "OSFiber_aarch64.c", + "OSFiber_arm.c", + "OSFiber_asm_aarch64.S", + "OSFiber_asm_arm.S", + "OSFiber_asm_x64.S", + "OSFiber_asm_x86.S", + "OSFiber_x64.c", + "OSFiber_x86.c", + ] + } + + public_deps = [ + ":Yarn_headers", + ] +} + +test("yarn_unittests") { + sources = [ + "ConditionVariable_test.cpp", + "Containers_test.cpp", + "Defer_test.cpp", + "OSFiber_test.cpp", + "Pool_test.cpp", + "Scheduler_test.cpp", + "Ticket_test.cpp", + "WaitGroup_test.cpp", + "Yarn_test.cpp", + ] + deps = [ + ":Yarn", + "//testing/gtest", + "//testing/gmock", + ] + + if (!swiftshader_standalone) { + deps += [ + "//third_party/googletest:gmock", + "//third_party/googletest:gtest", + ] + } +}
diff --git a/src/Yarn/BlockingCall.hpp b/src/Yarn/BlockingCall.hpp new file mode 100644 index 0000000..161bba9 --- /dev/null +++ b/src/Yarn/BlockingCall.hpp
@@ -0,0 +1,85 @@ +// Copyright 2019 The SwiftShader Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "WaitGroup.hpp" + +#include <thread> +#include <type_traits> + +namespace yarn { +namespace detail { + +template <typename RETURN_TYPE> +class OnNewThread +{ +public: + template <typename F, typename ... Args> + inline static RETURN_TYPE call(F&& f, Args&& ... args) + { + RETURN_TYPE result; + WaitGroup wg(1); + auto thread = std::thread([&] + { + defer(wg.done()); + result = f(args...); + }); + wg.wait(); + thread.join(); + return result; + } +}; + +template <> +class OnNewThread<void> +{ +public: + template <typename F, typename ... Args> + inline static void call(F&& f, Args&& ... args) + { + WaitGroup wg(1); + auto thread = std::thread([&] + { + defer(wg.done()); + f(args...); + }); + wg.wait(); + thread.join() + } +}; + +} // namespace detail + +// blocking_call() calls the function F on a new thread, yielding this fiber +// to execute other tasks until F has returned. +// +// Example: +// +// void runABlockingFunctionOnATask() +// { +// // Schedule a task that calls a blocking, non-yielding function. +// yarn::schedule([=] { +// // call_blocking_function() may block indefinitely. +// // Ensure this call does not block other tasks from running. +// auto result = yarn::blocking_call(call_blocking_function); +// // call_blocking_function() has now returned. +// // result holds the return value of the blocking function call. +// }); +// } +template <typename F, typename ... Args> +auto inline blocking_call(F&& f, Args&& ... args) -> decltype(f(args...)) +{ + return detail::OnNewThread<decltype(f(args...))>::call(std::forward<F>(f), std::forward<Args>(args)...); +} + +} // namespace yarn
diff --git a/src/Yarn/ConditionVariable.hpp b/src/Yarn/ConditionVariable.hpp new file mode 100644 index 0000000..7676c42 --- /dev/null +++ b/src/Yarn/ConditionVariable.hpp
@@ -0,0 +1,119 @@ +// Copyright 2019 The SwiftShader Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef yarn_condition_variable_hpp +#define yarn_condition_variable_hpp + +#include "Containers.hpp" +#include "Debug.hpp" +#include "Scheduler.hpp" + +#include <atomic> +#include <mutex> + +namespace yarn { + +// ConditionVariable is a synchronization primitive that can be used to block +// one or more fibers or threads, until another fiber or thread modifies a +// shared variable (the condition) and notifies the ConditionVariable. +// +// If the ConditionVariable is blocked on a thread with a Scheduler bound, the +// thread will work on other tasks until the ConditionVariable is unblocked. +class ConditionVariable +{ +public: + // Notifies and potentially unblocks one waiting fiber or thread. + inline void notify_one(); + + // Notifies and potentially unblocks all waiting fibers and/or threads. + inline void notify_all(); + + // Blocks the current fiber or thread until the predicate is satisfied + // and the ConditionVariable is notified. + template <typename Predicate> + inline void wait(std::unique_lock<std::mutex>& lock, Predicate pred); + +private: + std::mutex mutex; + containers::vector<Scheduler::Fiber*, 4> waiting; + std::condition_variable condition; + std::atomic<int> numWaiting = { 0 }; + std::atomic<int> numWaitingOnCondition = { 0 }; +}; + +void ConditionVariable::notify_one() +{ + if (numWaiting == 0) { return; } + std::unique_lock<std::mutex> lock(mutex); + if (waiting.size() > 0) + { + auto fiber = waiting.back(); + waiting.pop_back(); + fiber->schedule(); + } + lock.unlock(); + if (numWaitingOnCondition > 0) { condition.notify_one(); } +} + +void ConditionVariable::notify_all() +{ + if (numWaiting == 0) { return; } + std::unique_lock<std::mutex> lock(mutex); + while (waiting.size() > 0) + { + auto fiber = waiting.back(); + waiting.pop_back(); + fiber->schedule(); + } + lock.unlock(); + if (numWaitingOnCondition > 0) { condition.notify_all(); } +} + +template <typename Predicate> +void ConditionVariable::wait(std::unique_lock<std::mutex>& dataLock, Predicate pred) +{ + if (pred()) + { + return; + } + numWaiting++; + if (auto fiber = Scheduler::Fiber::current()) + { + // Currently executing on a scheduler fiber. + // Yield to let other tasks run that can unblock this fiber. + while (!pred()) + { + mutex.lock(); + waiting.push_back(fiber); + mutex.unlock(); + + dataLock.unlock(); + fiber->yield(); + dataLock.lock(); + } + } + else + { + // Currently running outside of the scheduler. + // Delegate to the std::condition_variable. + numWaitingOnCondition++; + condition.wait(dataLock, pred); + numWaitingOnCondition--; + } + numWaiting--; +} + +} // namespace yarn + +#endif // yarn_condition_variable_hpp
diff --git a/src/Yarn/ConditionVariable_test.cpp b/src/Yarn/ConditionVariable_test.cpp new file mode 100644 index 0000000..a7bc327 --- /dev/null +++ b/src/Yarn/ConditionVariable_test.cpp
@@ -0,0 +1,96 @@ +// Copyright 2019 The SwiftShader Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "ConditionVariable.hpp" + +#include "Yarn_test.hpp" + +TEST(WithoutBoundScheduler, ConditionVariable) +{ + bool trigger[3] = {false, false, false}; + bool signal[3] = {false, false, false}; + std::mutex mutex; + yarn::ConditionVariable cv; + + std::thread thread([&] + { + for (int i = 0; i < 3; i++) + { + std::unique_lock<std::mutex> lock(mutex); + cv.wait(lock, [&] { return trigger[i]; }); + signal[i] = true; + cv.notify_one(); + } + }); + + ASSERT_FALSE(signal[0]); + ASSERT_FALSE(signal[1]); + ASSERT_FALSE(signal[2]); + + for (int i = 0; i < 3; i++) + { + { + std::unique_lock<std::mutex> lock(mutex); + trigger[i] = true; + cv.notify_one(); + cv.wait(lock, [&] { return signal[i]; }); + } + + ASSERT_EQ(signal[0], 0 <= i); + ASSERT_EQ(signal[1], 1 <= i); + ASSERT_EQ(signal[2], 2 <= i); + } + + thread.join(); +} + + +TEST_P(WithBoundScheduler, ConditionVariable) +{ + bool trigger[3] = {false, false, false}; + bool signal[3] = {false, false, false}; + std::mutex mutex; + yarn::ConditionVariable cv; + + std::thread thread([&] + { + for (int i = 0; i < 3; i++) + { + std::unique_lock<std::mutex> lock(mutex); + cv.wait(lock, [&] { return trigger[i]; }); + signal[i] = true; + cv.notify_one(); + } + }); + + ASSERT_FALSE(signal[0]); + ASSERT_FALSE(signal[1]); + ASSERT_FALSE(signal[2]); + + for (int i = 0; i < 3; i++) + { + { + std::unique_lock<std::mutex> lock(mutex); + trigger[i] = true; + cv.notify_one(); + cv.wait(lock, [&] { return signal[i]; }); + } + + ASSERT_EQ(signal[0], 0 <= i); + ASSERT_EQ(signal[1], 1 <= i); + ASSERT_EQ(signal[2], 2 <= i); + } + + thread.join(); +}
diff --git a/src/Yarn/Containers.hpp b/src/Yarn/Containers.hpp new file mode 100644 index 0000000..b8b66a8 --- /dev/null +++ b/src/Yarn/Containers.hpp
@@ -0,0 +1,255 @@ +// Copyright 2019 The SwiftShader Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// The yarn::containers namespace holds STL-like container implementations that +// are optimized for avoiding heap allocations. +// Note that unlike many other types in yarn, these containers offer no +// protection for concurrent access and are considered 'thread-unsafe'. + +#ifndef yarn_containers_hpp +#define yarn_containers_hpp + +#include "Debug.hpp" + +#include <algorithm> // std::max +#include <type_traits> // std::aligned_storage +#include <utility> // std::move + +#include <stddef.h> // size_t + +namespace yarn { +namespace containers { + +//////////////////////////////////////////////////////////////////////////////// +// vector<T, BASE_CAPACITY> +//////////////////////////////////////////////////////////////////////////////// + +// vector is a container of contiguously stored elements. +// Unlike std::vector, yarn::containers::vector keeps the first BASE_CAPACITY +// elements internally, which will avoid dynamic heap allocations. +// Once the vector exceeds BASE_CAPACITY elements, vector will allocate storage +// from the heap. +template <typename T, int BASE_CAPACITY> +class vector +{ +public: + inline vector() = default; + + template <int BASE_CAPACITY_2> + inline vector(const vector<T, BASE_CAPACITY_2>& other); + + template <int BASE_CAPACITY_2> + inline vector(vector<T, BASE_CAPACITY_2>&& other); + + inline ~vector(); + + template <int BASE_CAPACITY_2> + inline vector<T, BASE_CAPACITY>& operator = (const vector<T, BASE_CAPACITY_2>&); + + template <int BASE_CAPACITY_2> + inline vector<T, BASE_CAPACITY>& operator = (vector<T, BASE_CAPACITY_2>&&); + + inline void push_back(const T& el); + inline void emplace_back(T&& el); + inline void pop_back(); + inline T& front(); + inline T& back(); + inline T* begin(); + inline T* end(); + inline T& operator[] (size_t i); + inline const T& operator[] (size_t i) const; + inline size_t size() const; + inline size_t cap() const; + inline void resize(size_t n); + inline void reserve(size_t n); + +private: + using TStorage = typename std::aligned_storage<sizeof(T), alignof(T)>::type; + + inline void free(); + + size_t count = 0; + size_t capacity = BASE_CAPACITY; + TStorage buffer[BASE_CAPACITY]; + TStorage* elements = buffer; +}; + +template <typename T, int BASE_CAPACITY> +template <int BASE_CAPACITY_2> +vector<T, BASE_CAPACITY>::vector(const vector<T, BASE_CAPACITY_2>& other) +{ + *this = other; +} + +template <typename T, int BASE_CAPACITY> +template <int BASE_CAPACITY_2> +vector<T, BASE_CAPACITY>::vector(vector<T, BASE_CAPACITY_2>&& other) +{ + *this = std::move(other); +} + +template <typename T, int BASE_CAPACITY> +vector<T, BASE_CAPACITY>::~vector() +{ + free(); +} + +template <typename T, int BASE_CAPACITY> +template <int BASE_CAPACITY_2> +vector<T, BASE_CAPACITY>& vector<T, BASE_CAPACITY>::operator = (const vector<T, BASE_CAPACITY_2>& other) +{ + free(); + reserve(other.size()); + count = other.size(); + for (size_t i = 0; i < count; i++) + { + new (&reinterpret_cast<T*>(elements)[i]) T(other[i]); + } + return *this; +} + +template <typename T, int BASE_CAPACITY> +template <int BASE_CAPACITY_2> +vector<T, BASE_CAPACITY>& vector<T, BASE_CAPACITY>::operator = (vector<T, BASE_CAPACITY_2>&& other) +{ + free(); + reserve(other.size()); + count = other.size(); + for (size_t i = 0; i < count; i++) + { + new (&reinterpret_cast<T*>(elements)[i]) T(std::move(other[i])); + } + other.resize(0); + return *this; +} + +template <typename T, int BASE_CAPACITY> +void vector<T, BASE_CAPACITY>::push_back(const T& el) +{ + reserve(count + 1); + new (&reinterpret_cast<T*>(elements)[count]) T(el); + count++; +} + +template <typename T, int BASE_CAPACITY> +void vector<T, BASE_CAPACITY>::emplace_back(T&& el) +{ + reserve(count + 1); + new (&reinterpret_cast<T*>(elements)[count]) T(std::move(el)); + count++; +} + +template <typename T, int BASE_CAPACITY> +void vector<T, BASE_CAPACITY>::pop_back() +{ + YARN_ASSERT(count > 0, "pop_back() called on empty vector"); + count--; + reinterpret_cast<T*>(elements)[count].~T(); +} + +template <typename T, int BASE_CAPACITY> +T& vector<T, BASE_CAPACITY>::front() +{ + YARN_ASSERT(count > 0, "front() called on empty vector"); + return reinterpret_cast<T*>(elements)[0]; +} + +template <typename T, int BASE_CAPACITY> +T& vector<T, BASE_CAPACITY>::back() +{ + YARN_ASSERT(count > 0, "back() called on empty vector"); + return reinterpret_cast<T*>(elements)[count - 1]; +} + +template <typename T, int BASE_CAPACITY> +T* vector<T, BASE_CAPACITY>::begin() +{ + return reinterpret_cast<T*>(elements); +} + +template <typename T, int BASE_CAPACITY> +T* vector<T, BASE_CAPACITY>::end() +{ + return reinterpret_cast<T*>(elements) + count; +} + +template <typename T, int BASE_CAPACITY> +T& vector<T, BASE_CAPACITY>::operator[] (size_t i) +{ + YARN_ASSERT(i < count, "index %d exceeds vector size %d", int(i), int(count)); + return reinterpret_cast<T*>(elements)[i]; +} + +template <typename T, int BASE_CAPACITY> +const T& vector<T, BASE_CAPACITY>::operator[] (size_t i) const +{ + YARN_ASSERT(i < count, "index %d exceeds vector size %d", int(i), int(count)); + return reinterpret_cast<T*>(elements)[i]; +} + +template <typename T, int BASE_CAPACITY> +size_t vector<T, BASE_CAPACITY>::size() const +{ + return count; +} + +template <typename T, int BASE_CAPACITY> +void vector<T, BASE_CAPACITY>::resize(size_t n) +{ + reserve(n); + while (count < n) + { + new (&reinterpret_cast<T*>(elements)[count++]) T(); + } + while (n < count) + { + reinterpret_cast<T*>(elements)[--count].~T(); + } +} + +template <typename T, int BASE_CAPACITY> +void vector<T, BASE_CAPACITY>::reserve(size_t n) +{ + if (n > capacity) + { + capacity = std::max<size_t>(n * 2, 8); + auto grown = new TStorage[capacity]; + for (size_t i = 0; i < count; i++) + { + new (&reinterpret_cast<T*>(grown)[i]) T(std::move(reinterpret_cast<T*>(elements)[i])); + } + free(); + elements = grown; + } +} + +template <typename T, int BASE_CAPACITY> +void vector<T, BASE_CAPACITY>::free() +{ + for (size_t i = 0; i < count; i++) + { + reinterpret_cast<T*>(elements)[i].~T(); + } + + if (elements != buffer) + { + delete []elements; + elements = nullptr; + } +} + +} // namespace containers +} // namespace yarn + +#endif // yarn_containers_hpp
diff --git a/src/Yarn/Containers_test.cpp b/src/Yarn/Containers_test.cpp new file mode 100644 index 0000000..d677606 --- /dev/null +++ b/src/Yarn/Containers_test.cpp
@@ -0,0 +1,201 @@ +// Copyright 2019 The SwiftShader Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "Containers.hpp" + +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +#include <string> + +class ContainersVectorTest : public testing::Test {}; + +TEST(ContainersVectorTest, Empty) +{ + yarn::containers::vector<std::string, 4> vector; + ASSERT_EQ(vector.size(), size_t(0)); +} + +TEST(ContainersVectorTest, WithinFixedCapIndex) +{ + yarn::containers::vector<std::string, 4> vector; + vector.resize(4); + vector[0] = "A"; + vector[1] = "B"; + vector[2] = "C"; + vector[3] = "D"; + + ASSERT_EQ(vector[0], "A"); + ASSERT_EQ(vector[1], "B"); + ASSERT_EQ(vector[2], "C"); + ASSERT_EQ(vector[3], "D"); +} + +TEST(ContainersVectorTest, BeyondFixedCapIndex) +{ + yarn::containers::vector<std::string, 1> vector; + vector.resize(4); + vector[0] = "A"; + vector[1] = "B"; + vector[2] = "C"; + vector[3] = "D"; + + ASSERT_EQ(vector[0], "A"); + ASSERT_EQ(vector[1], "B"); + ASSERT_EQ(vector[2], "C"); + ASSERT_EQ(vector[3], "D"); +} + +TEST(ContainersVectorTest, WithinFixedCapPushPop) +{ + yarn::containers::vector<std::string, 4> vector; + vector.push_back("A"); + vector.push_back("B"); + vector.push_back("C"); + vector.push_back("D"); + + ASSERT_EQ(vector.size(), size_t(4)); + ASSERT_EQ(vector.end() - vector.begin(), ptrdiff_t(4)); + + ASSERT_EQ(vector.front(), "A"); + ASSERT_EQ(vector.back(), "D"); + vector.pop_back(); + ASSERT_EQ(vector.size(), size_t(3)); + ASSERT_EQ(vector.end() - vector.begin(), ptrdiff_t(3)); + + ASSERT_EQ(vector.front(), "A"); + ASSERT_EQ(vector.back(), "C"); + vector.pop_back(); + ASSERT_EQ(vector.size(), size_t(2)); + ASSERT_EQ(vector.end() - vector.begin(), ptrdiff_t(2)); + + ASSERT_EQ(vector.front(), "A"); + ASSERT_EQ(vector.back(), "B"); + vector.pop_back(); + ASSERT_EQ(vector.size(), size_t(1)); + ASSERT_EQ(vector.end() - vector.begin(), ptrdiff_t(1)); + + ASSERT_EQ(vector.front(), "A"); + ASSERT_EQ(vector.back(), "A"); + vector.pop_back(); + ASSERT_EQ(vector.size(), size_t(0)); +} + +TEST(ContainersVectorTest, BeyondFixedCapPushPop) +{ + yarn::containers::vector<std::string, 2> vector; + vector.push_back("A"); + vector.push_back("B"); + vector.push_back("C"); + vector.push_back("D"); + + ASSERT_EQ(vector.size(), size_t(4)); + ASSERT_EQ(vector.end() - vector.begin(), ptrdiff_t(4)); + + ASSERT_EQ(vector.front(), "A"); + ASSERT_EQ(vector.back(), "D"); + vector.pop_back(); + ASSERT_EQ(vector.size(), size_t(3)); + ASSERT_EQ(vector.end() - vector.begin(), ptrdiff_t(3)); + + ASSERT_EQ(vector.front(), "A"); + ASSERT_EQ(vector.back(), "C"); + vector.pop_back(); + ASSERT_EQ(vector.size(), size_t(2)); + ASSERT_EQ(vector.end() - vector.begin(), ptrdiff_t(2)); + + ASSERT_EQ(vector.front(), "A"); + ASSERT_EQ(vector.back(), "B"); + vector.pop_back(); + ASSERT_EQ(vector.size(), size_t(1)); + ASSERT_EQ(vector.end() - vector.begin(), ptrdiff_t(1)); + + ASSERT_EQ(vector.front(), "A"); + ASSERT_EQ(vector.back(), "A"); + vector.pop_back(); + ASSERT_EQ(vector.size(), size_t(0)); +} + +TEST(ContainersVectorTest, CopyConstruct) +{ + yarn::containers::vector<std::string, 4> vectorA; + + vectorA.resize(3); + vectorA[0] = "A"; + vectorA[1] = "B"; + vectorA[2] = "C"; + + yarn::containers::vector<std::string, 2> vectorB(vectorA); + ASSERT_EQ(vectorB.size(), size_t(3)); + ASSERT_EQ(vectorB[0], "A"); + ASSERT_EQ(vectorB[1], "B"); + ASSERT_EQ(vectorB[2], "C"); +} + +TEST(ContainersVectorTest, MoveConstruct) +{ + yarn::containers::vector<std::string, 4> vectorA; + + vectorA.resize(3); + vectorA[0] = "A"; + vectorA[1] = "B"; + vectorA[2] = "C"; + + yarn::containers::vector<std::string, 2> vectorB(std::move(vectorA)); + ASSERT_EQ(vectorB.size(), size_t(3)); + ASSERT_EQ(vectorB[0], "A"); + ASSERT_EQ(vectorB[1], "B"); + ASSERT_EQ(vectorB[2], "C"); +} + +TEST(ContainersVectorTest, Copy) +{ + yarn::containers::vector<std::string, 4> vectorA; + yarn::containers::vector<std::string, 2> vectorB; + + vectorA.resize(3); + vectorA[0] = "A"; + vectorA[1] = "B"; + vectorA[2] = "C"; + + vectorB.resize(1); + vectorB[0] = "Z"; + + vectorB = vectorA; + ASSERT_EQ(vectorB.size(), size_t(3)); + ASSERT_EQ(vectorB[0], "A"); + ASSERT_EQ(vectorB[1], "B"); + ASSERT_EQ(vectorB[2], "C"); +} + +TEST(ContainersVectorTest, Move) +{ + yarn::containers::vector<std::string, 4> vectorA; + yarn::containers::vector<std::string, 2> vectorB; + + vectorA.resize(3); + vectorA[0] = "A"; + vectorA[1] = "B"; + vectorA[2] = "C"; + + vectorB.resize(1); + vectorB[0] = "Z"; + + vectorB = std::move(vectorA); + ASSERT_EQ(vectorA.size(), size_t(0)); + ASSERT_EQ(vectorB.size(), size_t(3)); + ASSERT_EQ(vectorB[0], "A"); + ASSERT_EQ(vectorB[1], "B"); + ASSERT_EQ(vectorB[2], "C"); +}
diff --git a/src/Yarn/Debug.cpp b/src/Yarn/Debug.cpp new file mode 100644 index 0000000..ead7c16 --- /dev/null +++ b/src/Yarn/Debug.cpp
@@ -0,0 +1,41 @@ +// Copyright 2019 The yarniftShader Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "Debug.hpp" + +#include "Scheduler.hpp" + +#include <cstdlib> + +#include <stdarg.h> +#include <stdio.h> + +namespace yarn +{ + +void fatal(const char* msg, ...) +{ + va_list vararg; + va_start(vararg, msg); + vfprintf(stderr, msg, vararg); + va_end(vararg); + abort(); +} + +void assert_has_bound_scheduler(const char* feature) +{ + YARN_ASSERT(Scheduler::get() != nullptr, "%s requires a yarn::Scheduler to be bound", feature); +} + +} // namespace yarn
diff --git a/src/Yarn/Debug.hpp b/src/Yarn/Debug.hpp new file mode 100644 index 0000000..76945b0 --- /dev/null +++ b/src/Yarn/Debug.hpp
@@ -0,0 +1,45 @@ +// Copyright 2019 The SwiftShader Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef yarn_debug_hpp +#define yarn_debug_hpp + +#if !defined(YARN_DEBUG_ENABLED) +# if !defined(NDEBUG) || defined(DCHECK_ALWAYS_ON) +# define YARN_DEBUG_ENABLED 1 +# else +# define YARN_DEBUG_ENABLED 0 +# endif +#endif + +namespace yarn { + +void fatal(const char* msg, ...); +void assert_has_bound_scheduler(const char* feature); + +#if YARN_DEBUG_ENABLED +#define YARN_FATAL(msg, ...) yarn::fatal(msg "\n", ##__VA_ARGS__); +#define YARN_ASSERT(cond, msg, ...) do { if (!(cond)) { YARN_FATAL("ASSERT: " msg, ##__VA_ARGS__); } } while (false); +#define YARN_ASSERT_HAS_BOUND_SCHEDULER(feature) assert_has_bound_scheduler(feature); +#define YARN_UNREACHABLE() YARN_FATAL("UNREACHABLE"); +#else +#define YARN_FATAL(msg, ...) +#define YARN_ASSERT(cond, msg, ...) +#define YARN_ASSERT_HAS_BOUND_SCHEDULER(feature) +#define YARN_UNREACHABLE() +#endif + +} // namespace yarn + +#endif // yarn_debug_hpp
diff --git a/src/Yarn/Defer.hpp b/src/Yarn/Defer.hpp new file mode 100644 index 0000000..565dfa1 --- /dev/null +++ b/src/Yarn/Defer.hpp
@@ -0,0 +1,44 @@ +// Copyright 2019 The SwiftShader Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef yarn_defer_hpp +#define yarn_defer_hpp + +#include "Finally.hpp" + +namespace yarn { + +#define YARN_CONCAT_(a, b) a ## b +#define YARN_CONCAT(a, b) YARN_CONCAT_(a,b) + +// defer() is a macro to defer execution of a statement until the surrounding +// scope is closed and is typically used to perform cleanup logic once a +// function returns. +// +// Note: Unlike golang's defer(), the defer statement is executed when the +// surrounding *scope* is closed, not necessarily the function. +// +// Example usage: +// +// void sayHelloWorld() +// { +// defer(printf("world\n")); +// printf("hello "); +// } +// +#define defer(x) auto YARN_CONCAT(defer_, __LINE__) = yarn::make_finally([&]{ x; }) + +} // namespace yarn + +#endif // yarn_defer_hpp
diff --git a/src/Yarn/Defer_test.cpp b/src/Yarn/Defer_test.cpp new file mode 100644 index 0000000..479022f --- /dev/null +++ b/src/Yarn/Defer_test.cpp
@@ -0,0 +1,40 @@ +// Copyright 2019 The SwiftShader Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "Defer.hpp" + +#include "Yarn_test.hpp" + +TEST(WithoutBoundScheduler, Defer) +{ + bool deferCalled = false; + { + defer(deferCalled = true); + } + ASSERT_TRUE(deferCalled); +} + +TEST(WithoutBoundScheduler, DeferOrder) +{ + int counter = 0; + int a = 0, b = 0, c = 0; + { + defer(a = ++counter); + defer(b = ++counter); + defer(c = ++counter); + } + ASSERT_EQ(a, 3); + ASSERT_EQ(b, 2); + ASSERT_EQ(c, 1); +} \ No newline at end of file
diff --git a/src/Yarn/Finally.hpp b/src/Yarn/Finally.hpp new file mode 100644 index 0000000..702c87b --- /dev/null +++ b/src/Yarn/Finally.hpp
@@ -0,0 +1,89 @@ +// Copyright 2019 The SwiftShader Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Finally can be used to execute a lambda or function when the final reference +// to the Finally is dropped. +// +// The purpose of a finally is to perform cleanup or termination logic and is +// especially useful when there are multiple early returns within a function. +// +// A moveable Finally can be constructed with yarn::make_finally(). +// A sharable Finally can be constructed with yarn::make_shared_finally(). + +#ifndef yarn_finally_hpp +#define yarn_finally_hpp + +#include <functional> +#include <memory> + +namespace yarn { + +// Finally is a pure virtual base class, implemented by the templated +// FinallyImpl. +class Finally +{ +public: + virtual ~Finally() = default; +}; + +// FinallyImpl implements a Finally. +// The template parameter F is the function type to be called when the finally +// is destructed. F must have the signature void(). +template <typename F> +class FinallyImpl : public Finally +{ +public: + inline FinallyImpl(const F& func); + inline FinallyImpl(F&& func); + inline FinallyImpl(FinallyImpl<F>&& other); + inline ~FinallyImpl(); + +private: + FinallyImpl(const FinallyImpl<F>& other) = delete; + FinallyImpl<F>& operator = (const FinallyImpl<F>& other) = delete; + FinallyImpl<F>& operator = (FinallyImpl<F>&&) = delete; + F func; + bool valid = true; +}; + +template<typename F> +FinallyImpl<F>::FinallyImpl(const F& func) : func(func) {} + +template<typename F> +FinallyImpl<F>::FinallyImpl(F&& func) : func(std::move(func)) {} + +template<typename F> +FinallyImpl<F>::FinallyImpl(FinallyImpl<F>&& other) : func(std::move(other.func)) +{ + other.valid = false; +} + +template<typename F> +FinallyImpl<F>::~FinallyImpl() +{ + if (valid) + { + func(); + } +} + +template<typename F> +inline FinallyImpl<F> make_finally(F&& f) { return FinallyImpl<F>(std::move(f)); } + +template<typename F> +inline std::shared_ptr<Finally> make_shared_finally(F&& f) { return std::make_shared<FinallyImpl<F>>(std::move(f)); } + +} // namespace yarn + +#endif // yarn_finally_hpp
diff --git a/src/Yarn/OSFiber.hpp b/src/Yarn/OSFiber.hpp new file mode 100644 index 0000000..d52de3e --- /dev/null +++ b/src/Yarn/OSFiber.hpp
@@ -0,0 +1,21 @@ +// Copyright 2019 The SwiftShader Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#if defined(_WIN32) +#include "OSFiber_windows.hpp" +#elif defined(YARN_FIBERS_USE_UCONTEXT) +#include "OSFiber_ucontext.hpp" +#else +#include "OSFiber_asm.hpp" +#endif
diff --git a/src/Yarn/OSFiber_aarch64.c b/src/Yarn/OSFiber_aarch64.c new file mode 100644 index 0000000..7b09c58 --- /dev/null +++ b/src/Yarn/OSFiber_aarch64.c
@@ -0,0 +1,33 @@ +// Copyright 2019 The SwiftShader Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#if defined(__aarch64__) + +#include "OSFiber_asm_aarch64.h" + +void yarn_fiber_trampoline(void(*target)(void*), void* arg) +{ + target(arg); +} + +void yarn_fiber_set_target(struct yarn_fiber_context* ctx, void* stack, uint32_t stack_size, void(*target)(void*), void* arg) +{ + uintptr_t* stack_top = (uintptr_t*)((uint8_t*)(stack) + stack_size); + ctx->LR = (uintptr_t)&yarn_fiber_trampoline; + ctx->r0 = (uintptr_t)target; + ctx->r1 = (uintptr_t)arg; + ctx->SP = ((uintptr_t)stack_top) & ~(uintptr_t)15; +} + +#endif // defined(__aarch64__)
diff --git a/src/Yarn/OSFiber_arm.c b/src/Yarn/OSFiber_arm.c new file mode 100644 index 0000000..4a6b78e --- /dev/null +++ b/src/Yarn/OSFiber_arm.c
@@ -0,0 +1,33 @@ +// Copyright 2019 The SwiftShader Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#if defined(__arm__) + +#include "OSFiber_asm_arm.h" + +void yarn_fiber_trampoline(void(*target)(void*), void* arg) +{ + target(arg); +} + +void yarn_fiber_set_target(struct yarn_fiber_context* ctx, void* stack, uint32_t stack_size, void(*target)(void*), void* arg) +{ + uintptr_t* stack_top = (uintptr_t*)((uint8_t*)(stack) + stack_size); + ctx->LR = (uintptr_t)&yarn_fiber_trampoline; + ctx->r0 = (uintptr_t)target; + ctx->r1 = (uintptr_t)arg; + ctx->SP = ((uintptr_t)stack_top) & ~(uintptr_t)15; +} + +#endif // defined(__arm__)
diff --git a/src/Yarn/OSFiber_asm.hpp b/src/Yarn/OSFiber_asm.hpp new file mode 100644 index 0000000..e0fa508 --- /dev/null +++ b/src/Yarn/OSFiber_asm.hpp
@@ -0,0 +1,103 @@ +// Copyright 2019 The SwiftShader Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Minimal assembly implementations of fiber context switching for Unix-based +// platforms. +// +// Note: Unlike makecontext, swapcontext or the Windows fiber APIs, these +// assembly implementations *do not* save or restore signal masks, +// floating-point control or status registers, FS and GS segment registers, +// thread-local storage state nor any SIMD registers. This should not be a +// problem as the yarn scheduler requires fibers to be executed on a single +// thread. + +#if defined(__x86_64__) +#include "OSFiber_asm_x64.h" +#elif defined(__i386__) +#include "OSFiber_asm_x86.h" +#elif defined(__aarch64__) +#include "OSFiber_asm_aarch64.h" +#elif defined(__arm__) +#include "OSFiber_asm_arm.h" +#else +#error "Unsupported target" +#endif + +#include <functional> +#include <memory> + +extern "C" +{ + +extern void yarn_fiber_set_target(yarn_fiber_context*, void* stack, uint32_t stack_size, void(*target)(void*), void* arg); +extern void yarn_fiber_swap(yarn_fiber_context* from, const yarn_fiber_context* to); + +} // extern "C" + +namespace yarn { + +class OSFiber +{ +public: + // createFiberFromCurrentThread() returns a fiber created from the current + // thread. + static inline OSFiber* createFiberFromCurrentThread(); + + // createFiber() returns a new fiber with the given stack size that will + // call func when switched to. func() must end by switching back to another + // fiber, and must not return. + static inline OSFiber* createFiber(size_t stackSize, const std::function<void()>& func); + + // switchTo() immediately switches execution to the given fiber. + // switchTo() must be called on the currently executing fiber. + inline void switchTo(OSFiber*); + +private: + static inline void run(OSFiber* self); + + yarn_fiber_context context; + std::function<void()> target; + std::unique_ptr<uint8_t[]> stack; +}; + +OSFiber* OSFiber::createFiberFromCurrentThread() +{ + auto out = new OSFiber(); + out->context = {}; + return out; +} + +OSFiber* OSFiber::createFiber(size_t stackSize, const std::function<void()>& func) +{ + auto out = new OSFiber(); + out->context = {}; + out->target = func; + out->stack = std::unique_ptr<uint8_t[]>(new uint8_t[stackSize]); + yarn_fiber_set_target(&out->context, out->stack.get(), stackSize, reinterpret_cast<void (*)(void*)>(&OSFiber::run), out); + return out; +} + +void OSFiber::run(OSFiber* self) +{ + std::function<void()> func; + std::swap(func, self->target); + func(); +} + +void OSFiber::switchTo(OSFiber* fiber) +{ + yarn_fiber_swap(&context, &fiber->context); +} + +} // namespace yarn
diff --git a/src/Yarn/OSFiber_asm_aarch64.S b/src/Yarn/OSFiber_asm_aarch64.S new file mode 100644 index 0000000..6a15cc7 --- /dev/null +++ b/src/Yarn/OSFiber_asm_aarch64.S
@@ -0,0 +1,102 @@ +// Copyright 2019 The SwiftShader Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#if defined(__aarch64__) + +#define YARN_BUILD_ASM 1 +#include "OSFiber_asm_aarch64.h" + +// void yarn_fiber_swap(yarn_fiber_context* from, const yarn_fiber_context* to) +// x0: from +// x1: to +.text +.global YARN_ASM_SYMBOL(yarn_fiber_swap) +.align 4 +YARN_ASM_SYMBOL(yarn_fiber_swap): + + // Save context 'from' + // TODO: pairs of str can be combined with stp. + + // Store special purpose registers + str x16, [x0, #YARN_REG_r16] + str x17, [x0, #YARN_REG_r17] + str x18, [x0, #YARN_REG_r18] + + // Store callee-preserved registers + str x19, [x0, #YARN_REG_r19] + str x20, [x0, #YARN_REG_r20] + str x21, [x0, #YARN_REG_r21] + str x22, [x0, #YARN_REG_r22] + str x23, [x0, #YARN_REG_r23] + str x24, [x0, #YARN_REG_r24] + str x25, [x0, #YARN_REG_r25] + str x26, [x0, #YARN_REG_r26] + str x27, [x0, #YARN_REG_r27] + str x28, [x0, #YARN_REG_r28] + + str d8, [x0, #YARN_REG_v8] + str d9, [x0, #YARN_REG_v9] + str d10, [x0, #YARN_REG_v10] + str d11, [x0, #YARN_REG_v11] + str d12, [x0, #YARN_REG_v12] + str d13, [x0, #YARN_REG_v13] + str d14, [x0, #YARN_REG_v14] + str d15, [x0, #YARN_REG_v15] + + // Store sp and lr + mov x2, sp + str x2, [x0, #YARN_REG_SP] + str x30, [x0, #YARN_REG_LR] + + // Load context 'to' + mov x7, x1 + + // Load special purpose registers + ldr x16, [x7, #YARN_REG_r16] + ldr x17, [x7, #YARN_REG_r17] + ldr x18, [x7, #YARN_REG_r18] + + // Load callee-preserved registers + ldr x19, [x7, #YARN_REG_r19] + ldr x20, [x7, #YARN_REG_r20] + ldr x21, [x7, #YARN_REG_r21] + ldr x22, [x7, #YARN_REG_r22] + ldr x23, [x7, #YARN_REG_r23] + ldr x24, [x7, #YARN_REG_r24] + ldr x25, [x7, #YARN_REG_r25] + ldr x26, [x7, #YARN_REG_r26] + ldr x27, [x7, #YARN_REG_r27] + ldr x28, [x7, #YARN_REG_r28] + + ldr d8, [x7, #YARN_REG_v8] + ldr d9, [x7, #YARN_REG_v9] + ldr d10, [x7, #YARN_REG_v10] + ldr d11, [x7, #YARN_REG_v11] + ldr d12, [x7, #YARN_REG_v12] + ldr d13, [x7, #YARN_REG_v13] + ldr d14, [x7, #YARN_REG_v14] + ldr d15, [x7, #YARN_REG_v15] + + // Load parameter registers + ldr x0, [x7, #YARN_REG_r0] + ldr x1, [x7, #YARN_REG_r1] + + // Load sp and lr + ldr x30, [x7, #YARN_REG_LR] + ldr x2, [x7, #YARN_REG_SP] + mov sp, x2 + + ret + +#endif // defined(__aarch64__)
diff --git a/src/Yarn/OSFiber_asm_aarch64.h b/src/Yarn/OSFiber_asm_aarch64.h new file mode 100644 index 0000000..fa3456c --- /dev/null +++ b/src/Yarn/OSFiber_asm_aarch64.h
@@ -0,0 +1,118 @@ +// Copyright 2019 The SwiftShader Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#define YARN_REG_r0 0x00 +#define YARN_REG_r1 0x08 +#define YARN_REG_r16 0x10 +#define YARN_REG_r17 0x18 +#define YARN_REG_r18 0x20 +#define YARN_REG_r19 0x28 +#define YARN_REG_r20 0x30 +#define YARN_REG_r21 0x38 +#define YARN_REG_r22 0x40 +#define YARN_REG_r23 0x48 +#define YARN_REG_r24 0x50 +#define YARN_REG_r25 0x58 +#define YARN_REG_r26 0x60 +#define YARN_REG_r27 0x68 +#define YARN_REG_r28 0x70 +#define YARN_REG_v8 0x78 +#define YARN_REG_v9 0x80 +#define YARN_REG_v10 0x88 +#define YARN_REG_v11 0x90 +#define YARN_REG_v12 0x98 +#define YARN_REG_v13 0xa0 +#define YARN_REG_v14 0xa8 +#define YARN_REG_v15 0xb0 +#define YARN_REG_SP 0xb8 +#define YARN_REG_LR 0xc0 + +#if defined(__APPLE__) +#define YARN_ASM_SYMBOL(x) _##x +#else +#define YARN_ASM_SYMBOL(x) x +#endif + +#ifndef YARN_BUILD_ASM + +#include <stdint.h> + +// Procedure Call Standard for the ARM 64-bit Architecture +// http://infocenter.arm.com/help/topic/com.arm.doc.ihi0055b/IHI0055B_aapcs64.pdf +struct yarn_fiber_context +{ + // parameter registers + uintptr_t r0; + uintptr_t r1; + + // special purpose registers + uintptr_t r16; + uintptr_t r17; + uintptr_t r18; // platform specific (maybe inter-procedural state) + + // callee-saved registers + uintptr_t r19; + uintptr_t r20; + uintptr_t r21; + uintptr_t r22; + uintptr_t r23; + uintptr_t r24; + uintptr_t r25; + uintptr_t r26; + uintptr_t r27; + uintptr_t r28; + + uintptr_t v8; + uintptr_t v9; + uintptr_t v10; + uintptr_t v11; + uintptr_t v12; + uintptr_t v13; + uintptr_t v14; + uintptr_t v15; + + uintptr_t SP; // stack pointer + uintptr_t LR; // link register (R30) +}; + +#ifdef __cplusplus +#include <cstddef> +static_assert(offsetof(yarn_fiber_context, r0) == YARN_REG_r0, "Bad register offset"); +static_assert(offsetof(yarn_fiber_context, r1) == YARN_REG_r1, "Bad register offset"); +static_assert(offsetof(yarn_fiber_context, r16) == YARN_REG_r16, "Bad register offset"); +static_assert(offsetof(yarn_fiber_context, r17) == YARN_REG_r17, "Bad register offset"); +static_assert(offsetof(yarn_fiber_context, r18) == YARN_REG_r18, "Bad register offset"); +static_assert(offsetof(yarn_fiber_context, r19) == YARN_REG_r19, "Bad register offset"); +static_assert(offsetof(yarn_fiber_context, r20) == YARN_REG_r20, "Bad register offset"); +static_assert(offsetof(yarn_fiber_context, r21) == YARN_REG_r21, "Bad register offset"); +static_assert(offsetof(yarn_fiber_context, r22) == YARN_REG_r22, "Bad register offset"); +static_assert(offsetof(yarn_fiber_context, r23) == YARN_REG_r23, "Bad register offset"); +static_assert(offsetof(yarn_fiber_context, r24) == YARN_REG_r24, "Bad register offset"); +static_assert(offsetof(yarn_fiber_context, r25) == YARN_REG_r25, "Bad register offset"); +static_assert(offsetof(yarn_fiber_context, r26) == YARN_REG_r26, "Bad register offset"); +static_assert(offsetof(yarn_fiber_context, r27) == YARN_REG_r27, "Bad register offset"); +static_assert(offsetof(yarn_fiber_context, r28) == YARN_REG_r28, "Bad register offset"); +static_assert(offsetof(yarn_fiber_context, v8) == YARN_REG_v8, "Bad register offset"); +static_assert(offsetof(yarn_fiber_context, v9) == YARN_REG_v9, "Bad register offset"); +static_assert(offsetof(yarn_fiber_context, v10) == YARN_REG_v10, "Bad register offset"); +static_assert(offsetof(yarn_fiber_context, v11) == YARN_REG_v11, "Bad register offset"); +static_assert(offsetof(yarn_fiber_context, v12) == YARN_REG_v12, "Bad register offset"); +static_assert(offsetof(yarn_fiber_context, v13) == YARN_REG_v13, "Bad register offset"); +static_assert(offsetof(yarn_fiber_context, v14) == YARN_REG_v14, "Bad register offset"); +static_assert(offsetof(yarn_fiber_context, v15) == YARN_REG_v15, "Bad register offset"); +static_assert(offsetof(yarn_fiber_context, SP) == YARN_REG_SP, "Bad register offset"); +static_assert(offsetof(yarn_fiber_context, LR) == YARN_REG_LR, "Bad register offset"); +#endif // __cplusplus + +#endif // YARN_BUILD_ASM
diff --git a/src/Yarn/OSFiber_asm_arm.S b/src/Yarn/OSFiber_asm_arm.S new file mode 100644 index 0000000..a810b33 --- /dev/null +++ b/src/Yarn/OSFiber_asm_arm.S
@@ -0,0 +1,74 @@ +// Copyright 2019 The SwiftShader Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#if defined(__arm__) + +#define YARN_BUILD_ASM 1 +#include "OSFiber_asm_arm.h" + +// void yarn_fiber_swap(yarn_fiber_context* from, const yarn_fiber_context* to) +// x0: from +// x1: to +.text +.global yarn_fiber_swap +.align 4 +yarn_fiber_swap: + + // Save context 'from' + // TODO: multiple registers can be stored in a single instruction with: stm rA, {rB-rC} + + // Store special purpose registers + str r12, [r0, #YARN_REG_r12] + + // Store callee-preserved registers + str r4, [r0, #YARN_REG_r4] + str r5, [r0, #YARN_REG_r5] + str r6, [r0, #YARN_REG_r6] + str r7, [r0, #YARN_REG_r7] + str r8, [r0, #YARN_REG_r8] + str r9, [r0, #YARN_REG_r9] + str r10, [r0, #YARN_REG_r10] + str r11, [r0, #YARN_REG_r11] + + // Store sp, lr and pc + str sp, [r0, #YARN_REG_SP] + str lr, [r0, #YARN_REG_LR] + + // Load context 'to' + // TODO: multiple registers can be loaded in a single instruction with: ldm rA, {rB-rC} + mov r3, r1 + + // Load special purpose registers + ldr r12, [r3, #YARN_REG_r12] + + // Load callee-preserved registers + ldr r4, [r3, #YARN_REG_r4] + ldr r5, [r3, #YARN_REG_r5] + ldr r6, [r3, #YARN_REG_r6] + ldr r7, [r3, #YARN_REG_r7] + ldr r8, [r3, #YARN_REG_r8] + ldr r9, [r3, #YARN_REG_r9] + ldr r10, [r3, #YARN_REG_r10] + ldr r11, [r3, #YARN_REG_r11] + + // Load parameter registers + ldr r0, [r3, #YARN_REG_r0] + ldr r1, [r3, #YARN_REG_r1] + + // Load sp, lr and pc + ldr sp, [r3, #YARN_REG_SP] + ldr lr, [r3, #YARN_REG_LR] + mov pc, lr + +#endif // defined(__arm__)
diff --git a/src/Yarn/OSFiber_asm_arm.h b/src/Yarn/OSFiber_asm_arm.h new file mode 100644 index 0000000..641d614 --- /dev/null +++ b/src/Yarn/OSFiber_asm_arm.h
@@ -0,0 +1,99 @@ +// Copyright 2019 The SwiftShader Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#define YARN_REG_r0 0x00 +#define YARN_REG_r1 0x04 +#define YARN_REG_r12 0x08 +#define YARN_REG_r4 0x0c +#define YARN_REG_r5 0x10 +#define YARN_REG_r6 0x14 +#define YARN_REG_r7 0x18 +#define YARN_REG_r8 0x1c +#define YARN_REG_r9 0x20 +#define YARN_REG_r10 0x24 +#define YARN_REG_r11 0x28 +#define YARN_REG_v8 0x2c +#define YARN_REG_v9 0x30 +#define YARN_REG_v10 0x34 +#define YARN_REG_v11 0x38 +#define YARN_REG_v12 0x3c +#define YARN_REG_v13 0x40 +#define YARN_REG_v14 0x44 +#define YARN_REG_v15 0x48 +#define YARN_REG_SP 0x4c +#define YARN_REG_LR 0x50 + +#ifndef YARN_BUILD_ASM +#include <stdint.h> + +// Procedure Call Standard for the ARM 64-bit Architecture +// http://infocenter.arm.com/help/topic/com.arm.doc.ihi0055b/IHI0055B_aapcs64.pdf +struct yarn_fiber_context +{ + // parameter registers + uintptr_t r0; + uintptr_t r1; + + // special purpose registers + uintptr_t r12; // Intra-Procedure-call + + // callee-saved registers + uintptr_t r4; + uintptr_t r5; + uintptr_t r6; + uintptr_t r7; + uintptr_t r8; + uintptr_t r9; + uintptr_t r10; + uintptr_t r11; + + uintptr_t v8; + uintptr_t v9; + uintptr_t v10; + uintptr_t v11; + uintptr_t v12; + uintptr_t v13; + uintptr_t v14; + uintptr_t v15; + + uintptr_t SP; // stack pointer (r13) + uintptr_t LR; // link register (r14) +}; + +#ifdef __cplusplus +#include <cstddef> +static_assert(offsetof(yarn_fiber_context, r0) == YARN_REG_r0, "Bad register offset"); +static_assert(offsetof(yarn_fiber_context, r1) == YARN_REG_r1, "Bad register offset"); +static_assert(offsetof(yarn_fiber_context, r12) == YARN_REG_r12, "Bad register offset"); +static_assert(offsetof(yarn_fiber_context, r4) == YARN_REG_r4, "Bad register offset"); +static_assert(offsetof(yarn_fiber_context, r5) == YARN_REG_r5, "Bad register offset"); +static_assert(offsetof(yarn_fiber_context, r6) == YARN_REG_r6, "Bad register offset"); +static_assert(offsetof(yarn_fiber_context, r7) == YARN_REG_r7, "Bad register offset"); +static_assert(offsetof(yarn_fiber_context, r8) == YARN_REG_r8, "Bad register offset"); +static_assert(offsetof(yarn_fiber_context, r9) == YARN_REG_r9, "Bad register offset"); +static_assert(offsetof(yarn_fiber_context, r10) == YARN_REG_r10, "Bad register offset"); +static_assert(offsetof(yarn_fiber_context, r11) == YARN_REG_r11, "Bad register offset"); +static_assert(offsetof(yarn_fiber_context, v8) == YARN_REG_v8, "Bad register offset"); +static_assert(offsetof(yarn_fiber_context, v9) == YARN_REG_v9, "Bad register offset"); +static_assert(offsetof(yarn_fiber_context, v10) == YARN_REG_v10, "Bad register offset"); +static_assert(offsetof(yarn_fiber_context, v11) == YARN_REG_v11, "Bad register offset"); +static_assert(offsetof(yarn_fiber_context, v12) == YARN_REG_v12, "Bad register offset"); +static_assert(offsetof(yarn_fiber_context, v13) == YARN_REG_v13, "Bad register offset"); +static_assert(offsetof(yarn_fiber_context, v14) == YARN_REG_v14, "Bad register offset"); +static_assert(offsetof(yarn_fiber_context, v15) == YARN_REG_v15, "Bad register offset"); +static_assert(offsetof(yarn_fiber_context, SP) == YARN_REG_SP, "Bad register offset"); +static_assert(offsetof(yarn_fiber_context, LR) == YARN_REG_LR, "Bad register offset"); +#endif // __cplusplus + +#endif // YARN_BUILD_ASM
diff --git a/src/Yarn/OSFiber_asm_x64.S b/src/Yarn/OSFiber_asm_x64.S new file mode 100644 index 0000000..3232b71 --- /dev/null +++ b/src/Yarn/OSFiber_asm_x64.S
@@ -0,0 +1,65 @@ +// Copyright 2019 The SwiftShader Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#if defined(__x86_64__) + +#define YARN_BUILD_ASM 1 +#include "OSFiber_asm_x64.h" + +// void yarn_fiber_swap(yarn_fiber_context* from, const yarn_fiber_context* to) +// rdi: from +// rsi: to +.text +.global YARN_ASM_SYMBOL(yarn_fiber_swap) +.align 4 +YARN_ASM_SYMBOL(yarn_fiber_swap): + + // Save context 'from' + + // Store callee-preserved registers + movq %rbx, YARN_REG_RBX(%rdi) + movq %rbp, YARN_REG_RBP(%rdi) + movq %r12, YARN_REG_R12(%rdi) + movq %r13, YARN_REG_R13(%rdi) + movq %r14, YARN_REG_R14(%rdi) + movq %r15, YARN_REG_R15(%rdi) + + movq (%rsp), %rcx /* call stores the return address on the stack before jumping */ + movq %rcx, YARN_REG_RIP(%rdi) + leaq 8(%rsp), %rcx /* skip the pushed return address */ + movq %rcx, YARN_REG_RSP(%rdi) + + // Load context 'to' + movq %rsi, %r8 + + // Load callee-preserved registers + movq YARN_REG_RBX(%r8), %rbx + movq YARN_REG_RBP(%r8), %rbp + movq YARN_REG_R12(%r8), %r12 + movq YARN_REG_R13(%r8), %r13 + movq YARN_REG_R14(%r8), %r14 + movq YARN_REG_R15(%r8), %r15 + + // Load first two call parameters + movq YARN_REG_RDI(%r8), %rdi + movq YARN_REG_RSI(%r8), %rsi + + // Load stack pointer + movq YARN_REG_RSP(%r8), %rsp + + // Load instruction pointer, and jump + movq YARN_REG_RIP(%r8), %rcx + jmp *%rcx + +#endif // defined(__x86_64__)
diff --git a/src/Yarn/OSFiber_asm_x64.h b/src/Yarn/OSFiber_asm_x64.h new file mode 100644 index 0000000..506351c --- /dev/null +++ b/src/Yarn/OSFiber_asm_x64.h
@@ -0,0 +1,69 @@ +// Copyright 2019 The SwiftShader Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#define YARN_REG_RBX 0x00 +#define YARN_REG_RBP 0x08 +#define YARN_REG_R12 0x10 +#define YARN_REG_R13 0x18 +#define YARN_REG_R14 0x20 +#define YARN_REG_R15 0x28 +#define YARN_REG_RDI 0x30 +#define YARN_REG_RSI 0x38 +#define YARN_REG_RSP 0x40 +#define YARN_REG_RIP 0x48 + +#if defined(__APPLE__) +#define YARN_ASM_SYMBOL(x) _##x +#else +#define YARN_ASM_SYMBOL(x) x +#endif + +#ifndef YARN_BUILD_ASM + +#include <stdint.h> + +struct yarn_fiber_context +{ + // callee-saved registers + uintptr_t RBX; + uintptr_t RBP; + uintptr_t R12; + uintptr_t R13; + uintptr_t R14; + uintptr_t R15; + + // parameter registers + uintptr_t RDI; + uintptr_t RSI; + + // stack and instruction registers + uintptr_t RSP; + uintptr_t RIP; +}; + +#ifdef __cplusplus +#include <cstddef> +static_assert(offsetof(yarn_fiber_context, RBX) == YARN_REG_RBX, "Bad register offset"); +static_assert(offsetof(yarn_fiber_context, RBP) == YARN_REG_RBP, "Bad register offset"); +static_assert(offsetof(yarn_fiber_context, R12) == YARN_REG_R12, "Bad register offset"); +static_assert(offsetof(yarn_fiber_context, R13) == YARN_REG_R13, "Bad register offset"); +static_assert(offsetof(yarn_fiber_context, R14) == YARN_REG_R14, "Bad register offset"); +static_assert(offsetof(yarn_fiber_context, R15) == YARN_REG_R15, "Bad register offset"); +static_assert(offsetof(yarn_fiber_context, RDI) == YARN_REG_RDI, "Bad register offset"); +static_assert(offsetof(yarn_fiber_context, RSI) == YARN_REG_RSI, "Bad register offset"); +static_assert(offsetof(yarn_fiber_context, RSP) == YARN_REG_RSP, "Bad register offset"); +static_assert(offsetof(yarn_fiber_context, RIP) == YARN_REG_RIP, "Bad register offset"); +#endif // __cplusplus + +#endif // YARN_BUILD_ASM
diff --git a/src/Yarn/OSFiber_asm_x86.S b/src/Yarn/OSFiber_asm_x86.S new file mode 100644 index 0000000..e1bb1f3 --- /dev/null +++ b/src/Yarn/OSFiber_asm_x86.S
@@ -0,0 +1,57 @@ +// Copyright 2019 The SwiftShader Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#if defined(__i386__) + +#define YARN_BUILD_ASM 1 +#include "OSFiber_asm_x86.h" + +// void yarn_fiber_swap(yarn_fiber_context* from, const yarn_fiber_context* to) +// esp+4: from +// esp+8: to +.text +.global yarn_fiber_swap +.align 4 +yarn_fiber_swap: + // Save context 'from' + movl 4(%esp), %eax + + // Store callee-preserved registers + movl %ebx, YARN_REG_EBX(%eax) + movl %ebp, YARN_REG_EBP(%eax) + movl %esi, YARN_REG_ESI(%eax) + movl %edi, YARN_REG_EDI(%eax) + + movl (%esp), %ecx /* call stores the return address on the stack before jumping */ + movl %ecx, YARN_REG_EIP(%eax) + lea 4(%esp), %ecx /* skip the pushed return address */ + movl %ecx, YARN_REG_ESP(%eax) + + // Load context 'to' + movl 8(%esp), %ecx + + // Load callee-preserved registers + movl YARN_REG_EBX(%ecx), %ebx + movl YARN_REG_EBP(%ecx), %ebp + movl YARN_REG_ESI(%ecx), %esi + movl YARN_REG_EDI(%ecx), %edi + + // Load stack pointer + movl YARN_REG_ESP(%ecx), %esp + + // Load instruction pointer, and jump + movl YARN_REG_EIP(%ecx), %ecx + jmp *%ecx + +#endif // defined(__i386__)
diff --git a/src/Yarn/OSFiber_asm_x86.h b/src/Yarn/OSFiber_asm_x86.h new file mode 100644 index 0000000..42f355f --- /dev/null +++ b/src/Yarn/OSFiber_asm_x86.h
@@ -0,0 +1,50 @@ +// Copyright 2019 The SwiftShader Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#define YARN_REG_EBX 0x00 +#define YARN_REG_EBP 0x04 +#define YARN_REG_ESI 0x08 +#define YARN_REG_EDI 0x0c +#define YARN_REG_ESP 0x10 +#define YARN_REG_EIP 0x14 + +#ifndef YARN_BUILD_ASM +#include <stdint.h> + +// Assumes cdecl calling convention. +// Registers EAX, ECX, and EDX are caller-saved, and the rest are callee-saved. +struct yarn_fiber_context +{ + // callee-saved registers + uintptr_t EBX; + uintptr_t EBP; + uintptr_t ESI; + uintptr_t EDI; + + // stack and instruction registers + uintptr_t ESP; + uintptr_t EIP; +}; + +#ifdef __cplusplus +#include <cstddef> +static_assert(offsetof(yarn_fiber_context, EBX) == YARN_REG_EBX, "Bad register offset"); +static_assert(offsetof(yarn_fiber_context, EBP) == YARN_REG_EBP, "Bad register offset"); +static_assert(offsetof(yarn_fiber_context, ESI) == YARN_REG_ESI, "Bad register offset"); +static_assert(offsetof(yarn_fiber_context, EDI) == YARN_REG_EDI, "Bad register offset"); +static_assert(offsetof(yarn_fiber_context, ESP) == YARN_REG_ESP, "Bad register offset"); +static_assert(offsetof(yarn_fiber_context, EIP) == YARN_REG_EIP, "Bad register offset"); +#endif // __cplusplus + +#endif // YARN_BUILD_ASM
diff --git a/src/Yarn/OSFiber_test.cpp b/src/Yarn/OSFiber_test.cpp new file mode 100644 index 0000000..a4afb6b --- /dev/null +++ b/src/Yarn/OSFiber_test.cpp
@@ -0,0 +1,44 @@ +// Copyright 2019 The SwiftShader Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "OSFiber.hpp" + +#include "Yarn_test.hpp" + +TEST(WithoutBoundScheduler, OSFiber) +{ + std::string str; + auto constexpr fiberStackSize = 8 * 1024; + auto main = std::unique_ptr<yarn::OSFiber>(yarn::OSFiber::createFiberFromCurrentThread()); + std::unique_ptr<yarn::OSFiber> fiberA, fiberB, fiberC; + fiberC = std::unique_ptr<yarn::OSFiber>(yarn::OSFiber::createFiber(fiberStackSize, [&] + { + str += "C"; + fiberC->switchTo(fiberB.get()); + })); + fiberB = std::unique_ptr<yarn::OSFiber>(yarn::OSFiber::createFiber(fiberStackSize, [&] + { + str += "B"; + fiberB->switchTo(fiberA.get()); + })); + fiberA = std::unique_ptr<yarn::OSFiber>(yarn::OSFiber::createFiber(fiberStackSize, [&] + { + str += "A"; + fiberA->switchTo(main.get()); + })); + + main->switchTo(fiberC.get()); + + ASSERT_EQ(str, "CBA"); +}
diff --git a/src/Yarn/OSFiber_ucontext.hpp b/src/Yarn/OSFiber_ucontext.hpp new file mode 100644 index 0000000..6ab1698 --- /dev/null +++ b/src/Yarn/OSFiber_ucontext.hpp
@@ -0,0 +1,114 @@ +// Copyright 2019 The SwiftShader Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#if !defined(_XOPEN_SOURCE) +// This must come before other #includes, otherwise we'll end up with ucontext_t +// definition mismatches, leading to memory corruption hilarity. +#define _XOPEN_SOURCE +#endif // !defined(_XOPEN_SOURCE) + +#include "Debug.hpp" + +#include <functional> +#include <memory> + +#include <ucontext.h> + +#if defined(__clang__) +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wdeprecated-declarations" +#endif // defined(__clang__) + +namespace yarn { + +class OSFiber +{ +public: + // createFiberFromCurrentThread() returns a fiber created from the current + // thread. + static inline OSFiber* createFiberFromCurrentThread(); + + // createFiber() returns a new fiber with the given stack size that will + // call func when switched to. func() must end by switching back to another + // fiber, and must not return. + static inline OSFiber* createFiber(size_t stackSize, const std::function<void()>& func); + + // switchTo() immediately switches execution to the given fiber. + // switchTo() must be called on the currently executing fiber. + inline void switchTo(OSFiber*); + +private: + std::unique_ptr<uint8_t[]> stack; + ucontext_t context; + std::function<void()> target; +}; + +OSFiber* OSFiber::createFiberFromCurrentThread() +{ + auto out = new OSFiber(); + out->context = {}; + getcontext(&out->context); + return out; +} + +OSFiber* OSFiber::createFiber(size_t stackSize, const std::function<void()>& func) +{ + union Args + { + OSFiber* self; + struct { int a; int b; }; + }; + + struct Target + { + static void Main(int a, int b) + { + Args u; + u.a = a; u.b = b; + std::function<void()> func; + std::swap(func, u.self->target); + func(); + } + }; + + auto out = new OSFiber(); + out->context = {}; + out->stack = std::unique_ptr<uint8_t[]>(new uint8_t[stackSize]); + out->target = func; + + auto alignmentOffset = 15 - (reinterpret_cast<uintptr_t>(out->stack.get() + 15) & 15); + auto res = getcontext(&out->context); + YARN_ASSERT(res == 0, "getcontext() returned %d", int(res)); + out->context.uc_stack.ss_sp = out->stack.get() + alignmentOffset; + out->context.uc_stack.ss_size = stackSize - alignmentOffset; + out->context.uc_link = nullptr; + + Args args; + args.self = out; + makecontext(&out->context, reinterpret_cast<void(*)()>(&Target::Main), 2, args.a, args.b); + + return out; +} + +void OSFiber::switchTo(OSFiber* fiber) +{ + auto res = swapcontext(&context, &fiber->context); + YARN_ASSERT(res == 0, "swapcontext() returned %d", int(res)); +} + +} // namespace yarn + +#if defined(__clang__) +#pragma clang diagnostic pop +#endif // defined(__clang__)
diff --git a/src/Yarn/OSFiber_windows.hpp b/src/Yarn/OSFiber_windows.hpp new file mode 100644 index 0000000..af07649 --- /dev/null +++ b/src/Yarn/OSFiber_windows.hpp
@@ -0,0 +1,90 @@ +// Copyright 2019 The SwiftShader Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include <functional> +#include <memory> + +#include <Windows.h> + +namespace yarn { + +class OSFiber +{ +public: + inline ~OSFiber(); + + // createFiberFromCurrentThread() returns a fiber created from the current + // thread. + static inline OSFiber* createFiberFromCurrentThread(); + + // createFiber() returns a new fiber with the given stack size that will + // call func when switched to. func() must end by switching back to another + // fiber, and must not return. + static inline OSFiber* createFiber(size_t stackSize, const std::function<void()>& func); + + // switchTo() immediately switches execution to the given fiber. + // switchTo() must be called on the currently executing fiber. + inline void switchTo(OSFiber*); + +private: + static inline void WINAPI run(void* self); + LPVOID fiber = nullptr; + bool isFiberFromThread = false; + std::function<void()> target; +}; + +OSFiber::~OSFiber() +{ + if (fiber != nullptr) + { + if (isFiberFromThread) + { + ConvertFiberToThread(); + } + else + { + DeleteFiber(fiber); + } + } +} + +OSFiber* OSFiber::createFiberFromCurrentThread() +{ + auto out = new OSFiber(); + out->fiber = ConvertThreadToFiber(nullptr); + out->isFiberFromThread = true; + return out; +} + +OSFiber* OSFiber::createFiber(size_t stackSize, const std::function<void()>& func) +{ + auto out = new OSFiber(); + out->fiber = CreateFiber(stackSize, &OSFiber::run, out); + out->target = func; + return out; +} + +void OSFiber::switchTo(OSFiber* fiber) +{ + SwitchToFiber(fiber->fiber); +} + +void WINAPI OSFiber::run(void* self) +{ + std::function<void()> func; + std::swap(func, reinterpret_cast<OSFiber*>(self)->target); + func(); +} + +} // namespace yarn
diff --git a/src/Yarn/OSFiber_x64.c b/src/Yarn/OSFiber_x64.c new file mode 100644 index 0000000..0675053 --- /dev/null +++ b/src/Yarn/OSFiber_x64.c
@@ -0,0 +1,34 @@ +// Copyright 2019 The SwiftShader Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#if defined(__x86_64__) + +#include "OSFiber_asm_x64.h" + +void yarn_fiber_trampoline(void(*target)(void*), void* arg) +{ + target(arg); +} + +void yarn_fiber_set_target(struct yarn_fiber_context* ctx, void* stack, uint32_t stack_size, void(*target)(void*), void* arg) +{ + uintptr_t* stack_top = (uintptr_t*)((uint8_t*)(stack) + stack_size); + ctx->RIP = (uintptr_t)&yarn_fiber_trampoline; + ctx->RDI = (uintptr_t)target; + ctx->RSI = (uintptr_t)arg; + ctx->RSP = (uintptr_t)&stack_top[-3]; + stack_top[-2] = 0; // No return target. +} + +#endif // defined(__x86_64__)
diff --git a/src/Yarn/OSFiber_x86.c b/src/Yarn/OSFiber_x86.c new file mode 100644 index 0000000..2bfe082 --- /dev/null +++ b/src/Yarn/OSFiber_x86.c
@@ -0,0 +1,34 @@ +// Copyright 2019 The SwiftShader Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#if defined(__i386__) + +#include "OSFiber_asm_x86.h" + +void yarn_fiber_trampoline(void(*target)(void*), void* arg) +{ + target(arg); +} + +void yarn_fiber_set_target(struct yarn_fiber_context* ctx, void* stack, uint32_t stack_size, void(*target)(void*), void* arg) +{ + uintptr_t* stack_top = (uintptr_t*)((uint8_t*)(stack) + stack_size); + ctx->EIP = (uintptr_t)&yarn_fiber_trampoline; + ctx->ESP = (uintptr_t)&stack_top[-3]; + stack_top[-1] = (uintptr_t)arg; + stack_top[-2] = (uintptr_t)target; + stack_top[-3] = 0; // No return target. +} + +#endif // defined(__i386__)
diff --git a/src/Yarn/Pool.hpp b/src/Yarn/Pool.hpp new file mode 100644 index 0000000..79d05b5 --- /dev/null +++ b/src/Yarn/Pool.hpp
@@ -0,0 +1,453 @@ +// Copyright 2019 The SwiftShader Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef yarn_pool_hpp +#define yarn_pool_hpp + +#include "ConditionVariable.hpp" + +#include <atomic> +#include <mutex> + +namespace yarn { + +// PoolPolicy controls whether pool items are constructed and destructed each +// time they are borrowed from and returned to a pool, or whether they persist +// constructed for the lifetime of the pool. +enum class PoolPolicy +{ + // Call the Pool items constructor on borrow(), and destruct the item + // when the item is returned. + Reconstruct, + + // Construct and destruct all items once for the lifetime of the Pool. + // Items will keep their state between loans. + Preserve, +}; + +//////////////////////////////////////////////////////////////////////////////// +// Pool<T> +//////////////////////////////////////////////////////////////////////////////// + +// Pool is the abstract base class for BoundedPool<> and UnboundedPool<>. +template <typename T> +class Pool +{ +protected: + struct Item; + class Storage; + +public: + // A Loan is returned by the pool's borrow() function. + // Loans track the number of references to the loaned item, and return the + // item to the pool when the final Loan reference is dropped. + class Loan + { + public: + inline Loan() = default; + inline Loan(Item*, const std::shared_ptr<Storage>&); + inline Loan(const Loan&); + inline Loan(Loan&&); + inline ~Loan(); + inline Loan& operator = (const Loan&); + inline Loan& operator = (Loan&&); + inline T& operator * (); + inline T* operator -> () const; + inline T* get() const; + void reset(); + + private: + Item *item = nullptr; + std::shared_ptr<Storage> storage; + }; + +protected: + Pool() = default; + + // The shared storage between the pool and all loans. + class Storage + { + public: + virtual ~Storage() = default; + virtual void return_(Item*) = 0; + }; + + // The backing data of a single item in the pool. + struct Item + { + // get() returns a pointer to the item's data. + inline T* get(); + + // construct() calls the constructor on the item's data. + inline void construct(); + + // destruct() calls the destructor on the item's data. + inline void destruct(); + + using Data = typename std::aligned_storage<sizeof(T), alignof(T)>::type; + Data data; + std::atomic<int> refcount = {0}; + Item *next = nullptr; // pointer to the next free item in the pool. + }; +}; + +// Loan<T> is an alias to Pool<T>::Loan. +template <typename T> +using Loan = typename Pool<T>::Loan; + +//////////////////////////////////////////////////////////////////////////////// +// Pool<T>::Item +//////////////////////////////////////////////////////////////////////////////// +template <typename T> +T* Pool<T>::Item::get() +{ + return reinterpret_cast<T*>(&data); +} + +template <typename T> +void Pool<T>::Item::construct() +{ + new (&data) T; +} + +template <typename T> +void Pool<T>::Item::destruct() +{ + get()->~T(); +} + +//////////////////////////////////////////////////////////////////////////////// +// Pool<T>::Loan +//////////////////////////////////////////////////////////////////////////////// +template <typename T> +Pool<T>::Loan::Loan(Item* item, const std::shared_ptr<Storage>& storage) : item(item), storage(storage) +{ + item->refcount++; +} + +template <typename T> +Pool<T>::Loan::Loan(const Loan& other) : item(other.item), storage(other.storage) +{ + if (item != nullptr) + { + item->refcount++; + } +} + +template <typename T> +Pool<T>::Loan::Loan(Loan&& other) : item(other.item), storage(other.storage) +{ + other.item = nullptr; + other.storage = nullptr; +} + +template <typename T> +Pool<T>::Loan::~Loan() +{ + reset(); +} + +template <typename T> +void Pool<T>::Loan::reset() +{ + if (item != nullptr) + { + auto refs = --item->refcount; + YARN_ASSERT(refs >= 0, "reset() called on zero-ref pool item"); + if (refs == 0) + { + storage->return_(item); + } + item = nullptr; + storage = nullptr; + } +} + +template <typename T> +typename Pool<T>::Loan& Pool<T>::Loan::operator = (const Pool<T>::Loan& rhs) +{ + reset(); + if (rhs.item != nullptr) + { + item = rhs.item; + storage = rhs.storage; + rhs.item->refcount++; + } + return *this; +} + +template <typename T> +typename Pool<T>::Loan& Pool<T>::Loan::operator = (Pool<T>::Loan&& rhs) +{ + reset(); + std::swap(item, rhs.item); + std::swap(storage, rhs.storage); + return *this; +} + +template <typename T> +T& Pool<T>::Loan::operator * () { return *item->get(); } + +template <typename T> +T* Pool<T>::Loan::operator -> () const { return item->get(); } + +template <typename T> +T* Pool<T>::Loan::get() const { return item->get(); } + +//////////////////////////////////////////////////////////////////////////////// +// BoundedPool<T, N, POLICY> +//////////////////////////////////////////////////////////////////////////////// + +// BoundedPool<T, N, POLICY> is a pool of items of type T, with a maximum +// capacity of N items. +// BoundedPool<> is initially populated with N default-constructed items. +// POLICY controls whether pool items are constructed and destructed each +// time they are borrowed from and returned to the pool. +template <typename T, int N, PoolPolicy POLICY = PoolPolicy::Reconstruct> +class BoundedPool : public Pool<T> +{ +public: + using Item = typename Pool<T>::Item; + using Loan = typename Pool<T>::Loan; + + // borrow() borrows a single item from the pool, blocking until an item is + // returned if the pool is empty. + inline Loan borrow() const; + + // borrow() borrows count items from the pool, blocking until there are at + // least count items in the pool. The function f() is called with each + // borrowed item. + // F must be a function with the signature: void(T&&) + template <typename F> + inline void borrow(size_t count, const F& f) const; + + // tryBorrow() attempts to borrow a single item from the pool without + // blocking. + // The boolean of the returned pair is true on success, or false if the pool + // is empty. + inline std::pair<Loan, bool> tryBorrow() const; + +private: + class Storage : public Pool<T>::Storage + { + public: + inline Storage(); + inline ~Storage(); + inline void return_(Item*) override; + + std::mutex mutex; + ConditionVariable returned; + Item items[N]; + Item *free = nullptr; + }; + std::shared_ptr<Storage> storage = std::make_shared<Storage>(); +}; + +template <typename T, int N, PoolPolicy POLICY> +BoundedPool<T, N, POLICY>::Storage::Storage() +{ + for (int i = 0; i < N; i++) + { + if (POLICY == PoolPolicy::Preserve) + { + items[i].construct(); + } + items[i].next = this->free; + this->free = &items[i]; + } +} + +template <typename T, int N, PoolPolicy POLICY> +BoundedPool<T, N, POLICY>::Storage::~Storage() +{ + if (POLICY == PoolPolicy::Preserve) + { + for (int i = 0; i < N; i++) + { + items[i].destruct(); + } + } +} + +template <typename T, int N, PoolPolicy POLICY> +typename BoundedPool<T, N, POLICY>::Loan BoundedPool<T, N, POLICY>::borrow() const +{ + Loan out; + borrow(1, [&](Loan&& loan) { out = std::move(loan); }); + return out; +} + +template <typename T, int N, PoolPolicy POLICY> +template <typename F> +void BoundedPool<T, N, POLICY>::borrow(size_t n, const F& f) const +{ + std::unique_lock<std::mutex> lock(storage->mutex); + for (size_t i = 0; i < n; i++) + { + storage->returned.wait(lock, [&] { return storage->free != nullptr; }); + auto item = storage->free; + storage->free = storage->free->next; + if (POLICY == PoolPolicy::Reconstruct) + { + item->construct(); + } + f(std::move(Loan(item, storage))); + } +} + +template <typename T, int N, PoolPolicy POLICY> +std::pair<typename BoundedPool<T, N, POLICY>::Loan, bool> BoundedPool<T, N, POLICY>::tryBorrow() const +{ + std::unique_lock<std::mutex> lock(storage->mutex); + if (storage->free == nullptr) + { + return std::make_pair(Loan(), false); + } + auto item = storage->free; + storage->free = storage->free->next; + item->pool = this; + lock.unlock(); + if (POLICY == PoolPolicy::Reconstruct) + { + item->construct(); + } + return std::make_pair(Loan(item, storage), true); +} + +template <typename T, int N, PoolPolicy POLICY> +void BoundedPool<T, N, POLICY>::Storage::return_(Item* item) +{ + if (POLICY == PoolPolicy::Reconstruct) + { + item->destruct(); + } + std::unique_lock<std::mutex> lock(mutex); + item->next = free; + free = item; + lock.unlock(); + returned.notify_one(); +} + +//////////////////////////////////////////////////////////////////////////////// +// UnboundedPool +//////////////////////////////////////////////////////////////////////////////// + +// UnboundedPool<T, POLICY> is a pool of items of type T. +// UnboundedPool<> will automatically allocate more items if the pool becomes +// empty. +// POLICY controls whether pool items are constructed and destructed each +// time they are borrowed from and returned to the pool. +template <typename T, PoolPolicy POLICY = PoolPolicy::Reconstruct> +class UnboundedPool : public Pool<T> +{ +public: + using Item = typename Pool<T>::Item; + using Loan = typename Pool<T>::Loan; + + // borrow() borrows a single item from the pool, automatically allocating + // more items if the pool is empty. + // This function does not block. + inline Loan borrow() const; + + // borrow() borrows count items from the pool, calling the function f() with + // each borrowed item. + // F must be a function with the signature: void(T&&) + // This function does not block. + template <typename F> + inline void borrow(size_t n, const F& f) const; + +private: + class Storage : public Pool<T>::Storage + { + public: + inline ~Storage(); + inline void return_(Item*) override; + + std::mutex mutex; + std::vector<Item*> items; + Item *free = nullptr; + }; + std::shared_ptr<Storage> storage = std::make_shared<Storage>(); +}; + +template <typename T, PoolPolicy POLICY> +UnboundedPool<T, POLICY>::Storage::~Storage() +{ + for (auto item : items) + { + if (POLICY == PoolPolicy::Preserve) + { + item->destruct(); + } + delete item; + } +} + +template <typename T, PoolPolicy POLICY> +Loan<T> UnboundedPool<T, POLICY>::borrow() const +{ + Loan out; + borrow(1, [&] (Loan&& loan) { out = std::move(loan); }); + return out; +} + +template <typename T, PoolPolicy POLICY> +template <typename F> +inline void UnboundedPool<T, POLICY>::borrow(size_t n, const F& f) const +{ + std::unique_lock<std::mutex> lock(storage->mutex); + for (size_t i = 0; i < n; i++) + { + if (storage->free == nullptr) + { + auto count = std::max<size_t>(storage->items.size(), 32); + for (size_t i = 0; i < count; i++) + { + auto item = new Item(); + if (POLICY == PoolPolicy::Preserve) + { + item->construct(); + } + storage->items.push_back(item); + item->next = storage->free; + storage->free = item; + } + } + + auto item = storage->free; + storage->free = storage->free->next; + if (POLICY == PoolPolicy::Reconstruct) + { + item->construct(); + } + f(std::move(Loan(item, storage))); + } +} + +template <typename T, PoolPolicy POLICY> +void UnboundedPool<T, POLICY>::Storage::return_(Item* item) +{ + if (POLICY == PoolPolicy::Reconstruct) + { + item->destruct(); + } + std::unique_lock<std::mutex> lock(mutex); + item->next = free; + free = item; + lock.unlock(); +} + +} // namespace yarn + +#endif // yarn_pool_hpp
diff --git a/src/Yarn/Pool_test.cpp b/src/Yarn/Pool_test.cpp new file mode 100644 index 0000000..b6face5 --- /dev/null +++ b/src/Yarn/Pool_test.cpp
@@ -0,0 +1,178 @@ +// Copyright 2019 The SwiftShader Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "Yarn_test.hpp" + +#include "Yarn/Pool.hpp" +#include "Yarn/WaitGroup.hpp" + +TEST_P(WithBoundScheduler, UnboundedPool_ConstructDestruct) +{ + yarn::UnboundedPool<int> pool; +} + +TEST_P(WithBoundScheduler, BoundedPool_ConstructDestruct) +{ + yarn::BoundedPool<int, 10> pool; +} + +TEST_P(WithBoundScheduler, UnboundedPool_Borrow) +{ + yarn::UnboundedPool<int> pool; + for (int i = 0; i < 100; i++) + { + pool.borrow(); + } +} + +TEST_P(WithBoundScheduler, UnboundedPool_ConcurrentBorrow) +{ + yarn::UnboundedPool<int> pool; + constexpr int iterations = 10000; + yarn::WaitGroup wg(iterations); + for (int i = 0; i < iterations; i++) { + yarn::schedule([=] { pool.borrow(); wg.done(); }); + } + wg.wait(); +} + +TEST_P(WithBoundScheduler, BoundedPool_Borrow) +{ + yarn::BoundedPool<int, 100> pool; + for (int i = 0; i < 100; i++) + { + pool.borrow(); + } +} + +TEST_P(WithBoundScheduler, BoundedPool_ConcurrentBorrow) +{ + yarn::BoundedPool<int, 10> pool; + constexpr int iterations = 10000; + yarn::WaitGroup wg(iterations); + for (int i = 0; i < iterations; i++) { + yarn::schedule([=] + { + pool.borrow(); + wg.done(); + }); + } + wg.wait(); +} + +struct CtorDtorCounter +{ + CtorDtorCounter() { ctor_count++; } + ~CtorDtorCounter() { dtor_count++; } + static void reset() { ctor_count = 0; dtor_count = 0; } + static int ctor_count; + static int dtor_count; +}; + +int CtorDtorCounter::ctor_count = -1; +int CtorDtorCounter::dtor_count = -1; + +TEST_P(WithBoundScheduler, UnboundedPool_PolicyReconstruct) +{ + CtorDtorCounter::reset(); + yarn::UnboundedPool<CtorDtorCounter, yarn::PoolPolicy::Reconstruct> pool; + ASSERT_EQ(CtorDtorCounter::ctor_count, 0); + ASSERT_EQ(CtorDtorCounter::dtor_count, 0); + { + auto loan = pool.borrow(); + ASSERT_EQ(CtorDtorCounter::ctor_count, 1); + ASSERT_EQ(CtorDtorCounter::dtor_count, 0); + } + ASSERT_EQ(CtorDtorCounter::ctor_count, 1); + ASSERT_EQ(CtorDtorCounter::dtor_count, 1); + { + auto loan = pool.borrow(); + ASSERT_EQ(CtorDtorCounter::ctor_count, 2); + ASSERT_EQ(CtorDtorCounter::dtor_count, 1); + } + ASSERT_EQ(CtorDtorCounter::ctor_count, 2); + ASSERT_EQ(CtorDtorCounter::dtor_count, 2); +} + +TEST_P(WithBoundScheduler, BoundedPool_PolicyReconstruct) +{ + CtorDtorCounter::reset(); + yarn::BoundedPool<CtorDtorCounter, 10, yarn::PoolPolicy::Reconstruct> pool; + ASSERT_EQ(CtorDtorCounter::ctor_count, 0); + ASSERT_EQ(CtorDtorCounter::dtor_count, 0); + { + auto loan = pool.borrow(); + ASSERT_EQ(CtorDtorCounter::ctor_count, 1); + ASSERT_EQ(CtorDtorCounter::dtor_count, 0); + } + ASSERT_EQ(CtorDtorCounter::ctor_count, 1); + ASSERT_EQ(CtorDtorCounter::dtor_count, 1); + { + auto loan = pool.borrow(); + ASSERT_EQ(CtorDtorCounter::ctor_count, 2); + ASSERT_EQ(CtorDtorCounter::dtor_count, 1); + } + ASSERT_EQ(CtorDtorCounter::ctor_count, 2); + ASSERT_EQ(CtorDtorCounter::dtor_count, 2); +} + +TEST_P(WithBoundScheduler, UnboundedPool_PolicyPreserve) +{ + CtorDtorCounter::reset(); + { + yarn::UnboundedPool<CtorDtorCounter, yarn::PoolPolicy::Preserve> pool; + int ctor_count; + { + auto loan = pool.borrow(); + ASSERT_NE(CtorDtorCounter::ctor_count, 0); + ASSERT_EQ(CtorDtorCounter::dtor_count, 0); + ctor_count = CtorDtorCounter::ctor_count; + } + ASSERT_EQ(CtorDtorCounter::ctor_count, ctor_count); + ASSERT_EQ(CtorDtorCounter::dtor_count, 0); + { + auto loan = pool.borrow(); + ASSERT_EQ(CtorDtorCounter::ctor_count, ctor_count); + ASSERT_EQ(CtorDtorCounter::dtor_count, 0); + } + ASSERT_EQ(CtorDtorCounter::ctor_count, ctor_count); + ASSERT_EQ(CtorDtorCounter::dtor_count, 0); + } + ASSERT_EQ(CtorDtorCounter::ctor_count, CtorDtorCounter::dtor_count); +} + +TEST_P(WithBoundScheduler, BoundedPool_PolicyPreserve) +{ + CtorDtorCounter::reset(); + { + yarn::BoundedPool<CtorDtorCounter, 10, yarn::PoolPolicy::Preserve> pool; + int ctor_count; + { + auto loan = pool.borrow(); + ASSERT_NE(CtorDtorCounter::ctor_count, 0); + ASSERT_EQ(CtorDtorCounter::dtor_count, 0); + ctor_count = CtorDtorCounter::ctor_count; + } + ASSERT_EQ(CtorDtorCounter::ctor_count, ctor_count); + ASSERT_EQ(CtorDtorCounter::dtor_count, 0); + { + auto loan = pool.borrow(); + ASSERT_EQ(CtorDtorCounter::ctor_count, ctor_count); + ASSERT_EQ(CtorDtorCounter::dtor_count, 0); + } + ASSERT_EQ(CtorDtorCounter::ctor_count, ctor_count); + ASSERT_EQ(CtorDtorCounter::dtor_count, 0); + } + ASSERT_EQ(CtorDtorCounter::ctor_count, CtorDtorCounter::dtor_count); +}
diff --git a/src/Yarn/SAL.hpp b/src/Yarn/SAL.hpp new file mode 100644 index 0000000..acd03c7 --- /dev/null +++ b/src/Yarn/SAL.hpp
@@ -0,0 +1,29 @@ +// Copyright 2019 The SwiftShader Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Stubs SAL annotation macros for platforms that do not support them. +// See https://docs.microsoft.com/en-us/visualstudio/code-quality/annotating-locking-behavior?view=vs-2019 + +#ifndef yarn_sal_hpp +#define yarn_sal_hpp + +#ifndef _Requires_lock_held_ +#define _Requires_lock_held_(x) +#endif + +#ifndef _Requires_lock_not_held_ +#define _Requires_lock_not_held_(x) +#endif + +#endif // yarn_sal_hpp
diff --git a/src/Yarn/Scheduler.cpp b/src/Yarn/Scheduler.cpp new file mode 100644 index 0000000..25b432e --- /dev/null +++ b/src/Yarn/Scheduler.cpp
@@ -0,0 +1,540 @@ +// Copyright 2019 The SwiftShader Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "OSFiber.hpp" // Must come first. See OSFiber_ucontext.hpp. + +#include "Scheduler.hpp" + +#include "Debug.hpp" +#include "Defer.hpp" +#include "Thread.hpp" +#include "Trace.hpp" + +#if defined(_WIN32) +#include <intrin.h> // __nop() +#endif + +// Enable to trace scheduler events. +#define ENABLE_TRACE_EVENTS 0 + +#if ENABLE_TRACE_EVENTS +#define TRACE(...) YARN_SCOPED_EVENT(__VA_ARGS__) +#else +#define TRACE(...) +#endif + +namespace +{ + +template <typename T> +inline T take(std::queue<T>& queue) +{ + auto out = std::move(queue.front()); + queue.pop(); + return out; +} + +inline void nop() +{ +#if defined(_WIN32) + __nop(); +#else + __asm__ __volatile__ ("nop"); +#endif +} + +} // anonymous namespace + +namespace yarn { + +//////////////////////////////////////////////////////////////////////////////// +// Scheduler +//////////////////////////////////////////////////////////////////////////////// +thread_local Scheduler* Scheduler::bound = nullptr; + +Scheduler* Scheduler::get() +{ + return bound; +} + +void Scheduler::bind() +{ + YARN_ASSERT(bound == nullptr, "Scheduler already bound"); + bound = this; + { + std::unique_lock<std::mutex> lock(singleThreadedWorkerMutex); + auto worker = std::unique_ptr<Worker>(new Worker(this, Worker::Mode::SingleThreaded, 0)); + worker->start(); + auto tid = std::this_thread::get_id(); + singleThreadedWorkers.emplace(tid, std::move(worker)); + } +} + +void Scheduler::unbind() +{ + YARN_ASSERT(bound != nullptr, "No scheduler bound"); + std::unique_ptr<Worker> worker; + { + std::unique_lock<std::mutex> lock(bound->singleThreadedWorkerMutex); + auto tid = std::this_thread::get_id(); + auto it = bound->singleThreadedWorkers.find(tid); + YARN_ASSERT(it != bound->singleThreadedWorkers.end(), "singleThreadedWorker not found"); + worker = std::move(it->second); + bound->singleThreadedWorkers.erase(tid); + } + worker->flush(); + worker->stop(); + bound = nullptr; +} + +Scheduler::Scheduler() +{ + for (size_t i = 0; i < spinningWorkers.size(); i++) + { + spinningWorkers[i] = -1; + } +} + +Scheduler::~Scheduler() +{ + { + std::unique_lock<std::mutex> lock(singleThreadedWorkerMutex); + YARN_ASSERT(singleThreadedWorkers.size() == 0, "Scheduler still bound on %d threads", int(singleThreadedWorkers.size())); + } + setWorkerThreadCount(0); +} + +void Scheduler::setThreadInitializer(const std::function<void()>& func) +{ + std::unique_lock<std::mutex> lock(threadInitFuncMutex); + threadInitFunc = func; +} + +const std::function<void()>& Scheduler::getThreadInitializer() +{ + std::unique_lock<std::mutex> lock(threadInitFuncMutex); + return threadInitFunc; +} + +void Scheduler::setWorkerThreadCount(int newCount) +{ + YARN_ASSERT(newCount >= 0, "count must be positive"); + auto oldCount = numWorkerThreads; + for (int idx = oldCount - 1; idx >= newCount; idx--) + { + workerThreads[idx]->stop(); + } + for (int idx = oldCount - 1; idx >= newCount; idx--) + { + delete workerThreads[idx]; + } + for (int idx = oldCount; idx < newCount; idx++) + { + workerThreads[idx] = new Worker(this, Worker::Mode::MultiThreaded, idx); + } + numWorkerThreads = newCount; + for (int idx = oldCount; idx < newCount; idx++) + { + workerThreads[idx]->start(); + } +} + +int Scheduler::getWorkerThreadCount() +{ + return numWorkerThreads; +} + +void Scheduler::enqueue(Task&& task) +{ + if (numWorkerThreads > 0) + { + while (true) + { + // Prioritize workers that have recently started spinning. + auto i = --nextSpinningWorkerIdx % spinningWorkers.size(); + auto idx = spinningWorkers[i].exchange(-1); + if (idx < 0) + { + // If a spinning worker couldn't be found, round-robin the + // workers. + idx = nextEnqueueIndex++ % numWorkerThreads; + } + + auto worker = workerThreads[idx]; + if (worker->tryLock()) + { + worker->enqueueAndUnlock(std::move(task)); + return; + } + } + } + else + { + auto tid = std::this_thread::get_id(); + std::unique_lock<std::mutex> lock(singleThreadedWorkerMutex); + auto it = singleThreadedWorkers.find(tid); + YARN_ASSERT(it != singleThreadedWorkers.end(), "singleThreadedWorker not found"); + it->second->enqueue(std::move(task)); + } +} + +bool Scheduler::stealWork(Worker* thief, uint64_t from, Task& out) +{ + if (numWorkerThreads > 0) + { + auto thread = workerThreads[from % numWorkerThreads]; + if (thread != thief) + { + if (thread->dequeue(out)) + { + return true; + } + } + } + return false; +} + +void Scheduler::onBeginSpinning(int workerId) +{ + auto idx = nextSpinningWorkerIdx++ % spinningWorkers.size(); + spinningWorkers[idx] = workerId; +} + +//////////////////////////////////////////////////////////////////////////////// +// Fiber +//////////////////////////////////////////////////////////////////////////////// +Scheduler::Fiber::Fiber(OSFiber* impl, uint32_t id) : + id(id), impl(impl), worker(Scheduler::Worker::getCurrent()) +{ + YARN_ASSERT(worker != nullptr, "No Scheduler::Worker bound"); +} + +Scheduler::Fiber::~Fiber() +{ + delete impl; +} + +Scheduler::Fiber* Scheduler::Fiber::current() +{ + auto worker = Scheduler::Worker::getCurrent(); + return worker != nullptr ? worker->getCurrentFiber() : nullptr; +} + +void Scheduler::Fiber::schedule() +{ + worker->enqueue(this); +} + +void Scheduler::Fiber::yield() +{ + YARN_SCOPED_EVENT("YIELD"); + worker->yield(this); +} + +void Scheduler::Fiber::switchTo(Fiber* to) +{ + if (to != this) + { + impl->switchTo(to->impl); + } +} + +Scheduler::Fiber* Scheduler::Fiber::create(uint32_t id, size_t stackSize, const std::function<void()>& func) +{ + return new Fiber(OSFiber::createFiber(stackSize, func), id); +} + +Scheduler::Fiber* Scheduler::Fiber::createFromCurrentThread(uint32_t id) +{ + return new Fiber(OSFiber::createFiberFromCurrentThread(), id); +} + +//////////////////////////////////////////////////////////////////////////////// +// Scheduler::Worker +//////////////////////////////////////////////////////////////////////////////// +thread_local Scheduler::Worker* Scheduler::Worker::current = nullptr; + +Scheduler::Worker::Worker(Scheduler *scheduler, Mode mode, uint32_t id) : id(id), mode(mode), scheduler(scheduler) {} + +void Scheduler::Worker::start() +{ + switch (mode) + { + case Mode::MultiThreaded: + thread = std::thread([=] + { + Thread::setName("Thread<%.2d>", int(id)); + + if (auto const &initFunc = scheduler->getThreadInitializer()) + { + initFunc(); + } + + Scheduler::bound = scheduler; + Worker::current = this; + mainFiber.reset(Fiber::createFromCurrentThread(0)); + currentFiber = mainFiber.get(); + run(); + mainFiber.reset(); + Worker::current = nullptr; + }); + break; + + case Mode::SingleThreaded: + Worker::current = this; + mainFiber.reset(Fiber::createFromCurrentThread(0)); + currentFiber = mainFiber.get(); + break; + + default: + YARN_ASSERT(false, "Unknown mode: %d", int(mode)); + } +} + +void Scheduler::Worker::stop() +{ + switch (mode) + { + case Mode::MultiThreaded: + shutdown = true; + enqueue([]{}); // Ensure the worker is woken up to notice the shutdown. + thread.join(); + break; + + case Mode::SingleThreaded: + Worker::current = nullptr; + break; + + default: + YARN_ASSERT(false, "Unknown mode: %d", int(mode)); + } +} + +void Scheduler::Worker::yield(Fiber *from) +{ + YARN_ASSERT(currentFiber == from, "Attempting to call yield from a non-current fiber"); + + // Current fiber is yielding as it is blocked. + + // First wait until there's something else this worker can do. + std::unique_lock<std::mutex> lock(work.mutex); + waitForWork(lock); + + + if (work.fibers.size() > 0) + { + // There's another fiber that has become unblocked, resume that. + work.num--; + auto to = take(work.fibers); + lock.unlock(); + switchToFiber(to); + } + else if (idleFibers.size() > 0) + { + // There's an old fiber we can reuse, resume that. + auto to = take(idleFibers); + lock.unlock(); + switchToFiber(to); + } + else + { + // Tasks to process and no existing fibers to resume. Spawn a new fiber. + lock.unlock(); + switchToFiber(createWorkerFiber()); + } +} + +bool Scheduler::Worker::tryLock() +{ + return work.mutex.try_lock(); +} + +void Scheduler::Worker::enqueue(Fiber* fiber) +{ + std::unique_lock<std::mutex> lock(work.mutex); + auto wasIdle = work.num == 0; + work.fibers.push(std::move(fiber)); + work.num++; + lock.unlock(); + if (wasIdle) { work.added.notify_one(); } +} + +void Scheduler::Worker::enqueue(Task&& task) +{ + work.mutex.lock(); + enqueueAndUnlock(std::move(task)); +} + +void Scheduler::Worker::enqueueAndUnlock(Task&& task) +{ + auto wasIdle = work.num == 0; + work.tasks.push(std::move(task)); + work.num++; + work.mutex.unlock(); + if (wasIdle) { work.added.notify_one(); } +} + +bool Scheduler::Worker::dequeue(Task& out) +{ + if (work.num.load() == 0) { return false; } + if (!work.mutex.try_lock()) { return false; } + defer(work.mutex.unlock()); + if (work.tasks.size() == 0) { return false; } + work.num--; + out = take(work.tasks); + return true; +} + +void Scheduler::Worker::flush() +{ + YARN_ASSERT(mode == Mode::SingleThreaded, "flush() can only be used on a single-threaded worker"); + std::unique_lock<std::mutex> lock(work.mutex); + runUntilIdle(lock); +} + +void Scheduler::Worker::run() +{ + switch (mode) + { + case Mode::MultiThreaded: + { + YARN_NAME_THREAD("Thread<%.2d> Fiber<%.2d>", int(id), Fiber::current()->id); + { + std::unique_lock<std::mutex> lock(work.mutex); + work.added.wait(lock, [this] { return work.num > 0 || shutdown; }); + while (!shutdown) + { + waitForWork(lock); + runUntilIdle(lock); + } + Worker::current = nullptr; + } + switchToFiber(mainFiber.get()); + break; + } + case Mode::SingleThreaded: + while (!shutdown) + { + flush(); + idleFibers.emplace(currentFiber); + switchToFiber(mainFiber.get()); + } + break; + + default: + YARN_ASSERT(false, "Unknown mode: %d", int(mode)); + } +} + +_Requires_lock_held_(lock) +void Scheduler::Worker::waitForWork(std::unique_lock<std::mutex> &lock) +{ + YARN_ASSERT(work.num == work.fibers.size() + work.tasks.size(), "work.num out of sync"); + if (work.num == 0) + { + scheduler->onBeginSpinning(id); + lock.unlock(); + spinForWork(); + lock.lock(); + } + work.added.wait(lock, [this] { return work.num > 0 || shutdown; }); +} + +void Scheduler::Worker::spinForWork() +{ + TRACE("SPIN"); + Task stolen; + + constexpr auto duration = std::chrono::milliseconds(1); + auto start = std::chrono::high_resolution_clock::now(); + while (std::chrono::high_resolution_clock::now() - start < duration) + { + for (int i = 0; i < 256; i++) // Empirically picked magic number! + { + nop(); nop(); nop(); nop(); nop(); nop(); nop(); nop(); + nop(); nop(); nop(); nop(); nop(); nop(); nop(); nop(); + nop(); nop(); nop(); nop(); nop(); nop(); nop(); nop(); + nop(); nop(); nop(); nop(); nop(); nop(); nop(); nop(); + if (work.num > 0) + { + return; + } + } + + if (scheduler->stealWork(this, rng(), stolen)) + { + std::unique_lock<std::mutex> lock(work.mutex); + work.tasks.emplace(std::move(stolen)); + work.num++; + return; + } + + std::this_thread::yield(); + } +} + +_Requires_lock_held_(lock) +void Scheduler::Worker::runUntilIdle(std::unique_lock<std::mutex> &lock) +{ + YARN_ASSERT(work.num == work.fibers.size() + work.tasks.size(), "work.num out of sync"); + while (work.fibers.size() > 0 || work.tasks.size() > 0) + { + // Note: we cannot take and store on the stack more than a single fiber + // or task at a time, as the Fiber may yield and these items may get + // held on suspended fiber stack. + + while (work.fibers.size() > 0) + { + work.num--; + auto fiber = take(work.fibers); + lock.unlock(); + idleFibers.push(currentFiber); + switchToFiber(fiber); + lock.lock(); + } + + if (work.tasks.size() > 0) + { + work.num--; + auto task = take(work.tasks); + lock.unlock(); + + // Run the task. + task(); + + // std::function<> can carry arguments with complex destructors. + // Ensure these are destructed outside of the lock. + task = Task(); + + lock.lock(); + } + } +} + +Scheduler::Fiber* Scheduler::Worker::createWorkerFiber() +{ + auto id = workerFibers.size() + 1; + auto fiber = Fiber::create(id, FiberStackSize, [&] { run(); }); + workerFibers.push_back(std::unique_ptr<Fiber>(fiber)); + return fiber; +} + +void Scheduler::Worker::switchToFiber(Fiber* to) +{ + auto from = currentFiber; + currentFiber = to; + from->switchTo(to); +} + +} // namespace yarn
diff --git a/src/Yarn/Scheduler.hpp b/src/Yarn/Scheduler.hpp new file mode 100644 index 0000000..7451698 --- /dev/null +++ b/src/Yarn/Scheduler.hpp
@@ -0,0 +1,337 @@ +// Copyright 2019 The SwiftShader Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef yarn_scheduler_hpp +#define yarn_scheduler_hpp + +#include "Debug.hpp" +#include "SAL.hpp" + +#include <array> +#include <atomic> +#include <condition_variable> +#include <functional> +#include <mutex> +#include <queue> +#include <thread> +#include <unordered_map> + +namespace yarn { + +class OSFiber; + +// Task is a unit of work for the scheduler. +using Task = std::function<void()>; + +// Scheduler asynchronously processes Tasks. +// A scheduler can be bound to one or more threads using the bind() method. +// Once bound to a thread, that thread can call yarn::schedule() to enqueue +// work tasks to be executed asynchronously. +// Scheduler are initially constructed in single-threaded mode. +// Call setWorkerThreadCount() to spawn dedicated worker threads. +class Scheduler +{ + class Worker; + +public: + Scheduler(); + ~Scheduler(); + + // get() returns the scheduler bound to the current thread. + static Scheduler *get(); + + // bind() binds this scheduler to the current thread. + // There must be no existing scheduler bound to the thread prior to calling. + void bind(); + + // unbind() unbinds the scheduler currently bound to the current thread. + // There must be a existing scheduler bound to the thread prior to calling. + static void unbind(); + + // enqueue() queues the task for asynchronous execution. + void enqueue(Task&& task); + + // setThreadInitializer() sets the worker thread initializer function which + // will be called for each new worker thread spawned. + // The initializer will only be called on newly created threads (call + // setThreadInitializer() before setWorkerThreadCount()). + void setThreadInitializer(const std::function<void()>& init); + + // getThreadInitializer() returns the thread initializer function set by + // setThreadInitializer(). + const std::function<void()>& getThreadInitializer(); + + // setWorkerThreadCount() adjusts the number of dedicated worker threads. + // A count of 0 puts the scheduler into single-threaded mode. + // Note: Currently the number of threads cannot be adjusted once tasks + // have been enqueued. This restriction may be lifted at a later time. + void setWorkerThreadCount(int count); + + // getWorkerThreadCount() returns the number of worker threads. + int getWorkerThreadCount(); + + // Fibers expose methods to perform cooperative multitasking and are + // automatically created by the Scheduler. + // + // The currently executing Fiber can be obtained by calling Fiber::current(). + // + // When execution becomes blocked, yield() can be called to suspend execution of + // the fiber and start executing other pending work. + // Once the block has been lifted, schedule() can be called to reschedule the + // Fiber on the same thread that previously executed it. + class Fiber + { + public: + ~Fiber(); + + // current() returns the currently executing fiber, or nullptr if called + // without a bound scheduler. + static Fiber* current(); + + // yield() suspends execution of this Fiber, allowing the thread to work + // on other tasks. + // yield() must only be called on the currently executing fiber. + void yield(); + + // schedule() reschedules the suspended Fiber for execution. + void schedule(); + + // id is the thread-unique identifier of the Fiber. + uint32_t const id; + + private: + friend class Scheduler; + + Fiber(OSFiber*, uint32_t id); + + // switchTo() switches execution to the given fiber. + // switchTo() must only be called on the currently executing fiber. + void switchTo(Fiber*); + + // create() constructs and returns a new fiber with the given identifier, + // stack size that will executed func when switched to. + static Fiber* create(uint32_t id, size_t stackSize, const std::function<void()>& func); + + // createFromCurrentThread() constructs and returns a new fiber with the + // given identifier for the current thread. + static Fiber* createFromCurrentThread(uint32_t id); + + OSFiber* const impl; + Worker* const worker; + }; + +private: + // Stack size in bytes of a new fiber. + // TODO: Make configurable so the default size can be reduced. + static constexpr size_t FiberStackSize = 1024 * 1024; + + // Maximum number of worker threads. + static constexpr size_t MaxWorkerThreads = 64; + + // TODO: Implement a queue that recycles elements to reduce number of + // heap allocations. + using TaskQueue = std::queue<Task>; + using FiberQueue = std::queue<Fiber*>; + + // Workers executes Tasks on a single thread. + // Once a task is started, it may yield to other tasks on the same Worker. + // Tasks are always resumed by the same Worker. + class Worker + { + public: + enum class Mode + { + // Worker will spawn a background thread to process tasks. + MultiThreaded, + + // Worker will execute tasks whenever it yields. + SingleThreaded, + }; + + Worker(Scheduler *scheduler, Mode mode, uint32_t id); + + // start() begins execution of the worker. + void start(); + + // stop() ceases execution of the worker, blocking until all pending + // tasks have fully finished. + void stop(); + + // yield() suspends execution of the current task, and looks for other + // tasks to start or continue execution. + void yield(Fiber* fiber); + + // enqueue(Fiber*) enqueues resuming of a suspended fiber. + void enqueue(Fiber* fiber); + + // enqueue(Task&&) enqueues a new, unstarted task. + void enqueue(Task&& task); + + // tryLock() attempts to lock the worker for task enqueing. + // If the lock was successful then true is returned, and the caller must + // call enqueueAndUnlock(). + bool tryLock(); + + // enqueueAndUnlock() enqueues the task and unlocks the worker. + // Must only be called after a call to tryLock() which returned true. + void enqueueAndUnlock(Task&& task); + + // flush() processes all pending tasks before returning. + void flush(); + + // dequeue() attempts to take a Task from the worker. Returns true if + // a task was taken and assigned to out, otherwise false. + bool dequeue(Task& out); + + // getCurrent() returns the Worker currently bound to the current + // thread. + static inline Worker* getCurrent(); + + // getCurrentFiber() returns the Fiber currently being executed. + inline Fiber* getCurrentFiber() const; + + // Unique identifier of the Worker. + const uint32_t id; + + private: + // run() is the task processing function for the worker. + // If the worker was constructed in Mode::MultiThreaded, run() will + // continue to process tasks until stop() is called. + // If the worker was constructed in Mode::SingleThreaded, run() call + // flush() and return. + void run(); + + // createWorkerFiber() creates a new fiber that when executed calls + // run(). + Fiber* createWorkerFiber(); + + // switchToFiber() switches execution to the given fiber. The fiber + // must belong to this worker. + void switchToFiber(Fiber*); + + // runUntilIdle() executes all pending tasks and then returns. + _Requires_lock_held_(lock) + void runUntilIdle(std::unique_lock<std::mutex> &lock); + + // waitForWork() blocks until new work is available, potentially calling + // spinForWork(). + _Requires_lock_held_(lock) + void waitForWork(std::unique_lock<std::mutex> &lock); + + // spinForWork() attempts to steal work from another Worker, and keeps + // the thread awake for a short duration. This reduces overheads of + // frequently putting the thread to sleep and re-waking. + void spinForWork(); + + // Work holds tasks and fibers that are enqueued on the Worker. + struct Work + { + std::atomic<uint64_t> num = { 0 }; // tasks.size() + fibers.size() + TaskQueue tasks; // guarded by mutex + FiberQueue fibers; // guarded by mutex + std::condition_variable added; + std::mutex mutex; + }; + + // https://en.wikipedia.org/wiki/Xorshift + class FastRnd + { + public: + inline uint64_t operator ()() + { + x ^= x << 13; + x ^= x >> 7; + x ^= x << 17; + return x; + } + private: + uint64_t x = std::chrono::system_clock::now().time_since_epoch().count(); + }; + + // The current worker bound to the current thread. + static thread_local Worker* current; + + Mode const mode; + Scheduler* const scheduler; + std::unique_ptr<Fiber> mainFiber; + Fiber* currentFiber = nullptr; + std::thread thread; + Work work; + FiberQueue idleFibers; // Fibers that have completed which can be reused. + std::vector<std::unique_ptr<Fiber>> workerFibers; // All fibers created by this worker. + FastRnd rng; + std::atomic<bool> shutdown = { false }; + }; + + // stealWork() attempts to steal a task from the worker with the given id. + // Returns true if a task was stolen and assigned to out, otherwise false. + bool stealWork(Worker* thief, uint64_t from, Task& out); + + // onBeginSpinning() is called when a Worker calls spinForWork(). + // The scheduler will prioritize this worker for new tasks to try to prevent + // it going to sleep. + void onBeginSpinning(int workerId); + + // The scheduler currently bound to the current thread. + static thread_local Scheduler* bound; + + std::function<void()> threadInitFunc; + std::mutex threadInitFuncMutex; + + std::array<std::atomic<int>, 8> spinningWorkers; + std::atomic<unsigned int> nextSpinningWorkerIdx = { 0x8000000 }; + + // TODO: Make this lot thread-safe so setWorkerThreadCount() can be called + // during execution of tasks. + std::atomic<unsigned int> nextEnqueueIndex = { 0 }; + unsigned int numWorkerThreads = 0; + std::array<Worker*, MaxWorkerThreads> workerThreads; + + std::mutex singleThreadedWorkerMutex; + std::unordered_map<std::thread::id, std::unique_ptr<Worker>> singleThreadedWorkers; +}; + +Scheduler::Worker* Scheduler::Worker::getCurrent() +{ + return Worker::current; +} + +Scheduler::Fiber* Scheduler::Worker::getCurrentFiber() const +{ + return currentFiber; +} + +// schedule() schedules the function f to be asynchronously called with the +// given arguments using the currently bound scheduler. +template<typename Function, typename ... Args> +inline void schedule(Function&& f, Args&& ... args) +{ + YARN_ASSERT_HAS_BOUND_SCHEDULER("yarn::schedule"); + auto scheduler = Scheduler::get(); + scheduler->enqueue(std::bind(std::forward<Function>(f), std::forward<Args>(args)...)); +} + +// schedule() schedules the function f to be asynchronously called using the +// currently bound scheduler. +template<typename Function> +inline void schedule(Function&& f) +{ + YARN_ASSERT_HAS_BOUND_SCHEDULER("yarn::schedule"); + auto scheduler = Scheduler::get(); + scheduler->enqueue(std::forward<Function>(f)); +} + +} // namespace yarn + +#endif // yarn_scheduler_hpp
diff --git a/src/Yarn/Scheduler_test.cpp b/src/Yarn/Scheduler_test.cpp new file mode 100644 index 0000000..1b48301 --- /dev/null +++ b/src/Yarn/Scheduler_test.cpp
@@ -0,0 +1,114 @@ +// Copyright 2019 The SwiftShader Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "Yarn_test.hpp" + +#include "Yarn/WaitGroup.hpp" + +TEST(WithoutBoundScheduler, SchedulerConstructAndDestruct) +{ + auto scheduler = new yarn::Scheduler(); + delete scheduler; +} + +TEST(WithoutBoundScheduler, SchedulerBindGetUnbind) +{ + auto scheduler = new yarn::Scheduler(); + scheduler->bind(); + auto got = yarn::Scheduler::get(); + ASSERT_EQ(scheduler, got); + scheduler->unbind(); + got = yarn::Scheduler::get(); + ASSERT_EQ(got, nullptr); + delete scheduler; +} + +TEST_P(WithBoundScheduler, SetAndGetWorkerThreadCount) +{ + ASSERT_EQ(yarn::Scheduler::get()->getWorkerThreadCount(), GetParam().numWorkerThreads); +} + +TEST_P(WithBoundScheduler, DestructWithPendingTasks) +{ + for (int i = 0; i < 10000; i++) + { + yarn::schedule([] {}); + } +} + +TEST_P(WithBoundScheduler, DestructWithPendingFibers) +{ + yarn::WaitGroup wg(1); + for (int i = 0; i < 10000; i++) + { + yarn::schedule([=] { wg.wait(); }); + } + wg.done(); + + auto scheduler = yarn::Scheduler::get(); + scheduler->unbind(); + delete scheduler; + + // Rebind a new scheduler so WithBoundScheduler::TearDown() is happy. + (new yarn::Scheduler())->bind(); +} + +TEST_P(WithBoundScheduler, FibersResumeOnSameYarnThread) +{ + yarn::WaitGroup fence(1); + yarn::WaitGroup wg(1000); + for (int i = 0; i < 1000; i++) + { + yarn::schedule([=] { + auto threadID = std::this_thread::get_id(); + fence.wait(); + ASSERT_EQ(threadID, std::this_thread::get_id()); + wg.done(); + }); + } + std::this_thread::sleep_for(std::chrono::milliseconds(10)); // just to try and get some tasks to yield. + fence.done(); + wg.wait(); +} + +TEST_P(WithBoundScheduler, FibersResumeOnSameStdThread) +{ + auto scheduler = yarn::Scheduler::get(); + + yarn::WaitGroup fence(1); + yarn::WaitGroup wg(1000); + + std::vector<std::thread> threads; + for (int i = 0; i < 1000; i++) + { + threads.push_back(std::thread([=] { + scheduler->bind(); + + auto threadID = std::this_thread::get_id(); + fence.wait(); + ASSERT_EQ(threadID, std::this_thread::get_id()); + wg.done(); + + scheduler->unbind(); + })); + } + std::this_thread::sleep_for(std::chrono::milliseconds(10)); // just to try and get some tasks to yield. + fence.done(); + wg.wait(); + + for (auto& thread : threads) + { + thread.join(); + } +} \ No newline at end of file
diff --git a/src/Yarn/Thread.cpp b/src/Yarn/Thread.cpp new file mode 100644 index 0000000..68dcf47 --- /dev/null +++ b/src/Yarn/Thread.cpp
@@ -0,0 +1,105 @@ +// Copyright 2019 The SwiftShader Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "Thread.hpp" + +#include "Trace.hpp" + +#include <cstdarg> + +#if defined(_WIN32) +# ifndef WIN32_LEAN_AND_MEAN +# define WIN32_LEAN_AND_MEAN +# endif +# include <windows.h> +#elif defined(__APPLE__) +# include <pthread.h> +# include <mach/thread_act.h> +# include <unistd.h> +#else +# include <pthread.h> +# include <unistd.h> +#endif + +namespace yarn { + +#if defined(_WIN32) + +void Thread::setName(const char* fmt, ...) +{ + static auto setThreadDescription = reinterpret_cast<HRESULT(WINAPI*)(HANDLE, PCWSTR)>(GetProcAddress(GetModuleHandle("kernelbase.dll"), "SetThreadDescription")); + if (setThreadDescription == nullptr) + { + return; + } + + char name[1024]; + va_list vararg; + va_start(vararg, fmt); + vsnprintf(name, sizeof(name), fmt, vararg); + va_end(vararg); + + wchar_t wname[1024]; + mbstowcs(wname, name, 1024); + setThreadDescription(GetCurrentThread(), wname); + YARN_NAME_THREAD("%s", name); +} + +unsigned int Thread::numLogicalCPUs() +{ + DWORD_PTR processAffinityMask = 1; + DWORD_PTR systemAffinityMask = 1; + + GetProcessAffinityMask(GetCurrentProcess(), &processAffinityMask, &systemAffinityMask); + + auto count = 0; + while (processAffinityMask > 0) + { + if (processAffinityMask & 1) + { + count++; + } + + processAffinityMask >>= 1; + } + return count; +} + +#else + +void Thread::setName(const char* fmt, ...) +{ + char name[1024]; + va_list vararg; + va_start(vararg, fmt); + vsnprintf(name, sizeof(name), fmt, vararg); + va_end(vararg); + +#if defined(__APPLE__) + pthread_setname_np(name); +#elif !defined(__Fuchsia__) + pthread_setname_np(pthread_self(), name); +#endif + + YARN_NAME_THREAD("%s", name); +} + +unsigned int Thread::numLogicalCPUs() +{ + return sysconf(_SC_NPROCESSORS_ONLN); +} + +#endif + +} // namespace yarn
diff --git a/src/Yarn/Thread.hpp b/src/Yarn/Thread.hpp new file mode 100644 index 0000000..878cb8e --- /dev/null +++ b/src/Yarn/Thread.hpp
@@ -0,0 +1,38 @@ +// Copyright 2019 The SwiftShader Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef yarn_thread_hpp +#define yarn_thread_hpp + +#include <bitset> + +namespace yarn { + +// Thread contains static methods that abstract OS-specific thread / cpu +// queries and control. +class Thread +{ +public: + // setName() sets the name of the currently executing thread for displaying + // in a debugger. + static void setName(const char* fmt, ...); + + // numLogicalCPUs() returns the number of available logical CPU cores for + // the system. + static unsigned int numLogicalCPUs(); +}; + +} // namespace yarn + +#endif // yarn_thread_hpp
diff --git a/src/Yarn/Ticket.hpp b/src/Yarn/Ticket.hpp new file mode 100644 index 0000000..ebe4d3b --- /dev/null +++ b/src/Yarn/Ticket.hpp
@@ -0,0 +1,267 @@ +// Copyright 2019 The SwiftShader Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef yarn_ticket_hpp +#define yarn_ticket_hpp + +#include "Pool.hpp" +#include "ConditionVariable.hpp" +#include "Scheduler.hpp" + +namespace yarn { + +// Ticket is a synchronization primitive used to serially order execution. +// +// Tickets exist in 3 mutually exclusive states: Waiting, Called and Finished. +// +// Tickets are obtained from a Ticket::Queue, using the Ticket::Queue::take() +// methods. The order in which tickets are taken from the queue dictates the +// order in which they are called. +// +// The first ticket to be taken from a queue will be in the 'called' state, +// others will be in the 'waiting' state until the previous ticket has finished. +// +// Ticket::wait() will block until the ticket is called. +// Ticket::done() sets the ticket into the 'finished' state and calls the next +// taken ticket from the queue. +// +// If a ticket is taken from a queue and does not have done() called before +// its last reference is dropped, it will implicitly call done(), calling the +// next ticket. +// +// Example: +// +// void runTasksConcurrentThenSerially(int numConcurrentTasks) +// { +// yarn::Ticket::Queue queue; +// for (int i = 0; i < numConcurrentTasks; i++) +// { +// auto ticket = queue.take(); +// yarn::schedule([=] { +// doConcurrentWork(); // <- function may be called concurrently +// ticket.wait(); // <- serialize tasks +// doSerialWork(); // <- function will not be called concurrently +// ticket.done(); // <- optional, as done() is called implicitly on dropping of last reference +// }); +// } +// } +class Ticket +{ + struct Shared; + struct Record; +public: + + // Queue hands out Tickets. + class Queue + { + public: + // take() returns a single ticket from the queue. + inline Ticket take(); + + // take() retrieves count tickets from the queue, calling f() with each + // retrieved ticket. + // F must be a function of the signature: void(Ticket&&) + template <typename F> + inline void take(size_t count, const F& f); + + private: + std::shared_ptr<Shared> shared = std::make_shared<Shared>(); + UnboundedPool<Record> pool; + }; + + inline Ticket() = default; + inline Ticket(const Ticket& other) = default; + inline Ticket(Ticket&& other) = default; + inline Ticket& operator = (const Ticket& other) = default; + + // wait() blocks until the ticket is called. + inline void wait() const; + + // done() marks the ticket as finished and calls the next ticket. + inline void done() const; + + // onCall() registers the function f to be invoked when this ticket is + // called. If the ticket is already called prior to calling onCall(), then + // f() will be executed immediately. + // F must be a function of the signature: void F() + template<typename F> + inline void onCall(F&& f) const; + +private: + // Internal doubly-linked-list data structure. One per ticket instance. + struct Record + { + inline ~Record(); + + inline void done(); + inline void callAndUnlock(std::unique_lock<std::mutex> &lock); + + ConditionVariable isCalledCondVar; + + std::shared_ptr<Shared> shared; + Record *next = nullptr; // guarded by shared->mutex + Record *prev = nullptr; // guarded by shared->mutex + inline void unlink(); // guarded by shared->mutex + Task onCall; // guarded by shared->mutex + bool isCalled = false; // guarded by shared->mutex + std::atomic<bool> isDone = { false }; + }; + + // Data shared between all tickets and the queue. + struct Shared + { + std::mutex mutex; + Record tail; + }; + + inline Ticket(Loan<Record>&& record); + + Loan<Record> record; +}; + +//////////////////////////////////////////////////////////////////////////////// +// Ticket +//////////////////////////////////////////////////////////////////////////////// + +Ticket::Ticket(Loan<Record>&& record) : record(std::move(record)) {} + +void Ticket::wait() const +{ + std::unique_lock<std::mutex> lock(record->shared->mutex); + record->isCalledCondVar.wait(lock, [this] { return record->isCalled; }); +} + +void Ticket::done() const +{ + record->done(); +} + +template<typename Function> +void Ticket::onCall(Function&& f) const +{ + std::unique_lock<std::mutex> lock(record->shared->mutex); + if (record->isCalled) + { + yarn::schedule(std::move(f)); + return; + } + if (record->onCall) + { + struct Joined + { + void operator() () const { a(); b(); } + Task a, b; + }; + record->onCall = std::move(Joined{ std::move(record->onCall), std::move(f) }); + } + else + { + record->onCall = std::move(f); + } +} + +//////////////////////////////////////////////////////////////////////////////// +// Ticket::Queue +//////////////////////////////////////////////////////////////////////////////// + +Ticket Ticket::Queue::take() +{ + Ticket out; + take(1, [&](Ticket&& ticket) { out = std::move(ticket); }); + return out; +} + +template <typename F> +void Ticket::Queue::take(size_t n, const F& f) +{ + Loan<Record> first, last; + pool.borrow(n, [&] (Loan<Record>&& record) { + Loan<Record> rec = std::move(record); + rec->shared = shared; + if (first.get() == nullptr) + { + first = rec; + } + if (last.get() != nullptr) + { + last->next = rec.get(); + rec->prev = last.get(); + } + last = rec; + f(std::move(Ticket(std::move(rec)))); + }); + last->next = &shared->tail; + std::unique_lock<std::mutex> lock(shared->mutex); + first->prev = shared->tail.prev; + shared->tail.prev = last.get(); + if (first->prev == nullptr) + { + first->callAndUnlock(lock); + } + else + { + first->prev->next = first.get(); + } +} + +//////////////////////////////////////////////////////////////////////////////// +// Ticket::Record +//////////////////////////////////////////////////////////////////////////////// + +Ticket::Record::~Record() +{ + if (shared != nullptr) + { + done(); + } +} + +void Ticket::Record::done() +{ + if (isDone.exchange(true)) { return; } + std::unique_lock<std::mutex> lock(shared->mutex); + auto callNext = (prev == nullptr && next != nullptr) ? next : nullptr; + unlink(); + if (callNext != nullptr) // lock needs to be held otherwise callNext might be destructed. + { + callNext->callAndUnlock(lock); + } +} + +void Ticket::Record::callAndUnlock(std::unique_lock<std::mutex> &lock) +{ + if (isCalled) { return; } + isCalled = true; + Task task; + std::swap(task, onCall); + isCalledCondVar.notify_all(); + lock.unlock(); + + if (task) + { + yarn::schedule(std::move(task)); + } +} + +void Ticket::Record::unlink() +{ + if (prev != nullptr) { prev->next = next; } + if (next != nullptr) { next->prev = prev; } + prev = nullptr; + next = nullptr; +} + +} // namespace yarn + +#endif // yarn_ticket_hpp
diff --git a/src/Yarn/Ticket_test.cpp b/src/Yarn/Ticket_test.cpp new file mode 100644 index 0000000..3bb4b3b --- /dev/null +++ b/src/Yarn/Ticket_test.cpp
@@ -0,0 +1,43 @@ +// Copyright 2019 The SwiftShader Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "Yarn_test.hpp" + +#include "Yarn/Ticket.hpp" + +TEST_P(WithBoundScheduler, Ticket) +{ + yarn::Ticket::Queue queue; + + constexpr int count = 1000; + std::atomic<int> next = { 0 }; + int result[count] = {}; + + for (int i = 0; i < count; i++) + { + auto ticket = queue.take(); + yarn::schedule([ticket, i, &result, &next] { + ticket.wait(); + result[next++] = i; + ticket.done(); + }); + } + + queue.take().wait(); + + for (int i = 0; i < count; i++) + { + ASSERT_EQ(result[i], i); + } +}
diff --git a/src/Yarn/Trace.cpp b/src/Yarn/Trace.cpp new file mode 100644 index 0000000..0b00e14 --- /dev/null +++ b/src/Yarn/Trace.cpp
@@ -0,0 +1,248 @@ +// Copyright 2019 The SwiftShader Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// The Trace API produces a trace event file that can be consumed with Chrome's +// about:tracing viewer. +// Documentation can be found at: +// https://www.chromium.org/developers/how-tos/trace-event-profiling-tool +// https://docs.google.com/document/d/1CvAClvFfyA5R-PhYUmn5OOQtYMH4h6I0nSsKchNAySU/edit + +#include "Trace.hpp" + +#include "Defer.hpp" +#include "Scheduler.hpp" +#include "Thread.hpp" + +#if YARN_TRACE_ENABLED + +#include <atomic> +#include <unordered_set> +#include <fstream> + +namespace +{ + +// Chrome traces can choke or error on very large trace files. +// Limit the number of events created to this number. +static constexpr int MaxEvents = 100000; + +uint64_t threadFiberID(uint32_t threadID, uint32_t fiberID) +{ + return static_cast<uint64_t>(threadID) * 31 + static_cast<uint64_t>(fiberID); +} + +} // anonymous namespace + +namespace yarn +{ + +Trace* Trace::get() +{ + static Trace trace; + return &trace; +} + +Trace::Trace() +{ + nameThread("main"); + thread = std::thread([&] { + Thread::setName("Trace worker"); + + auto out = std::fstream("chrome.trace", std::ios_base::out); + + out << "[" << std::endl; + defer(out << std::endl << "]" << std::endl); + + auto first = true; + for (int i = 0; i < MaxEvents; i++) + { + auto event = take(); + if (event->type() == Event::Type::Shutdown) + { + return; + } + if (!first) { out << "," << std::endl; }; + first = false; + out << "{" << std::endl; + event->write(out); + out << "}"; + } + + stopped = true; + + while(take()->type() != Event::Type::Shutdown) {} + }); +} + +Trace::~Trace() +{ + put(new Shutdown()); + thread.join(); +} + +void Trace::nameThread(const char* fmt, ...) +{ + if (stopped) { return; } + auto event = new NameThreadEvent(); + + va_list vararg; + va_start(vararg, fmt); + vsnprintf(event->name, Trace::MaxEventNameLength, fmt, vararg); + va_end(vararg); + + put(event); +} + +void Trace::beginEvent(const char* fmt, ...) +{ + if (stopped) { return; } + auto event = new BeginEvent(); + + va_list vararg; + va_start(vararg, fmt); + vsnprintf(event->name, Trace::MaxEventNameLength, fmt, vararg); + va_end(vararg); + + event->timestamp = timestamp(); + put(event); +} + +void Trace::endEvent() +{ + if (stopped) { return; } + auto event = new EndEvent(); + event->timestamp = timestamp(); + put(event); +} + +void Trace::beginAsyncEvent(uint32_t id, const char* fmt, ...) +{ + if (stopped) { return; } + auto event = new AsyncStartEvent(); + + va_list vararg; + va_start(vararg, fmt); + vsnprintf(event->name, Trace::MaxEventNameLength, fmt, vararg); + va_end(vararg); + + event->timestamp = timestamp(); + event->id = id; + put(event); +} + +void Trace::endAsyncEvent(uint32_t id, const char* fmt, ...) +{ + if (stopped) { return; } + auto event = new AsyncEndEvent(); + + va_list vararg; + va_start(vararg, fmt); + vsnprintf(event->name, Trace::MaxEventNameLength, fmt, vararg); + va_end(vararg); + + event->timestamp = timestamp(); + event->id = id; + put(event); +} + +uint64_t Trace::timestamp() +{ + auto now = std::chrono::high_resolution_clock::now(); + auto diff = std::chrono::duration_cast<std::chrono::microseconds>(now - createdAt); + return static_cast<uint64_t>(diff.count()); +} + +void Trace::put(Event* event) +{ + auto idx = eventQueueWriteIdx++ % eventQueues.size(); + auto &queue = eventQueues[idx]; + std::unique_lock<std::mutex> lock(queue.mutex); + auto notify = queue.data.size() == 0; + queue.data.push(std::unique_ptr<Event>(event)); + lock.unlock(); + if (notify) { queue.condition.notify_one(); } +} + +std::unique_ptr<Trace::Event> Trace::take() +{ + auto idx = eventQueueReadIdx++ % eventQueues.size(); + auto &queue = eventQueues[idx]; + std::unique_lock<std::mutex> lock(queue.mutex); + queue.condition.wait(lock, [&queue] { return queue.data.size() > 0; }); + auto event = std::move(queue.data.front()); + queue.data.pop(); + return event; +} + +#define QUOTE(x) "\"" << x << "\"" +#define INDENT " " + +Trace::Event::Event() : + threadID(std::hash<std::thread::id>()(std::this_thread::get_id())) +{ + if (auto fiber = Scheduler::Fiber::current()) + { + fiberID = fiber->id; + } +} + +void Trace::Event::write(std::ostream &out) const +{ + out << INDENT << QUOTE("name") << ": " << QUOTE(name) << "," << std::endl; + if (categories != nullptr) + { + out << INDENT << QUOTE("cat") << ": " << "\""; + auto first = true; + for (auto category = *categories; category != nullptr; category++) + { + if (!first) { out << ","; } + out << category; + } + out << "\"," << std::endl; + } + if (fiberID != 0) + { + out << INDENT << QUOTE("args") << ": " << "{" << std::endl + << INDENT << INDENT << QUOTE("fiber") << ": " << fiberID << std::endl + << INDENT << "}," << std::endl; + } + if (threadID != 0) + { + out << INDENT << QUOTE("tid") << ": " << threadFiberID(threadID, fiberID) << "," << std::endl; + } + out << INDENT << QUOTE("ph") << ": " << QUOTE(static_cast<char>(type())) << "," << std::endl + << INDENT << QUOTE("pid") << ": " << processID << "," << std::endl + << INDENT << QUOTE("ts") << ": " << timestamp << std::endl; +} + +void Trace::NameThreadEvent::write(std::ostream &out) const +{ + out << INDENT << QUOTE("name") << ": " << QUOTE("thread_name") << "," << std::endl + << INDENT << QUOTE("ph") << ": " << QUOTE("M") << "," << std::endl + << INDENT << QUOTE("pid") << ": " << processID << "," << std::endl + << INDENT << QUOTE("tid") << ": " << threadFiberID(threadID, fiberID) << "," << std::endl + << INDENT << QUOTE("args") << ": {" << QUOTE("name") << ": " << QUOTE(name) << "}" << std::endl; +} + +void Trace::AsyncEvent::write(std::ostream &out) const +{ + out << INDENT << QUOTE("id") << ": " << QUOTE(id) << "," << std::endl + << INDENT << QUOTE("cat") << ": " << QUOTE("async") << "," << std::endl; + Event::write(out); +} + + +} // namespace yarn + +#endif // YARN_TRACE_ENABLED \ No newline at end of file
diff --git a/src/Yarn/Trace.hpp b/src/Yarn/Trace.hpp new file mode 100644 index 0000000..3034ccf --- /dev/null +++ b/src/Yarn/Trace.hpp
@@ -0,0 +1,226 @@ +// Copyright 2019 The SwiftShader Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// The Trace API produces a trace event file that can be consumed with Chrome's +// chrome://tracing viewer. +// Documentation can be found at: +// https://www.chromium.org/developers/how-tos/trace-event-profiling-tool +// https://docs.google.com/document/d/1CvAClvFfyA5R-PhYUmn5OOQtYMH4h6I0nSsKchNAySU/edit + +#define YARN_TRACE_ENABLED 0 + +#if YARN_TRACE_ENABLED + +#include <array> +#include <atomic> +#include <chrono> +#include <condition_variable> +#include <cstdarg> +#include <cstring> +#include <mutex> +#include <ostream> +#include <queue> +#include <thread> + +namespace yarn +{ + +// Trace writes a trace event file into the current working directory that can +// be consumed with Chrome's chrome://tracing viewer. +// Use the YARN_* macros below instead of using this class directly. +class Trace +{ +public: + static constexpr size_t MaxEventNameLength = 64; + + static Trace* get(); + + void nameThread(const char* fmt, ...); + void beginEvent(const char* fmt, ...); + void endEvent(); + void beginAsyncEvent(uint32_t id, const char* fmt, ...); + void endAsyncEvent(uint32_t id, const char* fmt, ...); + + class ScopedEvent + { + public: + inline ScopedEvent(const char* fmt, ...); + inline ~ScopedEvent(); + private: + Trace * const trace; + }; + + class ScopedAsyncEvent + { + public: + inline ScopedAsyncEvent(uint32_t id, const char* fmt, ...); + inline ~ScopedAsyncEvent(); + private: + Trace * const trace; + const uint32_t id; + std::string name; + }; + +private: + Trace(); + ~Trace(); + Trace(const Trace&) = delete; + Trace& operator = (const Trace&) = delete; + + struct Event + { + enum class Type : uint8_t + { + Begin = 'B', + End = 'E', + Complete = 'X', + Instant = 'i', + Counter = 'C', + AsyncStart = 'b', + AsyncInstant = 'n', + AsyncEnd = 'e', + FlowStart = 's', + FlowStep = 't', + FlowEnd = 'f', + Sample = 'P', + ObjectCreated = 'N', + ObjectSnapshot = 'O', + ObjectDestroyed = 'D', + Metadata = 'M', + GlobalMemoryDump = 'V', + ProcessMemoryDump = 'v', + Mark = 'R', + ClockSync = 'c', + ContextEnter = '(', + ContextLeave = ')', + + // Internal types + Shutdown = 'S', + }; + + Event(); + virtual ~Event() = default; + virtual Type type() const = 0; + virtual void write(std::ostream &out) const; + + char name[MaxEventNameLength] = {}; + const char **categories = nullptr; + uint64_t timestamp = 0; // in microseconds + uint32_t processID = 0; + uint32_t threadID; + uint32_t fiberID; + }; + + struct BeginEvent : public Event { Type type() const override { return Type::Begin; } }; + struct EndEvent : public Event { Type type() const override { return Type::End; } }; + struct MetadataEvent : public Event { Type type() const override { return Type::Metadata; } }; + struct Shutdown : public Event { Type type() const override { return Type::Shutdown; } }; + + struct AsyncEvent : public Event + { + void write(std::ostream &out) const override; + uint32_t id; + }; + + struct AsyncStartEvent : public AsyncEvent { Type type() const override { return Type::AsyncStart; } }; + struct AsyncEndEvent : public AsyncEvent { Type type() const override { return Type::AsyncEnd; } }; + + struct NameThreadEvent : public MetadataEvent + { + void write(std::ostream &out) const override; + }; + + uint64_t timestamp(); // in microseconds + + void put(Event*); + std::unique_ptr<Event> take(); + + struct EventQueue + { + std::queue< std::unique_ptr<Event> > data; // guarded by mutes + std::condition_variable condition; + std::mutex mutex; + }; + std::array<EventQueue, 1> eventQueues; // TODO: Increasing this from 1 can cause events to go out of order. Investigate, fix. + std::atomic<unsigned int> eventQueueWriteIdx = { 0 }; + unsigned int eventQueueReadIdx = 0; + std::chrono::time_point<std::chrono::high_resolution_clock> createdAt = std::chrono::high_resolution_clock::now(); + std::thread thread; + std::atomic<bool> stopped = { false }; +}; + +Trace::ScopedEvent::ScopedEvent(const char* fmt, ...) : trace(Trace::get()) +{ + if (trace != nullptr) + { + char name[Trace::MaxEventNameLength]; + va_list vararg; + va_start(vararg, fmt); + vsnprintf(name, Trace::MaxEventNameLength, fmt, vararg); + va_end(vararg); + + trace->beginEvent(name); + } +} + +Trace::ScopedEvent::~ScopedEvent() +{ + if (trace != nullptr) + { + trace->endEvent(); + } +} + +Trace::ScopedAsyncEvent::ScopedAsyncEvent(uint32_t id, const char* fmt, ...) : trace(Trace::get()), id(id) +{ + if (trace != nullptr) + { + char buf[Trace::MaxEventNameLength]; + va_list vararg; + va_start(vararg, fmt); + vsnprintf(buf, Trace::MaxEventNameLength, fmt, vararg); + va_end(vararg); + name = buf; + + trace->beginAsyncEvent(id, "%s", buf); + } +} + +Trace::ScopedAsyncEvent::~ScopedAsyncEvent() +{ + if (trace != nullptr) + { + trace->endAsyncEvent(id, "%s", name.c_str()); + } +} + +} // namespace yarn + +#define YARN_CONCAT_(a, b) a ## b +#define YARN_CONCAT(a, b) YARN_CONCAT_(a,b) +#define YARN_SCOPED_EVENT(...) yarn::Trace::ScopedEvent YARN_CONCAT(scoped_event, __LINE__)(__VA_ARGS__); +#define YARN_BEGIN_ASYNC_EVENT(id, ...) do { if (auto t = yarn::Trace::get()) { t->beginAsyncEvent(id, __VA_ARGS__); } } while(false); +#define YARN_END_ASYNC_EVENT(id, ...) do { if (auto t = yarn::Trace::get()) { t->endAsyncEvent(id, __VA_ARGS__); } } while(false); +#define YARN_SCOPED_ASYNC_EVENT(id, ...) yarn::Trace::ScopedAsyncEvent YARN_CONCAT(defer_, __LINE__)(id, __VA_ARGS__); +#define YARN_NAME_THREAD(...) do { if (auto t = yarn::Trace::get()) { t->nameThread(__VA_ARGS__); } } while(false); + +#else // YARN_TRACE_ENABLED + +#define YARN_SCOPED_EVENT(...) +#define YARN_BEGIN_ASYNC_EVENT(id, ...) +#define YARN_END_ASYNC_EVENT(id, ...) +#define YARN_SCOPED_ASYNC_EVENT(id, ...) +#define YARN_NAME_THREAD(...) + +#endif // YARN_TRACE_ENABLED
diff --git a/src/Yarn/Utils.hpp b/src/Yarn/Utils.hpp new file mode 100644 index 0000000..87744d5 --- /dev/null +++ b/src/Yarn/Utils.hpp
@@ -0,0 +1,54 @@ +// Copyright 2019 The SwiftShader Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef yarn_util_hpp +#define yarn_util_hpp + +#include "Scheduler.hpp" +#include "WaitGroup.hpp" + +namespace yarn { + +// parallelize() is used to split a number of work items into N smaller batches +// which can be processed in parallel with the function f(). +// numTotal is the total number of work items to process. +// numPerTask is the maximum number of work items to process per call to f(). +// There will always be at least one call to f(). +// F must be a function with the signature: +// void(COUNTER taskIndex, COUNTER first, COUNTER count) +// COUNTER is any integer type. +template <typename F, typename COUNTER> +inline void parallelize(COUNTER numTotal, COUNTER numPerTask, const F& f) +{ + auto numTasks = (numTotal + numPerTask - 1) / numPerTask; + WaitGroup wg(numTasks - 1); + for (unsigned int task = 1; task < numTasks; task++) + { + schedule([=] { + auto first = task * numPerTask; + auto count = std::min(first + numPerTask, numTotal) - first; + f(task, first, count); + wg.done(); + }); + } + + // Run the first chunk on this fiber to reduce the amount of time spent + // waiting. + f(0, 0, std::min(numPerTask, numTotal)); + wg.wait(); +} + +} // namespace yarn + +#endif // yarn_util_hpp
diff --git a/src/Yarn/WaitGroup.hpp b/src/Yarn/WaitGroup.hpp new file mode 100644 index 0000000..a8c6b14 --- /dev/null +++ b/src/Yarn/WaitGroup.hpp
@@ -0,0 +1,108 @@ +// Copyright 2019 The SwiftShader Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef yarn_waitgroup_hpp +#define yarn_waitgroup_hpp + +#include "ConditionVariable.hpp" +#include "Debug.hpp" + +#include <atomic> +#include <mutex> + +namespace yarn { + +// WaitGroup is a synchronization primitive that holds an internal counter that +// can incremented, decremented and waited on until it reaches 0. +// WaitGroups can be used as a simple mechanism for waiting on a number of +// concurrently execute a number of tasks to complete. +// +// Example: +// +// void runTasksConcurrently(int numConcurrentTasks) +// { +// // Construct the WaitGroup with an initial count of numConcurrentTasks. +// yarn::WaitGroup wg(numConcurrentTasks); +// for (int i = 0; i < numConcurrentTasks; i++) +// { +// // Schedule a task to be run asynchronously. +// // These may all be run concurrently. +// yarn::schedule([=] { +// // Once the task has finished, decrement the waitgroup counter +// // to signal that this has completed. +// defer(wg.done()); +// doSomeWork(); +// }); +// } +// // Block until all tasks have completed. +// wg.wait(); +// } +class WaitGroup +{ +public: + // Constructs the WaitGroup with the specified initial count. + inline WaitGroup(unsigned int initialCount = 0); + + // add() increments the internal counter by count. + inline void add(unsigned int count = 1) const; + + // done() decrements the internal counter by one. + // Returns true if the internal count has reached zero. + inline bool done() const; + + // wait() blocks until the WaitGroup counter reaches zero. + inline void wait() const; + +private: + struct Data + { + std::atomic<unsigned int> count = { 0 }; + ConditionVariable condition; + std::mutex mutex; + }; + const std::shared_ptr<Data> data = std::make_shared<Data>(); +}; + +inline WaitGroup::WaitGroup(unsigned int initialCount /* = 0 */) +{ + data->count = initialCount; +} + +void WaitGroup::add(unsigned int count /* = 1 */) const +{ + data->count += count; +} + +bool WaitGroup::done() const +{ + YARN_ASSERT(data->count > 0, "yarn::WaitGroup::done() called too many times"); + auto count = --data->count; + if (count == 0) + { + std::unique_lock<std::mutex> lock(data->mutex); + data->condition.notify_all(); + return true; + } + return false; +} + +void WaitGroup::wait() const +{ + std::unique_lock<std::mutex> lock(data->mutex); + data->condition.wait(lock, [this]{ return data->count == 0; }); +} + +} // namespace yarn + +#endif // yarn_waitgroup_hpp
diff --git a/src/Yarn/WaitGroup_test.cpp b/src/Yarn/WaitGroup_test.cpp new file mode 100644 index 0000000..d2048ff --- /dev/null +++ b/src/Yarn/WaitGroup_test.cpp
@@ -0,0 +1,61 @@ +// Copyright 2019 The SwiftShader Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "Yarn_test.hpp" + +#include "Yarn/WaitGroup.hpp" + +TEST(WithoutBoundScheduler, WaitGroupDone) +{ + yarn::WaitGroup wg(2); // Should not require a scheduler. + wg.done(); + wg.done(); +} + +#if YARN_DEBUG_ENABLED +TEST(WithoutBoundScheduler, WaitGroupDoneTooMany) +{ + yarn::WaitGroup wg(2); // Should not require a scheduler. + wg.done(); + wg.done(); + EXPECT_DEATH(wg.done(), "done\\(\\) called too many times"); +} +#endif // YARN_DEBUG_ENABLED + +TEST_P(WithBoundScheduler, WaitGroup_OneTask) +{ + yarn::WaitGroup wg(1); + std::atomic<int> counter = {0}; + yarn::schedule([&counter, wg] { + counter++; + wg.done(); + }); + wg.wait(); + ASSERT_EQ(counter.load(), 1); +} + +TEST_P(WithBoundScheduler, WaitGroup_10Tasks) +{ + yarn::WaitGroup wg(10); + std::atomic<int> counter = {0}; + for (int i = 0; i < 10; i++) + { + yarn::schedule([&counter, wg] { + counter++; + wg.done(); + }); + } + wg.wait(); + ASSERT_EQ(counter.load(), 10); +}
diff --git a/src/Yarn/Yarn_test.cpp b/src/Yarn/Yarn_test.cpp new file mode 100644 index 0000000..a3ace60 --- /dev/null +++ b/src/Yarn/Yarn_test.cpp
@@ -0,0 +1,30 @@ +// Copyright 2019 The SwiftShader Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "Yarn_test.hpp" + +INSTANTIATE_TEST_SUITE_P(SchedulerParams, WithBoundScheduler, testing::Values( + SchedulerParams{0}, // Single-threaded mode test + SchedulerParams{1}, // Single worker thread + SchedulerParams{2}, // 2 worker threads... + SchedulerParams{4}, + SchedulerParams{8}, + SchedulerParams{64} +)); + +int main(int argc, char **argv) +{ + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +}
diff --git a/src/Yarn/Yarn_test.hpp b/src/Yarn/Yarn_test.hpp new file mode 100644 index 0000000..218b22f --- /dev/null +++ b/src/Yarn/Yarn_test.hpp
@@ -0,0 +1,55 @@ +// Copyright 2019 The SwiftShader Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +#include "Yarn/Scheduler.hpp" + +// SchedulerParams holds Scheduler construction parameters for testing. +struct SchedulerParams +{ + int numWorkerThreads; + + friend std::ostream& operator<<(std::ostream& os, const SchedulerParams& params) { + return os << "SchedulerParams{" << + "numWorkerThreads: " << params.numWorkerThreads << + "}"; + } +}; + +// WithoutBoundScheduler is a test fixture that does not bind a scheduler. +class WithoutBoundScheduler : public testing::Test {}; + +// WithBoundScheduler is a parameterized test fixture that performs tests with +// a bound scheduler using a number of different configurations. +class WithBoundScheduler : public testing::TestWithParam<SchedulerParams> +{ +public: + void SetUp() override + { + auto ¶ms = GetParam(); + + auto scheduler = new yarn::Scheduler(); + scheduler->bind(); + scheduler->setWorkerThreadCount(params.numWorkerThreads); + } + + void TearDown() override + { + auto scheduler = yarn::Scheduler::get(); + scheduler->unbind(); + delete scheduler; + } +};
diff --git a/tests/kokoro/gcp_ubuntu/continuous.sh b/tests/kokoro/gcp_ubuntu/continuous.sh index d7ee2ed..0021c2f 100644 --- a/tests/kokoro/gcp_ubuntu/continuous.sh +++ b/tests/kokoro/gcp_ubuntu/continuous.sh
@@ -23,6 +23,7 @@ cd .. # Some tests must be run from project root build/ReactorUnitTests +build/yarn-unittests build/gles-unittests if [ "${REACTOR_BACKEND}" != "Subzero" ]; then
diff --git a/tests/kokoro/gcp_windows/continuous.bat b/tests/kokoro/gcp_windows/continuous.bat index f874e23..0bd0196 100644 --- a/tests/kokoro/gcp_windows/continuous.bat +++ b/tests/kokoro/gcp_windows/continuous.bat
@@ -28,6 +28,9 @@ if !ERRORLEVEL! neq 0 exit /b !ERRORLEVEL! SET SWIFTSHADER_DISABLE_DEBUGGER_WAIT_DIALOG=1 +build\Debug\yarn-unittests.exe +if !ERRORLEVEL! neq 0 exit /b !ERRORLEVEL! + build\Debug\gles-unittests.exe if !ERRORLEVEL! neq 0 exit /b !ERRORLEVEL!
diff --git a/tests/kokoro/macos/continuous.sh b/tests/kokoro/macos/continuous.sh index a5675c0..ca1ae26 100755 --- a/tests/kokoro/macos/continuous.sh +++ b/tests/kokoro/macos/continuous.sh
@@ -23,6 +23,7 @@ cd .. # Some tests must be run from project root build/ReactorUnitTests +build/yarn-unittests build/gles-unittests if [ "${REACTOR_BACKEND}" != "Subzero" ]; then
diff --git a/third_party/marl/Android.bp b/third_party/marl/Android.bp deleted file mode 100644 index 52e1b4f..0000000 --- a/third_party/marl/Android.bp +++ /dev/null
@@ -1,46 +0,0 @@ -// -// Copyright (C) 2019 The Android Open Source Project -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// - -cc_library_static { - name: "swiftshader_marl", - vendor_available: true, - sdk_version: "27", - srcs: [ - "src/debug.cpp", - "src/scheduler.cpp", - "src/thread.cpp", - "src/trace.cpp", - ], - local_include_dirs: [ - "src", - ], - export_include_dirs: [ - "include", - ], - include_dirs: [ - "external/swiftshader/third_party/marl/include" - ], - arch: { - arm: { srcs: ["src/osfiber_arm.c", "src/osfiber_asm_arm.S"], }, - arm64: { srcs: ["src/osfiber_aarch64.c", "src/osfiber_asm_aarch64.S"], }, - x86: { srcs: ["src/osfiber_x86.c", "src/osfiber_asm_x86.S"], }, - x86_64: { srcs: ["src/osfiber_x64.c", "src/osfiber_asm_x64.S"], }, - }, - cflags: [ - "-Wno-unused-parameter", - ], - stl: "libc++_static", -}
diff --git a/third_party/marl/BUILD.gn b/third_party/marl/BUILD.gn deleted file mode 100644 index f4457c5..0000000 --- a/third_party/marl/BUILD.gn +++ /dev/null
@@ -1,103 +0,0 @@ -# Copyright 2019 The SwiftShader Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import("../../src/swiftshader.gni") - -import("//testing/test.gni") - -config("marl_config") { - include_dirs = [ "include" ] -} - -swiftshader_source_set("Marl_headers") { - sources = [ - "include/marl/conditionvariable.h", - "include/marl/containers.h", - "include/marl/debug.h", - "include/marl/defer.h", - "include/marl/finally.h", - "include/marl/osfiber.h", - "include/marl/pool.h", - "include/marl/sal.h", - "include/marl/scheduler.h", - "include/marl/thread.h", - "include/marl/ticket.h", - "include/marl/trace.h", - "include/marl/utils.h", - "include/marl/waitgroup.h", - ] - - if (is_win) { - sources += [ "include/marl/osfiber_windows.h" ] - } else { - sources += [ - "osfiber_asm.h", - "osfiber_asm_aarch64.h", - "osfiber_asm_arm.h", - "osfiber_asm_ppc64.h", - "osfiber_asm_x64.h", - "osfiber_asm_x86.h", - ] - } - - public_configs = [ - ":marl_config", - ] -} - -swiftshader_source_set("Marl") { - sources = [ - "src/debug.cpp", - "src/scheduler.cpp", - "src/thread.cpp", - "src/trace.cpp", - ] - - if (!is_win) { - sources += [ - "src/osfiber_aarch64.c", - "src/osfiber_arm.c", - "src/osfiber_asm_aarch64.S", - "src/osfiber_asm_arm.S", - "src/osfiber_asm_ppc64.S", - "src/osfiber_asm_x64.S", - "src/osfiber_asm_x86.S", - "src/osfiber_x64.c", - "src/osfiber_x86.c", - ] - } - - public_deps = [ - ":Marl_headers", - ] -} - -test("marl_unittests") { - sources = [ - "src/conditionvariable_test.cpp", - "src/containers_test.cpp", - "src/defer_test.cpp", - "src/osfiber_test.cpp", - "src/pool_test.cpp", - "src/scheduler_test.cpp", - "src/ticket_test.cpp", - "src/waitgroup_test.cpp", - "src/marl_test.cpp", - ] - deps = [ - ":Marl", - "//testing/gtest", - "//testing/gmock", - ] -}