| // Copyright (c) 2018 Google LLC | |
| // | |
| // Licensed under the Apache License, Version 2.0 (the "License"); | |
| // you may not use this file except in compliance with the License. | |
| // You may obtain a copy of the License at | |
| // | |
| // http://www.apache.org/licenses/LICENSE-2.0 | |
| // | |
| // Unless required by applicable law or agreed to in writing, software | |
| // distributed under the License is distributed on an "AS IS" BASIS, | |
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
| // See the License for the specific language governing permissions and | |
| // limitations under the License. | |
| #include "upgrade_memory_model.h" | |
| #include <utility> | |
| #include "source/opt/ir_builder.h" | |
| #include "source/opt/ir_context.h" | |
| #include "source/spirv_constant.h" | |
| #include "source/util/make_unique.h" | |
| #include "source/util/string_utils.h" | |
| namespace spvtools { | |
| namespace opt { | |
| Pass::Status UpgradeMemoryModel::Process() { | |
| // TODO: This pass needs changes to support cooperative matrices. | |
| if (context()->get_feature_mgr()->HasCapability( | |
| spv::Capability::CooperativeMatrixNV)) { | |
| return Pass::Status::SuccessWithoutChange; | |
| } | |
| // Only update Logical GLSL450 to Logical VulkanKHR. | |
| Instruction* memory_model = get_module()->GetMemoryModel(); | |
| if (memory_model->GetSingleWordInOperand(0u) != | |
| uint32_t(spv::AddressingModel::Logical) || | |
| memory_model->GetSingleWordInOperand(1u) != | |
| uint32_t(spv::MemoryModel::GLSL450)) { | |
| return Pass::Status::SuccessWithoutChange; | |
| } | |
| UpgradeMemoryModelInstruction(); | |
| UpgradeInstructions(); | |
| CleanupDecorations(); | |
| UpgradeBarriers(); | |
| UpgradeMemoryScope(); | |
| return Pass::Status::SuccessWithChange; | |
| } | |
| void UpgradeMemoryModel::UpgradeMemoryModelInstruction() { | |
| // Overall changes necessary: | |
| // 1. Add the OpExtension. | |
| // 2. Add the OpCapability. | |
| // 3. Modify the memory model. | |
| Instruction* memory_model = get_module()->GetMemoryModel(); | |
| context()->AddCapability(MakeUnique<Instruction>( | |
| context(), spv::Op::OpCapability, 0, 0, | |
| std::initializer_list<Operand>{ | |
| {SPV_OPERAND_TYPE_CAPABILITY, | |
| {uint32_t(spv::Capability::VulkanMemoryModelKHR)}}})); | |
| const std::string extension = "SPV_KHR_vulkan_memory_model"; | |
| std::vector<uint32_t> words = spvtools::utils::MakeVector(extension); | |
| context()->AddExtension( | |
| MakeUnique<Instruction>(context(), spv::Op::OpExtension, 0, 0, | |
| std::initializer_list<Operand>{ | |
| {SPV_OPERAND_TYPE_LITERAL_STRING, words}})); | |
| memory_model->SetInOperand(1u, {uint32_t(spv::MemoryModel::VulkanKHR)}); | |
| } | |
| void UpgradeMemoryModel::UpgradeInstructions() { | |
| // Coherent and Volatile decorations are deprecated. Remove them and replace | |
| // with flags on the memory/image operations. The decorations can occur on | |
| // OpVariable, OpFunctionParameter (of pointer type) and OpStructType (member | |
| // decoration). Trace from the decoration target(s) to the final memory/image | |
| // instructions. Additionally, Workgroup storage class variables and function | |
| // parameters are implicitly coherent in GLSL450. | |
| // Upgrade modf and frexp first since they generate new stores. | |
| // In SPIR-V 1.4 or later, normalize OpCopyMemory* access operands. | |
| for (auto& func : *get_module()) { | |
| func.ForEachInst([this](Instruction* inst) { | |
| if (inst->opcode() == spv::Op::OpExtInst) { | |
| auto ext_inst = inst->GetSingleWordInOperand(1u); | |
| if (ext_inst == GLSLstd450Modf || ext_inst == GLSLstd450Frexp) { | |
| auto import = | |
| get_def_use_mgr()->GetDef(inst->GetSingleWordInOperand(0u)); | |
| if (import->GetInOperand(0u).AsString() == "GLSL.std.450") { | |
| UpgradeExtInst(inst); | |
| } | |
| } | |
| } else if (get_module()->version() >= SPV_SPIRV_VERSION_WORD(1, 4)) { | |
| if (inst->opcode() == spv::Op::OpCopyMemory || | |
| inst->opcode() == spv::Op::OpCopyMemorySized) { | |
| uint32_t start_operand = | |
| inst->opcode() == spv::Op::OpCopyMemory ? 2u : 3u; | |
| if (inst->NumInOperands() > start_operand) { | |
| auto num_access_words = MemoryAccessNumWords( | |
| inst->GetSingleWordInOperand(start_operand)); | |
| if ((num_access_words + start_operand) == inst->NumInOperands()) { | |
| // There is a single memory access operand. Duplicate it to have a | |
| // separate operand for both source and target. | |
| for (uint32_t i = 0; i < num_access_words; ++i) { | |
| auto operand = inst->GetInOperand(start_operand + i); | |
| inst->AddOperand(std::move(operand)); | |
| } | |
| } | |
| } else { | |
| // Add two memory access operands. | |
| inst->AddOperand({SPV_OPERAND_TYPE_MEMORY_ACCESS, | |
| {uint32_t(spv::MemoryAccessMask::MaskNone)}}); | |
| inst->AddOperand({SPV_OPERAND_TYPE_MEMORY_ACCESS, | |
| {uint32_t(spv::MemoryAccessMask::MaskNone)}}); | |
| } | |
| } | |
| } | |
| }); | |
| } | |
| UpgradeMemoryAndImages(); | |
| UpgradeAtomics(); | |
| } | |
| void UpgradeMemoryModel::UpgradeMemoryAndImages() { | |
| for (auto& func : *get_module()) { | |
| func.ForEachInst([this](Instruction* inst) { | |
| bool is_coherent = false; | |
| bool is_volatile = false; | |
| bool src_coherent = false; | |
| bool src_volatile = false; | |
| bool dst_coherent = false; | |
| bool dst_volatile = false; | |
| uint32_t start_operand = 0u; | |
| spv::Scope scope = spv::Scope::QueueFamilyKHR; | |
| spv::Scope src_scope = spv::Scope::QueueFamilyKHR; | |
| spv::Scope dst_scope = spv::Scope::QueueFamilyKHR; | |
| switch (inst->opcode()) { | |
| case spv::Op::OpLoad: | |
| case spv::Op::OpStore: | |
| std::tie(is_coherent, is_volatile, scope) = | |
| GetInstructionAttributes(inst->GetSingleWordInOperand(0u)); | |
| break; | |
| case spv::Op::OpImageRead: | |
| case spv::Op::OpImageSparseRead: | |
| case spv::Op::OpImageWrite: | |
| std::tie(is_coherent, is_volatile, scope) = | |
| GetInstructionAttributes(inst->GetSingleWordInOperand(0u)); | |
| break; | |
| case spv::Op::OpCopyMemory: | |
| case spv::Op::OpCopyMemorySized: | |
| std::tie(dst_coherent, dst_volatile, dst_scope) = | |
| GetInstructionAttributes(inst->GetSingleWordInOperand(0u)); | |
| std::tie(src_coherent, src_volatile, src_scope) = | |
| GetInstructionAttributes(inst->GetSingleWordInOperand(1u)); | |
| break; | |
| default: | |
| break; | |
| } | |
| switch (inst->opcode()) { | |
| case spv::Op::OpLoad: | |
| UpgradeFlags(inst, 1u, is_coherent, is_volatile, kVisibility, | |
| kMemory); | |
| break; | |
| case spv::Op::OpStore: | |
| UpgradeFlags(inst, 2u, is_coherent, is_volatile, kAvailability, | |
| kMemory); | |
| break; | |
| case spv::Op::OpCopyMemory: | |
| case spv::Op::OpCopyMemorySized: | |
| start_operand = inst->opcode() == spv::Op::OpCopyMemory ? 2u : 3u; | |
| if (get_module()->version() >= SPV_SPIRV_VERSION_WORD(1, 4)) { | |
| // There are guaranteed to be two memory access operands at this | |
| // point so treat source and target separately. | |
| uint32_t num_access_words = MemoryAccessNumWords( | |
| inst->GetSingleWordInOperand(start_operand)); | |
| UpgradeFlags(inst, start_operand, dst_coherent, dst_volatile, | |
| kAvailability, kMemory); | |
| UpgradeFlags(inst, start_operand + num_access_words, src_coherent, | |
| src_volatile, kVisibility, kMemory); | |
| } else { | |
| UpgradeFlags(inst, start_operand, dst_coherent, dst_volatile, | |
| kAvailability, kMemory); | |
| UpgradeFlags(inst, start_operand, src_coherent, src_volatile, | |
| kVisibility, kMemory); | |
| } | |
| break; | |
| case spv::Op::OpImageRead: | |
| case spv::Op::OpImageSparseRead: | |
| UpgradeFlags(inst, 2u, is_coherent, is_volatile, kVisibility, kImage); | |
| break; | |
| case spv::Op::OpImageWrite: | |
| UpgradeFlags(inst, 3u, is_coherent, is_volatile, kAvailability, | |
| kImage); | |
| break; | |
| default: | |
| break; | |
| } | |
| // |is_coherent| is never used for the same instructions as | |
| // |src_coherent| and |dst_coherent|. | |
| if (is_coherent) { | |
| inst->AddOperand( | |
| {SPV_OPERAND_TYPE_SCOPE_ID, {GetScopeConstant(scope)}}); | |
| } | |
| if (get_module()->version() >= SPV_SPIRV_VERSION_WORD(1, 4)) { | |
| // There are two memory access operands. The first is for the target and | |
| // the second is for the source. | |
| if (dst_coherent || src_coherent) { | |
| start_operand = inst->opcode() == spv::Op::OpCopyMemory ? 2u : 3u; | |
| std::vector<Operand> new_operands; | |
| uint32_t num_access_words = | |
| MemoryAccessNumWords(inst->GetSingleWordInOperand(start_operand)); | |
| // The flags were already updated so subtract if we're adding a | |
| // scope. | |
| if (dst_coherent) --num_access_words; | |
| for (uint32_t i = 0; i < start_operand + num_access_words; ++i) { | |
| new_operands.push_back(inst->GetInOperand(i)); | |
| } | |
| // Add the target scope if necessary. | |
| if (dst_coherent) { | |
| new_operands.push_back( | |
| {SPV_OPERAND_TYPE_SCOPE_ID, {GetScopeConstant(dst_scope)}}); | |
| } | |
| // Copy the remaining current operands. | |
| for (uint32_t i = start_operand + num_access_words; | |
| i < inst->NumInOperands(); ++i) { | |
| new_operands.push_back(inst->GetInOperand(i)); | |
| } | |
| // Add the source scope if necessary. | |
| if (src_coherent) { | |
| new_operands.push_back( | |
| {SPV_OPERAND_TYPE_SCOPE_ID, {GetScopeConstant(src_scope)}}); | |
| } | |
| inst->SetInOperands(std::move(new_operands)); | |
| } | |
| } else { | |
| // According to SPV_KHR_vulkan_memory_model, if both available and | |
| // visible flags are used the first scope operand is for availability | |
| // (writes) and the second is for visibility (reads). | |
| if (dst_coherent) { | |
| inst->AddOperand( | |
| {SPV_OPERAND_TYPE_SCOPE_ID, {GetScopeConstant(dst_scope)}}); | |
| } | |
| if (src_coherent) { | |
| inst->AddOperand( | |
| {SPV_OPERAND_TYPE_SCOPE_ID, {GetScopeConstant(src_scope)}}); | |
| } | |
| } | |
| }); | |
| } | |
| } | |
| void UpgradeMemoryModel::UpgradeAtomics() { | |
| for (auto& func : *get_module()) { | |
| func.ForEachInst([this](Instruction* inst) { | |
| if (spvOpcodeIsAtomicOp(inst->opcode())) { | |
| bool unused_coherent = false; | |
| bool is_volatile = false; | |
| spv::Scope unused_scope = spv::Scope::QueueFamilyKHR; | |
| std::tie(unused_coherent, is_volatile, unused_scope) = | |
| GetInstructionAttributes(inst->GetSingleWordInOperand(0)); | |
| UpgradeSemantics(inst, 2u, is_volatile); | |
| if (inst->opcode() == spv::Op::OpAtomicCompareExchange || | |
| inst->opcode() == spv::Op::OpAtomicCompareExchangeWeak) { | |
| UpgradeSemantics(inst, 3u, is_volatile); | |
| } | |
| } | |
| }); | |
| } | |
| } | |
| void UpgradeMemoryModel::UpgradeSemantics(Instruction* inst, | |
| uint32_t in_operand, | |
| bool is_volatile) { | |
| if (!is_volatile) return; | |
| uint32_t semantics_id = inst->GetSingleWordInOperand(in_operand); | |
| const analysis::Constant* constant = | |
| context()->get_constant_mgr()->FindDeclaredConstant(semantics_id); | |
| const analysis::Integer* type = constant->type()->AsInteger(); | |
| assert(type && type->width() == 32); | |
| uint32_t value = 0; | |
| if (type->IsSigned()) { | |
| value = static_cast<uint32_t>(constant->GetS32()); | |
| } else { | |
| value = constant->GetU32(); | |
| } | |
| value |= uint32_t(spv::MemorySemanticsMask::Volatile); | |
| auto new_constant = context()->get_constant_mgr()->GetConstant(type, {value}); | |
| auto new_semantics = | |
| context()->get_constant_mgr()->GetDefiningInstruction(new_constant); | |
| inst->SetInOperand(in_operand, {new_semantics->result_id()}); | |
| } | |
| std::tuple<bool, bool, spv::Scope> UpgradeMemoryModel::GetInstructionAttributes( | |
| uint32_t id) { | |
| // |id| is a pointer used in a memory/image instruction. Need to determine if | |
| // that pointer points to volatile or coherent memory. Workgroup storage | |
| // class is implicitly coherent and cannot be decorated with volatile, so | |
| // short circuit that case. | |
| Instruction* inst = context()->get_def_use_mgr()->GetDef(id); | |
| analysis::Type* type = context()->get_type_mgr()->GetType(inst->type_id()); | |
| if (type->AsPointer() && | |
| type->AsPointer()->storage_class() == spv::StorageClass::Workgroup) { | |
| return std::make_tuple(true, false, spv::Scope::Workgroup); | |
| } | |
| bool is_coherent = false; | |
| bool is_volatile = false; | |
| std::unordered_set<uint32_t> visited; | |
| std::tie(is_coherent, is_volatile) = | |
| TraceInstruction(context()->get_def_use_mgr()->GetDef(id), | |
| std::vector<uint32_t>(), &visited); | |
| return std::make_tuple(is_coherent, is_volatile, spv::Scope::QueueFamilyKHR); | |
| } | |
| std::pair<bool, bool> UpgradeMemoryModel::TraceInstruction( | |
| Instruction* inst, std::vector<uint32_t> indices, | |
| std::unordered_set<uint32_t>* visited) { | |
| auto iter = cache_.find(std::make_pair(inst->result_id(), indices)); | |
| if (iter != cache_.end()) { | |
| return iter->second; | |
| } | |
| if (!visited->insert(inst->result_id()).second) { | |
| return std::make_pair(false, false); | |
| } | |
| // Initialize the cache before |indices| is (potentially) modified. | |
| auto& cached_result = cache_[std::make_pair(inst->result_id(), indices)]; | |
| cached_result.first = false; | |
| cached_result.second = false; | |
| bool is_coherent = false; | |
| bool is_volatile = false; | |
| switch (inst->opcode()) { | |
| case spv::Op::OpVariable: | |
| case spv::Op::OpFunctionParameter: | |
| is_coherent |= HasDecoration(inst, 0, spv::Decoration::Coherent); | |
| is_volatile |= HasDecoration(inst, 0, spv::Decoration::Volatile); | |
| if (!is_coherent || !is_volatile) { | |
| bool type_coherent = false; | |
| bool type_volatile = false; | |
| std::tie(type_coherent, type_volatile) = | |
| CheckType(inst->type_id(), indices); | |
| is_coherent |= type_coherent; | |
| is_volatile |= type_volatile; | |
| } | |
| break; | |
| case spv::Op::OpAccessChain: | |
| case spv::Op::OpInBoundsAccessChain: | |
| // Store indices in reverse order. | |
| for (uint32_t i = inst->NumInOperands() - 1; i > 0; --i) { | |
| indices.push_back(inst->GetSingleWordInOperand(i)); | |
| } | |
| break; | |
| case spv::Op::OpPtrAccessChain: | |
| // Store indices in reverse order. Skip the |Element| operand. | |
| for (uint32_t i = inst->NumInOperands() - 1; i > 1; --i) { | |
| indices.push_back(inst->GetSingleWordInOperand(i)); | |
| } | |
| break; | |
| default: | |
| break; | |
| } | |
| // No point searching further. | |
| if (is_coherent && is_volatile) { | |
| cached_result.first = true; | |
| cached_result.second = true; | |
| return std::make_pair(true, true); | |
| } | |
| // Variables and function parameters are sources. Continue searching until we | |
| // reach them. | |
| if (inst->opcode() != spv::Op::OpVariable && | |
| inst->opcode() != spv::Op::OpFunctionParameter) { | |
| inst->ForEachInId([this, &is_coherent, &is_volatile, &indices, | |
| &visited](const uint32_t* id_ptr) { | |
| Instruction* op_inst = context()->get_def_use_mgr()->GetDef(*id_ptr); | |
| const analysis::Type* type = | |
| context()->get_type_mgr()->GetType(op_inst->type_id()); | |
| if (type && | |
| (type->AsPointer() || type->AsImage() || type->AsSampledImage())) { | |
| bool operand_coherent = false; | |
| bool operand_volatile = false; | |
| std::tie(operand_coherent, operand_volatile) = | |
| TraceInstruction(op_inst, indices, visited); | |
| is_coherent |= operand_coherent; | |
| is_volatile |= operand_volatile; | |
| } | |
| }); | |
| } | |
| cached_result.first = is_coherent; | |
| cached_result.second = is_volatile; | |
| return std::make_pair(is_coherent, is_volatile); | |
| } | |
| std::pair<bool, bool> UpgradeMemoryModel::CheckType( | |
| uint32_t type_id, const std::vector<uint32_t>& indices) { | |
| bool is_coherent = false; | |
| bool is_volatile = false; | |
| Instruction* type_inst = context()->get_def_use_mgr()->GetDef(type_id); | |
| assert(type_inst->opcode() == spv::Op::OpTypePointer); | |
| Instruction* element_inst = context()->get_def_use_mgr()->GetDef( | |
| type_inst->GetSingleWordInOperand(1u)); | |
| for (int i = (int)indices.size() - 1; i >= 0; --i) { | |
| if (is_coherent && is_volatile) break; | |
| if (element_inst->opcode() == spv::Op::OpTypePointer) { | |
| element_inst = context()->get_def_use_mgr()->GetDef( | |
| element_inst->GetSingleWordInOperand(1u)); | |
| } else if (element_inst->opcode() == spv::Op::OpTypeStruct) { | |
| uint32_t index = indices.at(i); | |
| Instruction* index_inst = context()->get_def_use_mgr()->GetDef(index); | |
| assert(index_inst->opcode() == spv::Op::OpConstant); | |
| uint64_t value = GetIndexValue(index_inst); | |
| is_coherent |= HasDecoration(element_inst, static_cast<uint32_t>(value), | |
| spv::Decoration::Coherent); | |
| is_volatile |= HasDecoration(element_inst, static_cast<uint32_t>(value), | |
| spv::Decoration::Volatile); | |
| element_inst = context()->get_def_use_mgr()->GetDef( | |
| element_inst->GetSingleWordInOperand(static_cast<uint32_t>(value))); | |
| } else { | |
| assert(spvOpcodeIsComposite(element_inst->opcode())); | |
| element_inst = context()->get_def_use_mgr()->GetDef( | |
| element_inst->GetSingleWordInOperand(0u)); | |
| } | |
| } | |
| if (!is_coherent || !is_volatile) { | |
| bool remaining_coherent = false; | |
| bool remaining_volatile = false; | |
| std::tie(remaining_coherent, remaining_volatile) = | |
| CheckAllTypes(element_inst); | |
| is_coherent |= remaining_coherent; | |
| is_volatile |= remaining_volatile; | |
| } | |
| return std::make_pair(is_coherent, is_volatile); | |
| } | |
| std::pair<bool, bool> UpgradeMemoryModel::CheckAllTypes( | |
| const Instruction* inst) { | |
| std::unordered_set<const Instruction*> visited; | |
| std::vector<const Instruction*> stack; | |
| stack.push_back(inst); | |
| bool is_coherent = false; | |
| bool is_volatile = false; | |
| while (!stack.empty()) { | |
| const Instruction* def = stack.back(); | |
| stack.pop_back(); | |
| if (!visited.insert(def).second) continue; | |
| if (def->opcode() == spv::Op::OpTypeStruct) { | |
| // Any member decorated with coherent and/or volatile is enough to have | |
| // the related operation be flagged as coherent and/or volatile. | |
| is_coherent |= HasDecoration(def, std::numeric_limits<uint32_t>::max(), | |
| spv::Decoration::Coherent); | |
| is_volatile |= HasDecoration(def, std::numeric_limits<uint32_t>::max(), | |
| spv::Decoration::Volatile); | |
| if (is_coherent && is_volatile) | |
| return std::make_pair(is_coherent, is_volatile); | |
| // Check the subtypes. | |
| for (uint32_t i = 0; i < def->NumInOperands(); ++i) { | |
| stack.push_back(context()->get_def_use_mgr()->GetDef( | |
| def->GetSingleWordInOperand(i))); | |
| } | |
| } else if (spvOpcodeIsComposite(def->opcode())) { | |
| stack.push_back(context()->get_def_use_mgr()->GetDef( | |
| def->GetSingleWordInOperand(0u))); | |
| } else if (def->opcode() == spv::Op::OpTypePointer) { | |
| stack.push_back(context()->get_def_use_mgr()->GetDef( | |
| def->GetSingleWordInOperand(1u))); | |
| } | |
| } | |
| return std::make_pair(is_coherent, is_volatile); | |
| } | |
| uint64_t UpgradeMemoryModel::GetIndexValue(Instruction* index_inst) { | |
| const analysis::Constant* index_constant = | |
| context()->get_constant_mgr()->GetConstantFromInst(index_inst); | |
| assert(index_constant->AsIntConstant()); | |
| if (index_constant->type()->AsInteger()->IsSigned()) { | |
| if (index_constant->type()->AsInteger()->width() == 32) { | |
| return index_constant->GetS32(); | |
| } else { | |
| return index_constant->GetS64(); | |
| } | |
| } else { | |
| if (index_constant->type()->AsInteger()->width() == 32) { | |
| return index_constant->GetU32(); | |
| } else { | |
| return index_constant->GetU64(); | |
| } | |
| } | |
| } | |
| bool UpgradeMemoryModel::HasDecoration(const Instruction* inst, uint32_t value, | |
| spv::Decoration decoration) { | |
| // If the iteration was terminated early then an appropriate decoration was | |
| // found. | |
| return !context()->get_decoration_mgr()->WhileEachDecoration( | |
| inst->result_id(), (uint32_t)decoration, [value](const Instruction& i) { | |
| if (i.opcode() == spv::Op::OpDecorate || | |
| i.opcode() == spv::Op::OpDecorateId) { | |
| return false; | |
| } else if (i.opcode() == spv::Op::OpMemberDecorate) { | |
| if (value == i.GetSingleWordInOperand(1u) || | |
| value == std::numeric_limits<uint32_t>::max()) | |
| return false; | |
| } | |
| return true; | |
| }); | |
| } | |
| void UpgradeMemoryModel::UpgradeFlags(Instruction* inst, uint32_t in_operand, | |
| bool is_coherent, bool is_volatile, | |
| OperationType operation_type, | |
| InstructionType inst_type) { | |
| if (!is_coherent && !is_volatile) return; | |
| uint32_t flags = 0; | |
| if (inst->NumInOperands() > in_operand) { | |
| flags |= inst->GetSingleWordInOperand(in_operand); | |
| } | |
| if (is_coherent) { | |
| if (inst_type == kMemory) { | |
| flags |= uint32_t(spv::MemoryAccessMask::NonPrivatePointerKHR); | |
| if (operation_type == kVisibility) { | |
| flags |= uint32_t(spv::MemoryAccessMask::MakePointerVisibleKHR); | |
| } else { | |
| flags |= uint32_t(spv::MemoryAccessMask::MakePointerAvailableKHR); | |
| } | |
| } else { | |
| flags |= uint32_t(spv::ImageOperandsMask::NonPrivateTexelKHR); | |
| if (operation_type == kVisibility) { | |
| flags |= uint32_t(spv::ImageOperandsMask::MakeTexelVisibleKHR); | |
| } else { | |
| flags |= uint32_t(spv::ImageOperandsMask::MakeTexelAvailableKHR); | |
| } | |
| } | |
| } | |
| if (is_volatile) { | |
| if (inst_type == kMemory) { | |
| flags |= uint32_t(spv::MemoryAccessMask::Volatile); | |
| } else { | |
| flags |= uint32_t(spv::ImageOperandsMask::VolatileTexelKHR); | |
| } | |
| } | |
| if (inst->NumInOperands() > in_operand) { | |
| inst->SetInOperand(in_operand, {flags}); | |
| } else if (inst_type == kMemory) { | |
| inst->AddOperand({SPV_OPERAND_TYPE_OPTIONAL_MEMORY_ACCESS, {flags}}); | |
| } else { | |
| inst->AddOperand({SPV_OPERAND_TYPE_OPTIONAL_IMAGE, {flags}}); | |
| } | |
| } | |
| uint32_t UpgradeMemoryModel::GetScopeConstant(spv::Scope scope) { | |
| analysis::Integer int_ty(32, false); | |
| uint32_t int_id = context()->get_type_mgr()->GetTypeInstruction(&int_ty); | |
| const analysis::Constant* constant = | |
| context()->get_constant_mgr()->GetConstant( | |
| context()->get_type_mgr()->GetType(int_id), | |
| {static_cast<uint32_t>(scope)}); | |
| return context() | |
| ->get_constant_mgr() | |
| ->GetDefiningInstruction(constant) | |
| ->result_id(); | |
| } | |
| void UpgradeMemoryModel::CleanupDecorations() { | |
| // All of the volatile and coherent decorations have been dealt with, so now | |
| // we can just remove them. | |
| get_module()->ForEachInst([this](Instruction* inst) { | |
| if (inst->result_id() != 0) { | |
| context()->get_decoration_mgr()->RemoveDecorationsFrom( | |
| inst->result_id(), [](const Instruction& dec) { | |
| switch (dec.opcode()) { | |
| case spv::Op::OpDecorate: | |
| case spv::Op::OpDecorateId: | |
| if (spv::Decoration(dec.GetSingleWordInOperand(1u)) == | |
| spv::Decoration::Coherent || | |
| spv::Decoration(dec.GetSingleWordInOperand(1u)) == | |
| spv::Decoration::Volatile) | |
| return true; | |
| break; | |
| case spv::Op::OpMemberDecorate: | |
| if (spv::Decoration(dec.GetSingleWordInOperand(2u)) == | |
| spv::Decoration::Coherent || | |
| spv::Decoration(dec.GetSingleWordInOperand(2u)) == | |
| spv::Decoration::Volatile) | |
| return true; | |
| break; | |
| default: | |
| break; | |
| } | |
| return false; | |
| }); | |
| } | |
| }); | |
| } | |
| void UpgradeMemoryModel::UpgradeBarriers() { | |
| std::vector<Instruction*> barriers; | |
| // Collects all the control barriers in |function|. Returns true if the | |
| // function operates on the Output storage class. | |
| ProcessFunction CollectBarriers = [this, &barriers](Function* function) { | |
| bool operates_on_output = false; | |
| for (auto& block : *function) { | |
| block.ForEachInst([this, &barriers, | |
| &operates_on_output](Instruction* inst) { | |
| if (inst->opcode() == spv::Op::OpControlBarrier) { | |
| barriers.push_back(inst); | |
| } else if (!operates_on_output) { | |
| // This instruction operates on output storage class if it is a | |
| // pointer to output type or any input operand is a pointer to output | |
| // type. | |
| analysis::Type* type = | |
| context()->get_type_mgr()->GetType(inst->type_id()); | |
| if (type && type->AsPointer() && | |
| type->AsPointer()->storage_class() == spv::StorageClass::Output) { | |
| operates_on_output = true; | |
| return; | |
| } | |
| inst->ForEachInId([this, &operates_on_output](uint32_t* id_ptr) { | |
| Instruction* op_inst = | |
| context()->get_def_use_mgr()->GetDef(*id_ptr); | |
| analysis::Type* op_type = | |
| context()->get_type_mgr()->GetType(op_inst->type_id()); | |
| if (op_type && op_type->AsPointer() && | |
| op_type->AsPointer()->storage_class() == | |
| spv::StorageClass::Output) | |
| operates_on_output = true; | |
| }); | |
| } | |
| }); | |
| } | |
| return operates_on_output; | |
| }; | |
| std::queue<uint32_t> roots; | |
| for (auto& e : get_module()->entry_points()) | |
| if (spv::ExecutionModel(e.GetSingleWordInOperand(0u)) == | |
| spv::ExecutionModel::TessellationControl) { | |
| roots.push(e.GetSingleWordInOperand(1u)); | |
| if (context()->ProcessCallTreeFromRoots(CollectBarriers, &roots)) { | |
| for (auto barrier : barriers) { | |
| // Add OutputMemoryKHR to the semantics of the barriers. | |
| uint32_t semantics_id = barrier->GetSingleWordInOperand(2u); | |
| Instruction* semantics_inst = | |
| context()->get_def_use_mgr()->GetDef(semantics_id); | |
| analysis::Type* semantics_type = | |
| context()->get_type_mgr()->GetType(semantics_inst->type_id()); | |
| uint64_t semantics_value = GetIndexValue(semantics_inst); | |
| const analysis::Constant* constant = | |
| context()->get_constant_mgr()->GetConstant( | |
| semantics_type, | |
| {static_cast<uint32_t>(semantics_value) | | |
| uint32_t(spv::MemorySemanticsMask::OutputMemoryKHR)}); | |
| barrier->SetInOperand(2u, {context() | |
| ->get_constant_mgr() | |
| ->GetDefiningInstruction(constant) | |
| ->result_id()}); | |
| } | |
| } | |
| barriers.clear(); | |
| } | |
| } | |
| void UpgradeMemoryModel::UpgradeMemoryScope() { | |
| get_module()->ForEachInst([this](Instruction* inst) { | |
| // Don't need to handle all the operations that take a scope. | |
| // * Group operations can only be subgroup | |
| // * Non-uniform can only be workgroup or subgroup | |
| // * Named barriers are not supported by Vulkan | |
| // * Workgroup ops (e.g. async_copy) have at most workgroup scope. | |
| if (spvOpcodeIsAtomicOp(inst->opcode())) { | |
| if (IsDeviceScope(inst->GetSingleWordInOperand(1))) { | |
| inst->SetInOperand(1, {GetScopeConstant(spv::Scope::QueueFamilyKHR)}); | |
| } | |
| } else if (inst->opcode() == spv::Op::OpControlBarrier) { | |
| if (IsDeviceScope(inst->GetSingleWordInOperand(1))) { | |
| inst->SetInOperand(1, {GetScopeConstant(spv::Scope::QueueFamilyKHR)}); | |
| } | |
| } else if (inst->opcode() == spv::Op::OpMemoryBarrier) { | |
| if (IsDeviceScope(inst->GetSingleWordInOperand(0))) { | |
| inst->SetInOperand(0, {GetScopeConstant(spv::Scope::QueueFamilyKHR)}); | |
| } | |
| } | |
| }); | |
| } | |
| bool UpgradeMemoryModel::IsDeviceScope(uint32_t scope_id) { | |
| const analysis::Constant* constant = | |
| context()->get_constant_mgr()->FindDeclaredConstant(scope_id); | |
| assert(constant && "Memory scope must be a constant"); | |
| const analysis::Integer* type = constant->type()->AsInteger(); | |
| assert(type); | |
| assert(type->width() == 32 || type->width() == 64); | |
| if (type->width() == 32) { | |
| if (type->IsSigned()) | |
| return static_cast<spv::Scope>(constant->GetS32()) == spv::Scope::Device; | |
| else | |
| return static_cast<spv::Scope>(constant->GetU32()) == spv::Scope::Device; | |
| } else { | |
| if (type->IsSigned()) | |
| return static_cast<spv::Scope>(constant->GetS64()) == spv::Scope::Device; | |
| else | |
| return static_cast<spv::Scope>(constant->GetU64()) == spv::Scope::Device; | |
| } | |
| assert(false); | |
| return false; | |
| } | |
| void UpgradeMemoryModel::UpgradeExtInst(Instruction* ext_inst) { | |
| const bool is_modf = ext_inst->GetSingleWordInOperand(1u) == GLSLstd450Modf; | |
| auto ptr_id = ext_inst->GetSingleWordInOperand(3u); | |
| auto ptr_type_id = get_def_use_mgr()->GetDef(ptr_id)->type_id(); | |
| auto pointee_type_id = | |
| get_def_use_mgr()->GetDef(ptr_type_id)->GetSingleWordInOperand(1u); | |
| auto element_type_id = ext_inst->type_id(); | |
| std::vector<const analysis::Type*> element_types(2); | |
| element_types[0] = context()->get_type_mgr()->GetType(element_type_id); | |
| element_types[1] = context()->get_type_mgr()->GetType(pointee_type_id); | |
| analysis::Struct struct_type(element_types); | |
| uint32_t struct_id = | |
| context()->get_type_mgr()->GetTypeInstruction(&struct_type); | |
| // Change the operation | |
| GLSLstd450 new_op = is_modf ? GLSLstd450ModfStruct : GLSLstd450FrexpStruct; | |
| ext_inst->SetOperand(3u, {static_cast<uint32_t>(new_op)}); | |
| // Remove the pointer argument | |
| ext_inst->RemoveOperand(5u); | |
| // Set the type id to the new struct. | |
| ext_inst->SetResultType(struct_id); | |
| // The result is now a struct of the original result. The zero'th element is | |
| // old result and should replace the old result. The one'th element needs to | |
| // be stored via a new instruction. | |
| auto where = ext_inst->NextNode(); | |
| InstructionBuilder builder( | |
| context(), where, | |
| IRContext::kAnalysisDefUse | IRContext::kAnalysisInstrToBlockMapping); | |
| auto extract_0 = | |
| builder.AddCompositeExtract(element_type_id, ext_inst->result_id(), {0}); | |
| context()->ReplaceAllUsesWith(ext_inst->result_id(), extract_0->result_id()); | |
| // The extract's input was just changed to itself, so fix that. | |
| extract_0->SetInOperand(0u, {ext_inst->result_id()}); | |
| auto extract_1 = | |
| builder.AddCompositeExtract(pointee_type_id, ext_inst->result_id(), {1}); | |
| builder.AddStore(ptr_id, extract_1->result_id()); | |
| } | |
| uint32_t UpgradeMemoryModel::MemoryAccessNumWords(uint32_t mask) { | |
| uint32_t result = 1; | |
| if (mask & uint32_t(spv::MemoryAccessMask::Aligned)) ++result; | |
| if (mask & uint32_t(spv::MemoryAccessMask::MakePointerAvailableKHR)) ++result; | |
| if (mask & uint32_t(spv::MemoryAccessMask::MakePointerVisibleKHR)) ++result; | |
| return result; | |
| } | |
| } // namespace opt | |
| } // namespace spvtools |