commit 86509d9074c0bb99a36f1aff67970ef4ba21169a
author Nicolas Capens <capn@google.com> Thu Mar 21 13:23:50 2019 -0400
committer Nicolas Capens <nicolascapens@google.com> Fri Mar 22 16:36:28 2019 +0000
tree 25e4a571f2d8e91c0bd867741a7c54b6037db2fa
parent fabdec519281a283de3fdb0935d7d3edfb4a705d

Implement atomic load/store memory ordering

Bug b/127472316

Test: dEQP-VK.spirv_assembly.instruction.compute.opatomic.load
Test: dEQP-VK.spirv_assembly.instruction.compute.opatomic.store
Change-Id: I0379a1a74bc9742a3a9ae4d52b319e0838fd21bf
Reviewed-on: https://swiftshader-review.googlesource.com/c/SwiftShader/+/27728
Presubmit-Ready: Nicolas Capens <nicolascapens@google.com>
Tested-by: Nicolas Capens <nicolascapens@google.com>
Kokoro-Presubmit: kokoro <noreply+kokoro@google.com>
Reviewed-by: Chris Forbes <chrisforbes@google.com>

src/Reactor/LLVMReactor.cpp

diff --git a/src/Reactor/LLVMReactor.cpp b/src/Reactor/LLVMReactor.cpp
index 8c8c7e3..624af83 100644
--- a/src/Reactor/LLVMReactor.cpp
+++ b/src/Reactor/LLVMReactor.cpp
@@ -838,6 +838,29 @@
 		}
 	}
 
+	static llvm::AtomicOrdering atomicOrdering(bool atomic, std::memory_order memoryOrder)
+	{
+		#if REACTOR_LLVM_VERSION < 7
+			return llvm::AtomicOrdering::NotAtomic;
+		#endif
+
+		if(!atomic)
+		{
+			return llvm::AtomicOrdering::NotAtomic;
+		}
+
+		switch(memoryOrder)
+		{
+		case std::memory_order_relaxed: return llvm::AtomicOrdering::Monotonic;  // https://llvm.org/docs/Atomics.html#monotonic
+		case std::memory_order_consume: return llvm::AtomicOrdering::Acquire;    // https://llvm.org/docs/Atomics.html#acquire: "It should also be used for C++11/C11 memory_order_consume."
+		case std::memory_order_acquire: return llvm::AtomicOrdering::Acquire;
+		case std::memory_order_release: return llvm::AtomicOrdering::Release;
+		case std::memory_order_acq_rel: return llvm::AtomicOrdering::AcquireRelease;
+		case std::memory_order_seq_cst: return llvm::AtomicOrdering::SequentiallyConsistent;
+		default: assert(false);         return llvm::AtomicOrdering::AcquireRelease;
+		}
+	}
+
 	Nucleus::Nucleus()
 	{
 		::codegenMutex.lock();   // Reactor and LLVM are currently not thread safe
@@ -1190,7 +1213,7 @@
 		return V(::builder->CreateNot(V(v)));
 	}
 
-	Value *Nucleus::createLoad(Value *ptr, Type *type, bool isVolatile, unsigned int alignment)
+	Value *Nucleus::createLoad(Value *ptr, Type *type, bool isVolatile, unsigned int alignment, bool atomic, std::memory_order memoryOrder)
 	{
 		switch(asInternalType(type))
 		{
@@ -1201,7 +1224,7 @@
 			return createBitCast(
 				createInsertElement(
 					V(llvm::UndefValue::get(llvm::VectorType::get(T(Long::getType()), 2))),
-					createLoad(createBitCast(ptr, Pointer<Long>::getType()), Long::getType(), isVolatile, alignment),
+					createLoad(createBitCast(ptr, Pointer<Long>::getType()), Long::getType(), isVolatile, alignment, atomic, memoryOrder),
 					0),
 				type);
 		case Type_v2i16:
@@ -1209,21 +1232,26 @@
 			if(alignment != 0)   // Not a local variable (all vectors are 128-bit).
 			{
 				Value *u = V(llvm::UndefValue::get(llvm::VectorType::get(T(Long::getType()), 2)));
-				Value *i = createLoad(createBitCast(ptr, Pointer<Int>::getType()), Int::getType(), isVolatile, alignment);
+				Value *i = createLoad(createBitCast(ptr, Pointer<Int>::getType()), Int::getType(), isVolatile, alignment, atomic, memoryOrder);
 				i = createZExt(i, Long::getType());
 				Value *v = createInsertElement(u, i, 0);
 				return createBitCast(v, type);
 			}
 			// Fallthrough to non-emulated case.
 		case Type_LLVM:
-			assert(V(ptr)->getType()->getContainedType(0) == T(type));
-			return V(::builder->Insert(new llvm::LoadInst(V(ptr), "", isVolatile, alignment)));
+			{
+				assert(V(ptr)->getType()->getContainedType(0) == T(type));
+				auto load = new llvm::LoadInst(V(ptr), "", isVolatile, alignment);
+				load->setAtomic(atomicOrdering(atomic, memoryOrder));
+
+				return V(::builder->Insert(load));
+			}
 		default:
 			assert(false); return nullptr;
 		}
 	}
 
-	Value *Nucleus::createStore(Value *value, Value *ptr, Type *type, bool isVolatile, unsigned int alignment)
+	Value *Nucleus::createStore(Value *value, Value *ptr, Type *type, bool isVolatile, unsigned int alignment, bool atomic, std::memory_order memoryOrder)
 	{
 		switch(asInternalType(type))
 		{
@@ -1235,7 +1263,7 @@
 				createExtractElement(
 					createBitCast(value, T(llvm::VectorType::get(T(Long::getType()), 2))), Long::getType(), 0),
 				createBitCast(ptr, Pointer<Long>::getType()),
-				Long::getType(), isVolatile, alignment);
+				Long::getType(), isVolatile, alignment, atomic, memoryOrder);
 			return value;
 		case Type_v2i16:
 		case Type_v4i8:
@@ -1244,14 +1272,18 @@
 				createStore(
 					createExtractElement(createBitCast(value, Int4::getType()), Int::getType(), 0),
 					createBitCast(ptr, Pointer<Int>::getType()),
-					Int::getType(), isVolatile, alignment);
+					Int::getType(), isVolatile, alignment, atomic, memoryOrder);
 				return value;
 			}
 			// Fallthrough to non-emulated case.
 		case Type_LLVM:
-			assert(V(ptr)->getType()->getContainedType(0) == T(type));
-			::builder->Insert(new llvm::StoreInst(V(value), V(ptr), isVolatile, alignment));
-			return value;
+			{
+				assert(V(ptr)->getType()->getContainedType(0) == T(type));
+				auto store = ::builder->Insert(new llvm::StoreInst(V(value), V(ptr), isVolatile, alignment));
+				store->setAtomic(atomicOrdering(atomic, memoryOrder));
+
+				return value;
+			}
 		default:
 			assert(false); return nullptr;
 		}