Merge changes If776b87d,I27098964
* changes:
Update SPIR-V Tools to f7da52775
Squashed 'third_party/SPIRV-Tools/' changes from 82b378d671..f7da527757
diff --git a/src/Reactor/LLVMJIT.cpp b/src/Reactor/LLVMJIT.cpp
index 707f668..9912b41 100644
--- a/src/Reactor/LLVMJIT.cpp
+++ b/src/Reactor/LLVMJIT.cpp
@@ -116,13 +116,13 @@
{
switch(level)
{
- case rr::Optimization::Level::None: return ::llvm::CodeGenOpt::None;
- case rr::Optimization::Level::Less: return ::llvm::CodeGenOpt::Less;
- case rr::Optimization::Level::Default: return ::llvm::CodeGenOpt::Default;
- case rr::Optimization::Level::Aggressive: return ::llvm::CodeGenOpt::Aggressive;
+ case rr::Optimization::Level::None: return llvm::CodeGenOpt::None;
+ case rr::Optimization::Level::Less: return llvm::CodeGenOpt::Less;
+ case rr::Optimization::Level::Default: return llvm::CodeGenOpt::Default;
+ case rr::Optimization::Level::Aggressive: return llvm::CodeGenOpt::Aggressive;
default: UNREACHABLE("Unknown Optimization Level %d", int(level));
}
- return ::llvm::CodeGenOpt::Default;
+ return llvm::CodeGenOpt::Default;
}
class MemoryMapper final : public llvm::SectionMemoryManager::MemoryMapper
diff --git a/src/Reactor/LLVMJIT_ORCv1.cpp b/src/Reactor/LLVMJIT_ORCv1.cpp
index bedc2d0..5303311 100644
--- a/src/Reactor/LLVMJIT_ORCv1.cpp
+++ b/src/Reactor/LLVMJIT_ORCv1.cpp
@@ -205,13 +205,13 @@
{
switch(level)
{
- case rr::Optimization::Level::None: return ::llvm::CodeGenOpt::None;
- case rr::Optimization::Level::Less: return ::llvm::CodeGenOpt::Less;
- case rr::Optimization::Level::Default: return ::llvm::CodeGenOpt::Default;
- case rr::Optimization::Level::Aggressive: return ::llvm::CodeGenOpt::Aggressive;
+ case rr::Optimization::Level::None: return llvm::CodeGenOpt::None;
+ case rr::Optimization::Level::Less: return llvm::CodeGenOpt::Less;
+ case rr::Optimization::Level::Default: return llvm::CodeGenOpt::Default;
+ case rr::Optimization::Level::Aggressive: return llvm::CodeGenOpt::Aggressive;
default: UNREACHABLE("Unknown Optimization Level %d", int(level));
}
- return ::llvm::CodeGenOpt::Default;
+ return llvm::CodeGenOpt::Default;
}
JITGlobals::JITGlobals(const char *mcpu,
diff --git a/src/Reactor/LLVMReactor.cpp b/src/Reactor/LLVMReactor.cpp
index 4ab185b..7966da7 100644
--- a/src/Reactor/LLVMReactor.cpp
+++ b/src/Reactor/LLVMReactor.cpp
@@ -355,84 +355,6 @@
return jit->builder->CreateTrunc(mulh, ty);
}
-llvm::Value *createGather(llvm::Value *base, llvm::Type *elTy, llvm::Value *offsets, llvm::Value *mask, unsigned int alignment, bool zeroMaskedLanes)
-{
- ASSERT(base->getType()->isPointerTy());
- ASSERT(offsets->getType()->isVectorTy());
- ASSERT(mask->getType()->isVectorTy());
-
- auto numEls = llvm::cast<llvm::VectorType>(mask->getType())->getNumElements();
- auto i1Ty = ::llvm::Type::getInt1Ty(jit->context);
- auto i32Ty = ::llvm::Type::getInt32Ty(jit->context);
- auto i8Ty = ::llvm::Type::getInt8Ty(jit->context);
- auto i8PtrTy = i8Ty->getPointerTo();
- auto elPtrTy = elTy->getPointerTo();
- auto elVecTy = ::llvm::VectorType::get(elTy, numEls, false);
- auto elPtrVecTy = ::llvm::VectorType::get(elPtrTy, numEls, false);
- auto i8Base = jit->builder->CreatePointerCast(base, i8PtrTy);
- auto i8Ptrs = jit->builder->CreateGEP(i8Base, offsets);
- auto elPtrs = jit->builder->CreatePointerCast(i8Ptrs, elPtrVecTy);
- auto i8Mask = jit->builder->CreateIntCast(mask, ::llvm::VectorType::get(i1Ty, numEls, false), false); // vec<int, int, ...> -> vec<bool, bool, ...>
- auto passthrough = zeroMaskedLanes ? ::llvm::Constant::getNullValue(elVecTy) : llvm::UndefValue::get(elVecTy);
- auto align = ::llvm::ConstantInt::get(i32Ty, alignment);
- auto func = ::llvm::Intrinsic::getDeclaration(jit->module.get(), llvm::Intrinsic::masked_gather, { elVecTy, elPtrVecTy });
- return jit->builder->CreateCall(func, { elPtrs, align, i8Mask, passthrough });
-}
-
-void createScatter(llvm::Value *base, llvm::Value *val, llvm::Value *offsets, llvm::Value *mask, unsigned int alignment)
-{
- ASSERT(base->getType()->isPointerTy());
- ASSERT(val->getType()->isVectorTy());
- ASSERT(offsets->getType()->isVectorTy());
- ASSERT(mask->getType()->isVectorTy());
-
- auto numEls = llvm::cast<llvm::VectorType>(mask->getType())->getNumElements();
- auto i1Ty = ::llvm::Type::getInt1Ty(jit->context);
- auto i32Ty = ::llvm::Type::getInt32Ty(jit->context);
- auto i8Ty = ::llvm::Type::getInt8Ty(jit->context);
- auto i8PtrTy = i8Ty->getPointerTo();
- auto elVecTy = val->getType();
- auto elTy = llvm::cast<llvm::VectorType>(elVecTy)->getElementType();
- auto elPtrTy = elTy->getPointerTo();
- auto elPtrVecTy = ::llvm::VectorType::get(elPtrTy, numEls, false);
- auto i8Base = jit->builder->CreatePointerCast(base, i8PtrTy);
- auto i8Ptrs = jit->builder->CreateGEP(i8Base, offsets);
- auto elPtrs = jit->builder->CreatePointerCast(i8Ptrs, elPtrVecTy);
- auto i1Mask = jit->builder->CreateIntCast(mask, ::llvm::VectorType::get(i1Ty, numEls, false), false); // vec<int, int, ...> -> vec<bool, bool, ...>
- auto align = ::llvm::ConstantInt::get(i32Ty, alignment);
- auto func = ::llvm::Intrinsic::getDeclaration(jit->module.get(), llvm::Intrinsic::masked_scatter, { elVecTy, elPtrVecTy });
- jit->builder->CreateCall(func, { val, elPtrs, align, i1Mask });
-
-#if __has_feature(memory_sanitizer)
- // Mark memory writes as initialized by calling __msan_unpoison
- {
- // void __msan_unpoison(const volatile void *a, size_t size)
- auto voidTy = ::llvm::Type::getVoidTy(jit->context);
- auto int8Ty = ::llvm::Type::getInt8Ty(jit->context);
- auto int8PtrTy = int8Ty->getPointerTo();
- auto sizetTy = ::llvm::IntegerType::get(jit->context, sizeof(size_t) * 8);
- auto funcTy = ::llvm::FunctionType::get(voidTy, { int8PtrTy, sizetTy }, false);
- auto func = jit->module->getOrInsertFunction("__msan_unpoison", funcTy);
- auto size = jit->module->getDataLayout().getTypeStoreSize(elTy);
- for(unsigned i = 0; i < numEls; i++)
- {
- // Check mask for this element
- auto idx = ::llvm::ConstantInt::get(i32Ty, i);
- auto thenBlock = ::llvm::BasicBlock::Create(jit->context, "", jit->function);
- auto mergeBlock = ::llvm::BasicBlock::Create(jit->context, "", jit->function);
- jit->builder->CreateCondBr(jit->builder->CreateExtractElement(i1Mask, idx), thenBlock, mergeBlock);
- jit->builder->SetInsertPoint(thenBlock);
-
- // Insert __msan_unpoison call in conditional block
- auto elPtr = jit->builder->CreateExtractElement(elPtrs, idx);
- jit->builder->CreateCall(func, { jit->builder->CreatePointerCast(elPtr, int8PtrTy),
- ::llvm::ConstantInt::get(sizetTy, size) });
- jit->builder->CreateBr(mergeBlock);
- jit->builder->SetInsertPoint(mergeBlock);
- }
- }
-#endif
-}
} // namespace
namespace rr {
@@ -562,7 +484,7 @@
}
}
-static ::llvm::Function *createFunction(const char *name, ::llvm::Type *retTy, const std::vector<::llvm::Type *> ¶ms)
+static llvm::Function *createFunction(const char *name, llvm::Type *retTy, const std::vector<llvm::Type *> ¶ms)
{
llvm::FunctionType *functionType = llvm::FunctionType::get(retTy, params, false);
auto func = llvm::Function::Create(functionType, llvm::GlobalValue::InternalLinkage, name, jit->module.get());
@@ -978,7 +900,7 @@
// above, but certain backends cannot deal with this.
// Load as an integer and bitcast. See b/136037244.
auto size = jit->module->getDataLayout().getTypeStoreSize(elTy);
- auto elAsIntTy = ::llvm::IntegerType::get(jit->context, size * 8);
+ auto elAsIntTy = llvm::IntegerType::get(jit->context, size * 8);
auto ptrCast = jit->builder->CreatePointerCast(V(ptr), elAsIntTy->getPointerTo());
auto load = jit->builder->CreateAlignedLoad(ptrCast, alignment, isVolatile);
load->setAtomic(atomicOrdering(atomic, memoryOrder));
@@ -989,20 +911,20 @@
{
// More exotic types require falling back to the extern:
// void __atomic_load(size_t size, void *ptr, void *ret, int ordering)
- auto sizetTy = ::llvm::IntegerType::get(jit->context, sizeof(size_t) * 8);
- auto intTy = ::llvm::IntegerType::get(jit->context, sizeof(int) * 8);
- auto i8Ty = ::llvm::Type::getInt8Ty(jit->context);
+ auto sizetTy = llvm::IntegerType::get(jit->context, sizeof(size_t) * 8);
+ auto intTy = llvm::IntegerType::get(jit->context, sizeof(int) * 8);
+ auto i8Ty = llvm::Type::getInt8Ty(jit->context);
auto i8PtrTy = i8Ty->getPointerTo();
- auto voidTy = ::llvm::Type::getVoidTy(jit->context);
- auto funcTy = ::llvm::FunctionType::get(voidTy, { sizetTy, i8PtrTy, i8PtrTy, intTy }, false);
+ auto voidTy = llvm::Type::getVoidTy(jit->context);
+ auto funcTy = llvm::FunctionType::get(voidTy, { sizetTy, i8PtrTy, i8PtrTy, intTy }, false);
auto func = jit->module->getOrInsertFunction("__atomic_load", funcTy);
auto size = jit->module->getDataLayout().getTypeStoreSize(elTy);
auto out = allocateStackVariable(type);
jit->builder->CreateCall(func, {
- ::llvm::ConstantInt::get(sizetTy, size),
+ llvm::ConstantInt::get(sizetTy, size),
jit->builder->CreatePointerCast(V(ptr), i8PtrTy),
jit->builder->CreatePointerCast(V(out), i8PtrTy),
- ::llvm::ConstantInt::get(intTy, uint64_t(atomicOrdering(true, memoryOrder))),
+ llvm::ConstantInt::get(intTy, uint64_t(atomicOrdering(true, memoryOrder))),
});
return V(jit->builder->CreateLoad(V(out)));
}
@@ -1044,21 +966,21 @@
auto elTy = T(type);
ASSERT(V(ptr)->getType()->getContainedType(0) == elTy);
-#if __has_feature(memory_sanitizer)
- // Mark all memory writes as initialized by calling __msan_unpoison
+ if(__has_feature(memory_sanitizer))
{
+ // Mark all memory writes as initialized by calling __msan_unpoison
// void __msan_unpoison(const volatile void *a, size_t size)
- auto voidTy = ::llvm::Type::getVoidTy(jit->context);
- auto i8Ty = ::llvm::Type::getInt8Ty(jit->context);
+ auto voidTy = llvm::Type::getVoidTy(jit->context);
+ auto i8Ty = llvm::Type::getInt8Ty(jit->context);
auto voidPtrTy = i8Ty->getPointerTo();
- auto sizetTy = ::llvm::IntegerType::get(jit->context, sizeof(size_t) * 8);
- auto funcTy = ::llvm::FunctionType::get(voidTy, { voidPtrTy, sizetTy }, false);
+ auto sizetTy = llvm::IntegerType::get(jit->context, sizeof(size_t) * 8);
+ auto funcTy = llvm::FunctionType::get(voidTy, { voidPtrTy, sizetTy }, false);
auto func = jit->module->getOrInsertFunction("__msan_unpoison", funcTy);
auto size = jit->module->getDataLayout().getTypeStoreSize(elTy);
+
jit->builder->CreateCall(func, { jit->builder->CreatePointerCast(V(ptr), voidPtrTy),
- ::llvm::ConstantInt::get(sizetTy, size) });
+ llvm::ConstantInt::get(sizetTy, size) });
}
-#endif
if(!atomic)
{
@@ -1077,7 +999,7 @@
// above, but certain backends cannot deal with this.
// Store as an bitcast integer. See b/136037244.
auto size = jit->module->getDataLayout().getTypeStoreSize(elTy);
- auto elAsIntTy = ::llvm::IntegerType::get(jit->context, size * 8);
+ auto elAsIntTy = llvm::IntegerType::get(jit->context, size * 8);
auto valCast = jit->builder->CreateBitCast(V(value), elAsIntTy);
auto ptrCast = jit->builder->CreatePointerCast(V(ptr), elAsIntTy->getPointerTo());
auto store = jit->builder->CreateAlignedStore(valCast, ptrCast, alignment, isVolatile);
@@ -1087,21 +1009,21 @@
{
// More exotic types require falling back to the extern:
// void __atomic_store(size_t size, void *ptr, void *val, int ordering)
- auto sizetTy = ::llvm::IntegerType::get(jit->context, sizeof(size_t) * 8);
- auto intTy = ::llvm::IntegerType::get(jit->context, sizeof(int) * 8);
- auto i8Ty = ::llvm::Type::getInt8Ty(jit->context);
+ auto sizetTy = llvm::IntegerType::get(jit->context, sizeof(size_t) * 8);
+ auto intTy = llvm::IntegerType::get(jit->context, sizeof(int) * 8);
+ auto i8Ty = llvm::Type::getInt8Ty(jit->context);
auto i8PtrTy = i8Ty->getPointerTo();
- auto voidTy = ::llvm::Type::getVoidTy(jit->context);
- auto funcTy = ::llvm::FunctionType::get(voidTy, { sizetTy, i8PtrTy, i8PtrTy, intTy }, false);
+ auto voidTy = llvm::Type::getVoidTy(jit->context);
+ auto funcTy = llvm::FunctionType::get(voidTy, { sizetTy, i8PtrTy, i8PtrTy, intTy }, false);
auto func = jit->module->getOrInsertFunction("__atomic_store", funcTy);
auto size = jit->module->getDataLayout().getTypeStoreSize(elTy);
auto copy = allocateStackVariable(type);
jit->builder->CreateStore(V(value), V(copy));
jit->builder->CreateCall(func, {
- ::llvm::ConstantInt::get(sizetTy, size),
+ llvm::ConstantInt::get(sizetTy, size),
jit->builder->CreatePointerCast(V(ptr), i8PtrTy),
jit->builder->CreatePointerCast(V(copy), i8PtrTy),
- ::llvm::ConstantInt::get(intTy, uint64_t(atomicOrdering(true, memoryOrder))),
+ llvm::ConstantInt::get(intTy, uint64_t(atomicOrdering(true, memoryOrder))),
});
}
@@ -1121,14 +1043,14 @@
ASSERT(V(mask)->getType()->isVectorTy());
auto numEls = llvm::cast<llvm::VectorType>(V(mask)->getType())->getNumElements();
- auto i1Ty = ::llvm::Type::getInt1Ty(jit->context);
- auto i32Ty = ::llvm::Type::getInt32Ty(jit->context);
- auto elVecTy = ::llvm::VectorType::get(T(elTy), numEls, false);
+ auto i1Ty = llvm::Type::getInt1Ty(jit->context);
+ auto i32Ty = llvm::Type::getInt32Ty(jit->context);
+ auto elVecTy = llvm::VectorType::get(T(elTy), numEls, false);
auto elVecPtrTy = elVecTy->getPointerTo();
- auto i8Mask = jit->builder->CreateIntCast(V(mask), ::llvm::VectorType::get(i1Ty, numEls, false), false); // vec<int, int, ...> -> vec<bool, bool, ...>
- auto passthrough = zeroMaskedLanes ? ::llvm::Constant::getNullValue(elVecTy) : llvm::UndefValue::get(elVecTy);
- auto align = ::llvm::ConstantInt::get(i32Ty, alignment);
- auto func = ::llvm::Intrinsic::getDeclaration(jit->module.get(), llvm::Intrinsic::masked_load, { elVecTy, elVecPtrTy });
+ auto i8Mask = jit->builder->CreateIntCast(V(mask), llvm::VectorType::get(i1Ty, numEls, false), false); // vec<int, int, ...> -> vec<bool, bool, ...>
+ auto passthrough = zeroMaskedLanes ? llvm::Constant::getNullValue(elVecTy) : llvm::UndefValue::get(elVecTy);
+ auto align = llvm::ConstantInt::get(i32Ty, alignment);
+ auto func = llvm::Intrinsic::getDeclaration(jit->module.get(), llvm::Intrinsic::masked_load, { elVecTy, elVecPtrTy });
return V(jit->builder->CreateCall(func, { V(ptr), align, i8Mask, passthrough }));
}
@@ -1141,43 +1063,99 @@
ASSERT(V(mask)->getType()->isVectorTy());
auto numEls = llvm::cast<llvm::VectorType>(V(mask)->getType())->getNumElements();
- auto i1Ty = ::llvm::Type::getInt1Ty(jit->context);
- auto i32Ty = ::llvm::Type::getInt32Ty(jit->context);
+ auto i1Ty = llvm::Type::getInt1Ty(jit->context);
+ auto i32Ty = llvm::Type::getInt32Ty(jit->context);
auto elVecTy = V(val)->getType();
auto elVecPtrTy = elVecTy->getPointerTo();
- auto i1Mask = jit->builder->CreateIntCast(V(mask), ::llvm::VectorType::get(i1Ty, numEls, false), false); // vec<int, int, ...> -> vec<bool, bool, ...>
- auto align = ::llvm::ConstantInt::get(i32Ty, alignment);
- auto func = ::llvm::Intrinsic::getDeclaration(jit->module.get(), llvm::Intrinsic::masked_store, { elVecTy, elVecPtrTy });
+ auto i1Mask = jit->builder->CreateIntCast(V(mask), llvm::VectorType::get(i1Ty, numEls, false), false); // vec<int, int, ...> -> vec<bool, bool, ...>
+ auto align = llvm::ConstantInt::get(i32Ty, alignment);
+ auto func = llvm::Intrinsic::getDeclaration(jit->module.get(), llvm::Intrinsic::masked_store, { elVecTy, elVecPtrTy });
jit->builder->CreateCall(func, { V(val), V(ptr), align, i1Mask });
-#if __has_feature(memory_sanitizer)
- // Mark memory writes as initialized by calling __msan_unpoison
+ if(__has_feature(memory_sanitizer))
{
+ // Mark memory writes as initialized by calling __msan_unpoison
// void __msan_unpoison(const volatile void *a, size_t size)
- auto voidTy = ::llvm::Type::getVoidTy(jit->context);
+ auto voidTy = llvm::Type::getVoidTy(jit->context);
auto voidPtrTy = voidTy->getPointerTo();
- auto sizetTy = ::llvm::IntegerType::get(jit->context, sizeof(size_t) * 8);
- auto funcTy = ::llvm::FunctionType::get(voidTy, { voidPtrTy, sizetTy }, false);
+ auto sizetTy = llvm::IntegerType::get(jit->context, sizeof(size_t) * 8);
+ auto funcTy = llvm::FunctionType::get(voidTy, { voidPtrTy, sizetTy }, false);
auto func = jit->module->getOrInsertFunction("__msan_unpoison", funcTy);
auto size = jit->module->getDataLayout().getTypeStoreSize(llvm::cast<llvm::VectorType>(elVecTy)->getElementType());
+
for(unsigned i = 0; i < numEls; i++)
{
// Check mask for this element
- auto idx = ::llvm::ConstantInt::get(i32Ty, i);
- auto thenBlock = ::llvm::BasicBlock::Create(jit->context, "", jit->function);
- auto mergeBlock = ::llvm::BasicBlock::Create(jit->context, "", jit->function);
+ auto idx = llvm::ConstantInt::get(i32Ty, i);
+ auto thenBlock = llvm::BasicBlock::Create(jit->context, "", jit->function);
+ auto mergeBlock = llvm::BasicBlock::Create(jit->context, "", jit->function);
jit->builder->CreateCondBr(jit->builder->CreateExtractElement(i1Mask, idx), thenBlock, mergeBlock);
jit->builder->SetInsertPoint(thenBlock);
// Insert __msan_unpoison call in conditional block
auto elPtr = jit->builder->CreateGEP(V(ptr), idx);
jit->builder->CreateCall(func, { jit->builder->CreatePointerCast(elPtr, voidPtrTy),
- ::llvm::ConstantInt::get(sizetTy, size) });
+ llvm::ConstantInt::get(sizetTy, size) });
+
jit->builder->CreateBr(mergeBlock);
jit->builder->SetInsertPoint(mergeBlock);
}
}
-#endif
+} // namespace rr
+
+static llvm::Value *createGather(llvm::Value *base, llvm::Type *elTy, llvm::Value *offsets, llvm::Value *mask, unsigned int alignment, bool zeroMaskedLanes)
+{
+ ASSERT(base->getType()->isPointerTy());
+ ASSERT(offsets->getType()->isVectorTy());
+ ASSERT(mask->getType()->isVectorTy());
+
+ auto numEls = llvm::cast<llvm::VectorType>(mask->getType())->getNumElements();
+ auto i1Ty = llvm::Type::getInt1Ty(jit->context);
+ auto i32Ty = llvm::Type::getInt32Ty(jit->context);
+ auto i8Ty = llvm::Type::getInt8Ty(jit->context);
+ auto i8PtrTy = i8Ty->getPointerTo();
+ auto elPtrTy = elTy->getPointerTo();
+ auto elVecTy = llvm::VectorType::get(elTy, numEls, false);
+ auto elPtrVecTy = llvm::VectorType::get(elPtrTy, numEls, false);
+ auto i8Base = jit->builder->CreatePointerCast(base, i8PtrTy);
+ auto i8Ptrs = jit->builder->CreateGEP(i8Base, offsets);
+ auto elPtrs = jit->builder->CreatePointerCast(i8Ptrs, elPtrVecTy);
+ auto i1Mask = jit->builder->CreateIntCast(mask, llvm::VectorType::get(i1Ty, numEls, false), false); // vec<int, int, ...> -> vec<bool, bool, ...>
+ auto passthrough = zeroMaskedLanes ? llvm::Constant::getNullValue(elVecTy) : llvm::UndefValue::get(elVecTy);
+
+ if(!__has_feature(memory_sanitizer))
+ {
+ auto align = llvm::ConstantInt::get(i32Ty, alignment);
+ auto func = llvm::Intrinsic::getDeclaration(jit->module.get(), llvm::Intrinsic::masked_gather, { elVecTy, elPtrVecTy });
+ return jit->builder->CreateCall(func, { elPtrs, align, i1Mask, passthrough });
+ }
+ else // __has_feature(memory_sanitizer)
+ {
+ // MemorySanitizer currently does not support instrumenting llvm::Intrinsic::masked_gather
+ // Work around it by emulating gather with element-wise loads.
+ // TODO(b/172238865): Remove when supported by MemorySanitizer.
+
+ Value *result = Nucleus::allocateStackVariable(T(elVecTy));
+ Nucleus::createStore(V(passthrough), result, T(elVecTy));
+
+ for(unsigned i = 0; i < numEls; i++)
+ {
+ // Check mask for this element
+ Value *elementMask = Nucleus::createExtractElement(V(i1Mask), T(i1Ty), i);
+
+ If(RValue<Bool>(elementMask))
+ {
+ Value *elPtr = Nucleus::createExtractElement(V(elPtrs), T(elPtrTy), i);
+ Value *el = Nucleus::createLoad(elPtr, T(elTy), /*isVolatile */ false, alignment, /* atomic */ false, std::memory_order_relaxed);
+
+ Value *v = Nucleus::createLoad(result, T(elVecTy));
+ v = Nucleus::createInsertElement(v, el, i);
+ Nucleus::createStore(v, result, T(elVecTy));
+ }
+ }
+
+ return V(Nucleus::createLoad(result, T(elVecTy)));
+ }
}
RValue<Float4> Gather(RValue<Pointer<Float>> base, RValue<Int4> offsets, RValue<Int4> mask, unsigned int alignment, bool zeroMaskedLanes /* = false */)
@@ -1187,7 +1165,60 @@
RValue<Int4> Gather(RValue<Pointer<Int>> base, RValue<Int4> offsets, RValue<Int4> mask, unsigned int alignment, bool zeroMaskedLanes /* = false */)
{
- return As<Int4>(V(createGather(V(base.value()), T(Float::type()), V(offsets.value()), V(mask.value()), alignment, zeroMaskedLanes)));
+ return As<Int4>(V(createGather(V(base.value()), T(Int::type()), V(offsets.value()), V(mask.value()), alignment, zeroMaskedLanes)));
+}
+
+static void createScatter(llvm::Value *base, llvm::Value *val, llvm::Value *offsets, llvm::Value *mask, unsigned int alignment)
+{
+ ASSERT(base->getType()->isPointerTy());
+ ASSERT(val->getType()->isVectorTy());
+ ASSERT(offsets->getType()->isVectorTy());
+ ASSERT(mask->getType()->isVectorTy());
+
+ auto numEls = llvm::cast<llvm::VectorType>(mask->getType())->getNumElements();
+ auto i1Ty = llvm::Type::getInt1Ty(jit->context);
+ auto i32Ty = llvm::Type::getInt32Ty(jit->context);
+ auto i8Ty = llvm::Type::getInt8Ty(jit->context);
+ auto i8PtrTy = i8Ty->getPointerTo();
+ auto elVecTy = val->getType();
+ auto elTy = llvm::cast<llvm::VectorType>(elVecTy)->getElementType();
+ auto elPtrTy = elTy->getPointerTo();
+ auto elPtrVecTy = llvm::VectorType::get(elPtrTy, numEls, false);
+
+ auto i8Base = jit->builder->CreatePointerCast(base, i8PtrTy);
+ auto i8Ptrs = jit->builder->CreateGEP(i8Base, offsets);
+ auto elPtrs = jit->builder->CreatePointerCast(i8Ptrs, elPtrVecTy);
+ auto i1Mask = jit->builder->CreateIntCast(mask, llvm::VectorType::get(i1Ty, numEls, false), false); // vec<int, int, ...> -> vec<bool, bool, ...>
+
+ if(!__has_feature(memory_sanitizer))
+ {
+ auto align = llvm::ConstantInt::get(i32Ty, alignment);
+ auto func = llvm::Intrinsic::getDeclaration(jit->module.get(), llvm::Intrinsic::masked_scatter, { elVecTy, elPtrVecTy });
+ jit->builder->CreateCall(func, { val, elPtrs, align, i1Mask });
+ }
+ else // __has_feature(memory_sanitizer)
+ {
+ // MemorySanitizer currently does not support instrumenting llvm::Intrinsic::masked_scatter
+ // Work around it by emulating scatter with element-wise stores.
+ // TODO(b/172238865): Remove when supported by MemorySanitizer.
+
+ for(unsigned i = 0; i < numEls; i++)
+ {
+ // Check mask for this element
+ auto idx = llvm::ConstantInt::get(i32Ty, i);
+ auto thenBlock = llvm::BasicBlock::Create(jit->context, "", jit->function);
+ auto mergeBlock = llvm::BasicBlock::Create(jit->context, "", jit->function);
+ jit->builder->CreateCondBr(jit->builder->CreateExtractElement(i1Mask, idx), thenBlock, mergeBlock);
+ jit->builder->SetInsertPoint(thenBlock);
+
+ auto el = jit->builder->CreateExtractElement(val, idx);
+ auto elPtr = jit->builder->CreateExtractElement(elPtrs, idx);
+ Nucleus::createStore(V(el), V(elPtr), T(elTy), /*isVolatile */ false, alignment, /* atomic */ false, std::memory_order_relaxed);
+
+ jit->builder->CreateBr(mergeBlock);
+ jit->builder->SetInsertPoint(mergeBlock);
+ }
+ }
}
void Scatter(RValue<Pointer<Float>> base, RValue<Float4> val, RValue<Int4> offsets, RValue<Int4> mask, unsigned int alignment)
@@ -1609,9 +1640,9 @@
return T(llvm::PointerType::get(T(ElementType), 0));
}
-static ::llvm::Type *getNaturalIntType()
+static llvm::Type *getNaturalIntType()
{
- return ::llvm::Type::getIntNTy(jit->context, sizeof(int) * 8);
+ return llvm::Type::getIntNTy(jit->context, sizeof(int) * 8);
}
Type *Nucleus::getPrintfStorageType(Type *valueType)
@@ -3215,9 +3246,9 @@
static RValue<Float4> TransformFloat4PerElement(RValue<Float4> v, const char *name)
{
- auto funcTy = ::llvm::FunctionType::get(T(Float::type()), ::llvm::ArrayRef<llvm::Type *>(T(Float::type())), false);
+ auto funcTy = llvm::FunctionType::get(T(Float::type()), llvm::ArrayRef<llvm::Type *>(T(Float::type())), false);
auto func = jit->module->getOrInsertFunction(name, funcTy);
- llvm::Value *out = ::llvm::UndefValue::get(T(Float4::type()));
+ llvm::Value *out = llvm::UndefValue::get(T(Float4::type()));
for(uint64_t i = 0; i < 4; i++)
{
auto el = jit->builder->CreateCall(func, V(Nucleus::createExtractElement(v.value(), Float::type(), i)));
@@ -3283,12 +3314,12 @@
RValue<Float4> Atan2(RValue<Float4> x, RValue<Float4> y)
{
RR_DEBUG_INFO_UPDATE_LOC();
- ::llvm::SmallVector<::llvm::Type *, 2> paramTys;
+ llvm::SmallVector<llvm::Type *, 2> paramTys;
paramTys.push_back(T(Float::type()));
paramTys.push_back(T(Float::type()));
- auto funcTy = ::llvm::FunctionType::get(T(Float::type()), paramTys, false);
+ auto funcTy = llvm::FunctionType::get(T(Float::type()), paramTys, false);
auto func = jit->module->getOrInsertFunction("atan2f", funcTy);
- llvm::Value *out = ::llvm::UndefValue::get(T(Float4::type()));
+ llvm::Value *out = llvm::UndefValue::get(T(Float4::type()));
for(uint64_t i = 0; i < 4; i++)
{
auto el = jit->builder->CreateCall(func, { V(Nucleus::createExtractElement(x.value(), Float::type(), i)),
@@ -3338,7 +3369,7 @@
RR_DEBUG_INFO_UPDATE_LOC();
auto func = llvm::Intrinsic::getDeclaration(jit->module.get(), llvm::Intrinsic::ctlz, { T(UInt::type()) });
return RValue<UInt>(V(jit->builder->CreateCall(func, { V(v.value()),
- isZeroUndef ? ::llvm::ConstantInt::getTrue(jit->context) : ::llvm::ConstantInt::getFalse(jit->context) })));
+ isZeroUndef ? llvm::ConstantInt::getTrue(jit->context) : llvm::ConstantInt::getFalse(jit->context) })));
}
RValue<UInt4> Ctlz(RValue<UInt4> v, bool isZeroUndef)
@@ -3346,7 +3377,7 @@
RR_DEBUG_INFO_UPDATE_LOC();
auto func = llvm::Intrinsic::getDeclaration(jit->module.get(), llvm::Intrinsic::ctlz, { T(UInt4::type()) });
return RValue<UInt4>(V(jit->builder->CreateCall(func, { V(v.value()),
- isZeroUndef ? ::llvm::ConstantInt::getTrue(jit->context) : ::llvm::ConstantInt::getFalse(jit->context) })));
+ isZeroUndef ? llvm::ConstantInt::getTrue(jit->context) : llvm::ConstantInt::getFalse(jit->context) })));
}
RValue<UInt> Cttz(RValue<UInt> v, bool isZeroUndef)
@@ -3354,7 +3385,7 @@
RR_DEBUG_INFO_UPDATE_LOC();
auto func = llvm::Intrinsic::getDeclaration(jit->module.get(), llvm::Intrinsic::cttz, { T(UInt::type()) });
return RValue<UInt>(V(jit->builder->CreateCall(func, { V(v.value()),
- isZeroUndef ? ::llvm::ConstantInt::getTrue(jit->context) : ::llvm::ConstantInt::getFalse(jit->context) })));
+ isZeroUndef ? llvm::ConstantInt::getTrue(jit->context) : llvm::ConstantInt::getFalse(jit->context) })));
}
RValue<UInt4> Cttz(RValue<UInt4> v, bool isZeroUndef)
@@ -3362,7 +3393,7 @@
RR_DEBUG_INFO_UPDATE_LOC();
auto func = llvm::Intrinsic::getDeclaration(jit->module.get(), llvm::Intrinsic::cttz, { T(UInt4::type()) });
return RValue<UInt4>(V(jit->builder->CreateCall(func, { V(v.value()),
- isZeroUndef ? ::llvm::ConstantInt::getTrue(jit->context) : ::llvm::ConstantInt::getFalse(jit->context) })));
+ isZeroUndef ? llvm::ConstantInt::getTrue(jit->context) : llvm::ConstantInt::getFalse(jit->context) })));
}
RValue<Int> MinAtomic(RValue<Pointer<Int>> x, RValue<Int> y, std::memory_order memoryOrder)
@@ -3403,7 +3434,7 @@
RR_DEBUG_INFO_UPDATE_LOC();
// Note: this should work for 32-bit pointers as well because 'inttoptr'
// is defined to truncate (and zero extend) if necessary.
- auto ptrAsInt = ::llvm::ConstantInt::get(::llvm::Type::getInt64Ty(jit->context), reinterpret_cast<uintptr_t>(ptr));
+ auto ptrAsInt = llvm::ConstantInt::get(llvm::Type::getInt64Ty(jit->context), reinterpret_cast<uintptr_t>(ptr));
return RValue<Pointer<Byte>>(V(jit->builder->CreateIntToPtr(ptrAsInt, T(Pointer<Byte>::type()))));
}
@@ -3418,14 +3449,14 @@
Value *Call(RValue<Pointer<Byte>> fptr, Type *retTy, std::initializer_list<Value *> args, std::initializer_list<Type *> argTys)
{
RR_DEBUG_INFO_UPDATE_LOC();
- ::llvm::SmallVector<::llvm::Type *, 8> paramTys;
+ llvm::SmallVector<llvm::Type *, 8> paramTys;
for(auto ty : argTys) { paramTys.push_back(T(ty)); }
- auto funcTy = ::llvm::FunctionType::get(T(retTy), paramTys, false);
+ auto funcTy = llvm::FunctionType::get(T(retTy), paramTys, false);
auto funcPtrTy = funcTy->getPointerTo();
auto funcPtr = jit->builder->CreatePointerCast(V(fptr.value()), funcPtrTy);
- ::llvm::SmallVector<::llvm::Value *, 8> arguments;
+ llvm::SmallVector<llvm::Value *, 8> arguments;
for(auto arg : args) { arguments.push_back(V(arg)); }
return V(jit->builder->CreateCall(funcTy, funcPtr, arguments));
}
@@ -3816,9 +3847,9 @@
#ifdef ENABLE_RR_PRINT
void VPrintf(const std::vector<Value *> &vals)
{
- auto i32Ty = ::llvm::Type::getInt32Ty(jit->context);
- auto i8PtrTy = ::llvm::Type::getInt8PtrTy(jit->context);
- auto funcTy = ::llvm::FunctionType::get(i32Ty, { i8PtrTy }, true);
+ auto i32Ty = llvm::Type::getInt32Ty(jit->context);
+ auto i8PtrTy = llvm::Type::getInt8PtrTy(jit->context);
+ auto funcTy = llvm::FunctionType::get(i32Ty, { i8PtrTy }, true);
auto func = jit->module->getOrInsertFunction("rr::DebugPrintf", funcTy);
jit->builder->CreateCall(func, V(vals));
}
@@ -3826,8 +3857,8 @@
void Nop()
{
- auto voidTy = ::llvm::Type::getVoidTy(jit->context);
- auto funcTy = ::llvm::FunctionType::get(voidTy, {}, false);
+ auto voidTy = llvm::Type::getVoidTy(jit->context);
+ auto funcTy = llvm::FunctionType::get(voidTy, {}, false);
auto func = jit->module->getOrInsertFunction("nop", funcTy);
jit->builder->CreateCall(func);
}
@@ -3882,29 +3913,29 @@
ASSERT(jit->coroutine.id == nullptr);
// Types
- auto voidTy = ::llvm::Type::getVoidTy(jit->context);
- auto i1Ty = ::llvm::Type::getInt1Ty(jit->context);
- auto i8Ty = ::llvm::Type::getInt8Ty(jit->context);
- auto i32Ty = ::llvm::Type::getInt32Ty(jit->context);
- auto i8PtrTy = ::llvm::Type::getInt8PtrTy(jit->context);
+ auto voidTy = llvm::Type::getVoidTy(jit->context);
+ auto i1Ty = llvm::Type::getInt1Ty(jit->context);
+ auto i8Ty = llvm::Type::getInt8Ty(jit->context);
+ auto i32Ty = llvm::Type::getInt32Ty(jit->context);
+ auto i8PtrTy = llvm::Type::getInt8PtrTy(jit->context);
auto promiseTy = jit->coroutine.yieldType;
auto promisePtrTy = promiseTy->getPointerTo();
// LLVM intrinsics
- auto coro_id = ::llvm::Intrinsic::getDeclaration(jit->module.get(), llvm::Intrinsic::coro_id);
- auto coro_size = ::llvm::Intrinsic::getDeclaration(jit->module.get(), llvm::Intrinsic::coro_size, { i32Ty });
- auto coro_begin = ::llvm::Intrinsic::getDeclaration(jit->module.get(), llvm::Intrinsic::coro_begin);
- auto coro_resume = ::llvm::Intrinsic::getDeclaration(jit->module.get(), ::llvm::Intrinsic::coro_resume);
- auto coro_end = ::llvm::Intrinsic::getDeclaration(jit->module.get(), ::llvm::Intrinsic::coro_end);
- auto coro_free = ::llvm::Intrinsic::getDeclaration(jit->module.get(), ::llvm::Intrinsic::coro_free);
- auto coro_destroy = ::llvm::Intrinsic::getDeclaration(jit->module.get(), ::llvm::Intrinsic::coro_destroy);
- auto coro_promise = ::llvm::Intrinsic::getDeclaration(jit->module.get(), ::llvm::Intrinsic::coro_promise);
- auto coro_done = ::llvm::Intrinsic::getDeclaration(jit->module.get(), ::llvm::Intrinsic::coro_done);
- auto coro_suspend = ::llvm::Intrinsic::getDeclaration(jit->module.get(), ::llvm::Intrinsic::coro_suspend);
+ auto coro_id = llvm::Intrinsic::getDeclaration(jit->module.get(), llvm::Intrinsic::coro_id);
+ auto coro_size = llvm::Intrinsic::getDeclaration(jit->module.get(), llvm::Intrinsic::coro_size, { i32Ty });
+ auto coro_begin = llvm::Intrinsic::getDeclaration(jit->module.get(), llvm::Intrinsic::coro_begin);
+ auto coro_resume = llvm::Intrinsic::getDeclaration(jit->module.get(), llvm::Intrinsic::coro_resume);
+ auto coro_end = llvm::Intrinsic::getDeclaration(jit->module.get(), llvm::Intrinsic::coro_end);
+ auto coro_free = llvm::Intrinsic::getDeclaration(jit->module.get(), llvm::Intrinsic::coro_free);
+ auto coro_destroy = llvm::Intrinsic::getDeclaration(jit->module.get(), llvm::Intrinsic::coro_destroy);
+ auto coro_promise = llvm::Intrinsic::getDeclaration(jit->module.get(), llvm::Intrinsic::coro_promise);
+ auto coro_done = llvm::Intrinsic::getDeclaration(jit->module.get(), llvm::Intrinsic::coro_done);
+ auto coro_suspend = llvm::Intrinsic::getDeclaration(jit->module.get(), llvm::Intrinsic::coro_suspend);
- auto allocFrameTy = ::llvm::FunctionType::get(i8PtrTy, { i32Ty }, false);
+ auto allocFrameTy = llvm::FunctionType::get(i8PtrTy, { i32Ty }, false);
auto allocFrame = jit->module->getOrInsertFunction("coroutine_alloc_frame", allocFrameTy);
- auto freeFrameTy = ::llvm::FunctionType::get(voidTy, { i8PtrTy }, false);
+ auto freeFrameTy = llvm::FunctionType::get(voidTy, { i8PtrTy }, false);
auto freeFrame = jit->module->getOrInsertFunction("coroutine_free_frame", freeFrameTy);
auto oldInsertionPoint = jit->builder->saveIP();
@@ -3937,15 +3968,15 @@
jit->builder->CreateCondBr(done, doneBlock, resumeBlock);
jit->builder->SetInsertPoint(doneBlock);
- jit->builder->CreateRet(::llvm::ConstantInt::getFalse(i1Ty));
+ jit->builder->CreateRet(llvm::ConstantInt::getFalse(i1Ty));
jit->builder->SetInsertPoint(resumeBlock);
- auto promiseAlignment = ::llvm::ConstantInt::get(i32Ty, 4); // TODO: Get correct alignment.
- auto promisePtr = jit->builder->CreateCall(coro_promise, { handle, promiseAlignment, ::llvm::ConstantInt::get(i1Ty, 0) });
+ auto promiseAlignment = llvm::ConstantInt::get(i32Ty, 4); // TODO: Get correct alignment.
+ auto promisePtr = jit->builder->CreateCall(coro_promise, { handle, promiseAlignment, llvm::ConstantInt::get(i1Ty, 0) });
auto promise = jit->builder->CreateLoad(jit->builder->CreatePointerCast(promisePtr, promisePtrTy));
jit->builder->CreateStore(promise, outPtr);
jit->builder->CreateCall(coro_resume, { handle });
- jit->builder->CreateRet(::llvm::ConstantInt::getTrue(i1Ty));
+ jit->builder->CreateRet(llvm::ConstantInt::getTrue(i1Ty));
}
// Build the coroutine_destroy() function:
@@ -4006,10 +4037,10 @@
jit->builder->SetInsertPoint(jit->coroutine.entryBlock, jit->coroutine.entryBlock->begin());
jit->coroutine.promise = jit->builder->CreateAlloca(promiseTy, nullptr, "promise");
jit->coroutine.id = jit->builder->CreateCall(coro_id, {
- ::llvm::ConstantInt::get(i32Ty, 0),
+ llvm::ConstantInt::get(i32Ty, 0),
jit->builder->CreatePointerCast(jit->coroutine.promise, i8PtrTy),
- ::llvm::ConstantPointerNull::get(i8PtrTy),
- ::llvm::ConstantPointerNull::get(i8PtrTy),
+ llvm::ConstantPointerNull::get(i8PtrTy),
+ llvm::ConstantPointerNull::get(i8PtrTy),
});
auto size = jit->builder->CreateCall(coro_size, {});
auto frame = jit->builder->CreateCall(allocFrame, { size });
@@ -4017,18 +4048,18 @@
// Build the suspend block
jit->builder->SetInsertPoint(jit->coroutine.suspendBlock);
- jit->builder->CreateCall(coro_end, { jit->coroutine.handle, ::llvm::ConstantInt::get(i1Ty, 0) });
+ jit->builder->CreateCall(coro_end, { jit->coroutine.handle, llvm::ConstantInt::get(i1Ty, 0) });
jit->builder->CreateRet(jit->coroutine.handle);
// Build the end block
jit->builder->SetInsertPoint(jit->coroutine.endBlock);
auto action = jit->builder->CreateCall(coro_suspend, {
- ::llvm::ConstantTokenNone::get(jit->context),
- ::llvm::ConstantInt::get(i1Ty, 1), // final: true
+ llvm::ConstantTokenNone::get(jit->context),
+ llvm::ConstantInt::get(i1Ty, 1), // final: true
});
auto switch_ = jit->builder->CreateSwitch(action, jit->coroutine.suspendBlock, 3);
- // switch_->addCase(::llvm::ConstantInt::get(i8Ty, SuspendActionResume), trapBlock); // TODO: Trap attempting to resume after final suspend
- switch_->addCase(::llvm::ConstantInt::get(i8Ty, SuspendActionDestroy), jit->coroutine.destroyBlock);
+ // switch_->addCase(llvm::ConstantInt::get(i8Ty, SuspendActionResume), trapBlock); // TODO: Trap attempting to resume after final suspend
+ switch_->addCase(llvm::ConstantInt::get(i8Ty, SuspendActionDestroy), jit->coroutine.destroyBlock);
// Build the destroy block
jit->builder->SetInsertPoint(jit->coroutine.destroyBlock);
@@ -4049,9 +4080,9 @@
// Coroutines are initially created as a regular function.
// Upon the first call to Yield(), the function is promoted to a true
// coroutine.
- auto voidTy = ::llvm::Type::getVoidTy(jit->context);
- auto i1Ty = ::llvm::Type::getInt1Ty(jit->context);
- auto i8PtrTy = ::llvm::Type::getInt8PtrTy(jit->context);
+ auto voidTy = llvm::Type::getVoidTy(jit->context);
+ auto i1Ty = llvm::Type::getInt1Ty(jit->context);
+ auto i8PtrTy = llvm::Type::getInt8PtrTy(jit->context);
auto handleTy = i8PtrTy;
auto boolTy = i1Ty;
auto promiseTy = T(YieldType);
@@ -4095,11 +4126,11 @@
Variable::materializeAll();
// Types
- auto i1Ty = ::llvm::Type::getInt1Ty(jit->context);
- auto i8Ty = ::llvm::Type::getInt8Ty(jit->context);
+ auto i1Ty = llvm::Type::getInt1Ty(jit->context);
+ auto i8Ty = llvm::Type::getInt8Ty(jit->context);
// Intrinsics
- auto coro_suspend = ::llvm::Intrinsic::getDeclaration(jit->module.get(), ::llvm::Intrinsic::coro_suspend);
+ auto coro_suspend = llvm::Intrinsic::getDeclaration(jit->module.get(), llvm::Intrinsic::coro_suspend);
// Create a block to resume execution.
auto resumeBlock = llvm::BasicBlock::Create(jit->context, "resume", jit->function);
@@ -4107,12 +4138,12 @@
// Store the promise (yield value)
jit->builder->CreateStore(V(val), jit->coroutine.promise);
auto action = jit->builder->CreateCall(coro_suspend, {
- ::llvm::ConstantTokenNone::get(jit->context),
- ::llvm::ConstantInt::get(i1Ty, 0), // final: true
+ llvm::ConstantTokenNone::get(jit->context),
+ llvm::ConstantInt::get(i1Ty, 0), // final: true
});
auto switch_ = jit->builder->CreateSwitch(action, jit->coroutine.suspendBlock, 3);
- switch_->addCase(::llvm::ConstantInt::get(i8Ty, SuspendActionResume), resumeBlock);
- switch_->addCase(::llvm::ConstantInt::get(i8Ty, SuspendActionDestroy), jit->coroutine.destroyBlock);
+ switch_->addCase(llvm::ConstantInt::get(i8Ty, SuspendActionResume), resumeBlock);
+ switch_->addCase(llvm::ConstantInt::get(i8Ty, SuspendActionDestroy), jit->coroutine.destroyBlock);
// Continue building in the resume block.
jit->builder->SetInsertPoint(resumeBlock);
diff --git a/src/Reactor/Reactor.hpp b/src/Reactor/Reactor.hpp
index b943920..1e08698 100644
--- a/src/Reactor/Reactor.hpp
+++ b/src/Reactor/Reactor.hpp
@@ -3503,8 +3503,8 @@
IFELSE_NUM__
};
-#define If(cond) \
- for(IfElseData ifElse__(cond); ifElse__ < IFELSE_NUM__; ++ifElse__) \
+#define If(cond) \
+ for(IfElseData ifElse__{ cond }; ifElse__ < IFELSE_NUM__; ++ifElse__) \
if(ifElse__ == IF_BLOCK__)
#define Else \
diff --git a/third_party/subzero/src/IceBitVector.h b/third_party/subzero/src/IceBitVector.h
index f81810f..742d2b4 100644
--- a/third_party/subzero/src/IceBitVector.h
+++ b/third_party/subzero/src/IceBitVector.h
@@ -768,8 +768,10 @@
Capacity = std::max(NumBitWords(NewSize), Capacity * 2);
assert(Capacity > 0 && "realloc-ing zero space");
auto *NewBits = Alloc.allocate(Capacity);
- std::memcpy(Bits, NewBits, OldCapacity * sizeof(BitWord));
- Alloc.deallocate(Bits, OldCapacity);
+ if (Bits) {
+ std::memcpy(NewBits, Bits, OldCapacity * sizeof(BitWord));
+ Alloc.deallocate(Bits, OldCapacity);
+ }
Bits = NewBits;
clear_unused_bits();
