src/Reactor/LLVMReactor.cpp

// Copyright 2016 The SwiftShader Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include "Reactor.hpp"

#include "x86.hpp"
#include "CPUID.hpp"
#include "Thread.hpp"
#include "ExecutableMemory.hpp"
#include "MutexLock.hpp"

#undef min
#undef max

#if REACTOR_LLVM_VERSION < 7
	#include "llvm/Analysis/LoopPass.h"
	#include "llvm/Constants.h"
	#include "llvm/Function.h"
	#include "llvm/GlobalVariable.h"
	#include "llvm/Intrinsics.h"
	#include "llvm/LLVMContext.h"
	#include "llvm/Module.h"
	#include "llvm/PassManager.h"
	#include "llvm/Support/IRBuilder.h"
	#include "llvm/Support/TargetSelect.h"
	#include "llvm/Target/TargetData.h"
	#include "llvm/Target/TargetOptions.h"
	#include "llvm/Transforms/Scalar.h"
	#include "../lib/ExecutionEngine/JIT/JIT.h"

	#include "LLVMRoutine.hpp"
	#include "LLVMRoutineManager.hpp"

	#define ARGS(...) __VA_ARGS__
#else
	#include "llvm/Analysis/LoopPass.h"
	#include "llvm/ExecutionEngine/ExecutionEngine.h"
	#include "llvm/ExecutionEngine/JITSymbol.h"
	#include "llvm/ExecutionEngine/Orc/CompileUtils.h"
	#include "llvm/ExecutionEngine/Orc/IRCompileLayer.h"
	#include "llvm/ExecutionEngine/Orc/LambdaResolver.h"
	#include "llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h"
	#include "llvm/ExecutionEngine/RTDyldMemoryManager.h"
	#include "llvm/ExecutionEngine/SectionMemoryManager.h"
	#include "llvm/IR/Constants.h"
	#include "llvm/IR/DataLayout.h"
	#include "llvm/IR/Function.h"
	#include "llvm/IR/GlobalVariable.h"
	#include "llvm/IR/IRBuilder.h"
	#include "llvm/IR/Intrinsics.h"
	#include "llvm/IR/LLVMContext.h"
	#include "llvm/IR/LegacyPassManager.h"
	#include "llvm/IR/Module.h"
	#include "llvm/Support/Error.h"
	#include "llvm/Support/TargetSelect.h"
	#include "llvm/Target/TargetOptions.h"
	#include "llvm/Transforms/InstCombine/InstCombine.h"
	#include "llvm/Transforms/Scalar.h"
	#include "llvm/Transforms/Scalar/GVN.h"

	#include "LLVMRoutine.hpp"

	#define ARGS(...) {__VA_ARGS__}
	#define CreateCall2 CreateCall
	#define CreateCall3 CreateCall

	#include <unordered_map>
#endif

#include <numeric>
#include <fstream>

#if defined(__i386__) || defined(__x86_64__)
#include <xmmintrin.h>
#endif

#include <math.h>

#if defined(__x86_64__) && defined(_WIN32)
extern "C" void X86CompilationCallback()
{
	assert(false);   // UNIMPLEMENTED
}
#endif

#if defined(_WIN32)
extern "C"
{
	bool (*CodeAnalystInitialize)() = 0;
	void (*CodeAnalystCompleteJITLog)() = 0;
	bool (*CodeAnalystLogJITCode)(const void *jitCodeStartAddr, unsigned int jitCodeSize, const wchar_t *functionName) = 0;
}
#endif

#if REACTOR_LLVM_VERSION < 7
namespace llvm
{
	extern bool JITEmitDebugInfo;
}
#endif

namespace rr
{
	class LLVMReactorJIT;
}

namespace
{
	rr::LLVMReactorJIT *reactorJIT = nullptr;
	llvm::IRBuilder<> *builder = nullptr;
	llvm::LLVMContext *context = nullptr;
	llvm::Module *module = nullptr;
	llvm::Function *function = nullptr;

	rr::MutexLock codegenMutex;

#if REACTOR_LLVM_VERSION >= 7
	llvm::Value *lowerPAVG(llvm::Value *x, llvm::Value *y)
	{
		llvm::VectorType *ty = llvm::cast<llvm::VectorType>(x->getType());

		llvm::VectorType *extTy =
			llvm::VectorType::getExtendedElementVectorType(ty);
		x = ::builder->CreateZExt(x, extTy);
		y = ::builder->CreateZExt(y, extTy);

		// (x + y + 1) >> 1
		llvm::Constant *one = llvm::ConstantInt::get(extTy, 1);
		llvm::Value *res = ::builder->CreateAdd(x, y);
		res = ::builder->CreateAdd(res, one);
		res = ::builder->CreateLShr(res, one);
		return ::builder->CreateTrunc(res, ty);
	}

	llvm::Value *lowerPMINMAX(llvm::Value *x, llvm::Value *y,
	                          llvm::ICmpInst::Predicate pred)
	{
		return ::builder->CreateSelect(::builder->CreateICmp(pred, x, y), x, y);
	}

	llvm::Value *lowerPCMP(llvm::ICmpInst::Predicate pred, llvm::Value *x,
	                       llvm::Value *y, llvm::Type *dstTy)
	{
		return ::builder->CreateSExt(::builder->CreateICmp(pred, x, y), dstTy, "");
	}

#if defined(__i386__) || defined(__x86_64__)
	llvm::Value *lowerPMOV(llvm::Value *op, llvm::Type *dstType, bool sext)
	{
		llvm::VectorType *srcTy = llvm::cast<llvm::VectorType>(op->getType());
		llvm::VectorType *dstTy = llvm::cast<llvm::VectorType>(dstType);

		llvm::Value *undef = llvm::UndefValue::get(srcTy);
		llvm::SmallVector<uint32_t, 16> mask(dstTy->getNumElements());
		std::iota(mask.begin(), mask.end(), 0);
		llvm::Value *v = ::builder->CreateShuffleVector(op, undef, mask);

		return sext ? ::builder->CreateSExt(v, dstTy)
		            : ::builder->CreateZExt(v, dstTy);
	}

	llvm::Value *lowerPABS(llvm::Value *v)
	{
		llvm::Value *zero = llvm::Constant::getNullValue(v->getType());
		llvm::Value *cmp = ::builder->CreateICmp(llvm::ICmpInst::ICMP_SGT, v, zero);
		llvm::Value *neg = ::builder->CreateNeg(v);
		return ::builder->CreateSelect(cmp, v, neg);
	}
#endif  // defined(__i386__) || defined(__x86_64__)

#if !defined(__i386__) && !defined(__x86_64__)
	llvm::Value *lowerPFMINMAX(llvm::Value *x, llvm::Value *y,
	                           llvm::FCmpInst::Predicate pred)
	{
		return ::builder->CreateSelect(::builder->CreateFCmp(pred, x, y), x, y);
	}

	llvm::Value *lowerRound(llvm::Value *x)
	{
		llvm::Function *nearbyint = llvm::Intrinsic::getDeclaration(
			::module, llvm::Intrinsic::nearbyint, {x->getType()});
		return ::builder->CreateCall(nearbyint, ARGS(x));
	}

	llvm::Value *lowerRoundInt(llvm::Value *x, llvm::Type *ty)
	{
		return ::builder->CreateFPToSI(lowerRound(x), ty);
	}

	llvm::Value *lowerFloor(llvm::Value *x)
	{
		llvm::Function *floor = llvm::Intrinsic::getDeclaration(
			::module, llvm::Intrinsic::floor, {x->getType()});
		return ::builder->CreateCall(floor, ARGS(x));
	}

	llvm::Value *lowerTrunc(llvm::Value *x)
	{
		llvm::Function *trunc = llvm::Intrinsic::getDeclaration(
			::module, llvm::Intrinsic::trunc, {x->getType()});
		return ::builder->CreateCall(trunc, ARGS(x));
	}

	// Packed add/sub saturatation
	llvm::Value *lowerPSAT(llvm::Value *x, llvm::Value *y, bool isAdd, bool isSigned)
	{
		llvm::VectorType *ty = llvm::cast<llvm::VectorType>(x->getType());
		llvm::VectorType *extTy = llvm::VectorType::getExtendedElementVectorType(ty);

		unsigned numBits = ty->getScalarSizeInBits();

		llvm::Value *max, *min, *extX, *extY;
		if (isSigned)
		{
			max = llvm::ConstantInt::get(extTy, (1LL << (numBits - 1)) - 1, true);
			min = llvm::ConstantInt::get(extTy, (-1LL << (numBits - 1)), true);
			extX = ::builder->CreateSExt(x, extTy);
			extY = ::builder->CreateSExt(y, extTy);
		}
		else
		{
			assert(numBits <= 64);
			uint64_t maxVal = (numBits == 64) ? ~0ULL : (1ULL << numBits) - 1;
			max = llvm::ConstantInt::get(extTy, maxVal, false);
			min = llvm::ConstantInt::get(extTy, 0, false);
			extX = ::builder->CreateZExt(x, extTy);
			extY = ::builder->CreateZExt(y, extTy);
		}

		llvm::Value *res = isAdd ? ::builder->CreateAdd(extX, extY)
		                         : ::builder->CreateSub(extX, extY);

		res = lowerPMINMAX(res, min, llvm::ICmpInst::ICMP_SGT);
		res = lowerPMINMAX(res, max, llvm::ICmpInst::ICMP_SLT);

		return ::builder->CreateTrunc(res, ty);
	}

	llvm::Value *lowerPUADDSAT(llvm::Value *x, llvm::Value *y)
	{
		return lowerPSAT(x, y, true, false);
	}

	llvm::Value *lowerPSADDSAT(llvm::Value *x, llvm::Value *y)
	{
		return lowerPSAT(x, y, true, true);
	}

	llvm::Value *lowerPUSUBSAT(llvm::Value *x, llvm::Value *y)
	{
		return lowerPSAT(x, y, false, false);
	}

	llvm::Value *lowerPSSUBSAT(llvm::Value *x, llvm::Value *y)
	{
		return lowerPSAT(x, y, false, true);
	}

	llvm::Value *lowerSQRT(llvm::Value *x)
	{
		llvm::Function *sqrt = llvm::Intrinsic::getDeclaration(
			::module, llvm::Intrinsic::sqrt, {x->getType()});
		return ::builder->CreateCall(sqrt, ARGS(x));
	}

	llvm::Value *lowerRCP(llvm::Value *x)
	{
		llvm::Type *ty = x->getType();
		llvm::Constant *one;
		if (llvm::VectorType *vectorTy = llvm::dyn_cast<llvm::VectorType>(ty))
		{
			one = llvm::ConstantVector::getSplat(
				vectorTy->getNumElements(),
				llvm::ConstantFP::get(vectorTy->getElementType(), 1));
		}
		else
		{
			one = llvm::ConstantFP::get(ty, 1);
		}
		return ::builder->CreateFDiv(one, x);
	}

	llvm::Value *lowerRSQRT(llvm::Value *x)
	{
		return lowerRCP(lowerSQRT(x));
	}

	llvm::Value *lowerVectorShl(llvm::Value *x, uint64_t scalarY)
	{
		llvm::VectorType *ty = llvm::cast<llvm::VectorType>(x->getType());
		llvm::Value *y = llvm::ConstantVector::getSplat(
			ty->getNumElements(),
			llvm::ConstantInt::get(ty->getElementType(), scalarY));
		return ::builder->CreateShl(x, y);
	}

	llvm::Value *lowerVectorAShr(llvm::Value *x, uint64_t scalarY)
	{
		llvm::VectorType *ty = llvm::cast<llvm::VectorType>(x->getType());
		llvm::Value *y = llvm::ConstantVector::getSplat(
			ty->getNumElements(),
			llvm::ConstantInt::get(ty->getElementType(), scalarY));
		return ::builder->CreateAShr(x, y);
	}

	llvm::Value *lowerVectorLShr(llvm::Value *x, uint64_t scalarY)
	{
		llvm::VectorType *ty = llvm::cast<llvm::VectorType>(x->getType());
		llvm::Value *y = llvm::ConstantVector::getSplat(
			ty->getNumElements(),
			llvm::ConstantInt::get(ty->getElementType(), scalarY));
		return ::builder->CreateLShr(x, y);
	}

	llvm::Value *lowerMulAdd(llvm::Value *x, llvm::Value *y)
	{
		llvm::VectorType *ty = llvm::cast<llvm::VectorType>(x->getType());
		llvm::VectorType *extTy = llvm::VectorType::getExtendedElementVectorType(ty);

		llvm::Value *extX = ::builder->CreateSExt(x, extTy);
		llvm::Value *extY = ::builder->CreateSExt(y, extTy);
		llvm::Value *mult = ::builder->CreateMul(extX, extY);

		llvm::Value *undef = llvm::UndefValue::get(extTy);

		llvm::SmallVector<uint32_t, 16> evenIdx;
		llvm::SmallVector<uint32_t, 16> oddIdx;
		for (uint64_t i = 0, n = ty->getNumElements(); i < n; i += 2)
		{
			evenIdx.push_back(i);
			oddIdx.push_back(i + 1);
		}

		llvm::Value *lhs = ::builder->CreateShuffleVector(mult, undef, evenIdx);
		llvm::Value *rhs = ::builder->CreateShuffleVector(mult, undef, oddIdx);
		return ::builder->CreateAdd(lhs, rhs);
	}

	llvm::Value *lowerMulHigh(llvm::Value *x, llvm::Value *y, bool sext)
	{
		llvm::VectorType *ty = llvm::cast<llvm::VectorType>(x->getType());
		llvm::VectorType *extTy = llvm::VectorType::getExtendedElementVectorType(ty);

		llvm::Value *extX, *extY;
		if (sext)
		{
			extX = ::builder->CreateSExt(x, extTy);
			extY = ::builder->CreateSExt(y, extTy);
		}
		else
		{
			extX = ::builder->CreateZExt(x, extTy);
			extY = ::builder->CreateZExt(y, extTy);
		}

		llvm::Value *mult = ::builder->CreateMul(extX, extY);

		llvm::IntegerType *intTy = llvm::cast<llvm::IntegerType>(ty->getElementType());
		llvm::Value *mulh = ::builder->CreateAShr(mult, intTy->getIntegerBitWidth());
		return ::builder->CreateTrunc(mulh, ty);
	}

	llvm::Value *lowerPack(llvm::Value *x, llvm::Value *y, bool isSigned)
	{
		llvm::VectorType *srcTy = llvm::cast<llvm::VectorType>(x->getType());
		llvm::VectorType *dstTy = llvm::VectorType::getTruncatedElementVectorType(srcTy);

		llvm::IntegerType *dstElemTy =
			llvm::cast<llvm::IntegerType>(dstTy->getElementType());

		uint64_t truncNumBits = dstElemTy->getIntegerBitWidth();
		assert(truncNumBits < 64 && "shift 64 must be handled separately");
		llvm::Constant *max, *min;
		if (isSigned)
		{
			max = llvm::ConstantInt::get(srcTy, (1LL << (truncNumBits - 1)) - 1, true);
			min = llvm::ConstantInt::get(srcTy, (-1LL << (truncNumBits - 1)), true);
		}
		else
		{
			max = llvm::ConstantInt::get(srcTy, (1ULL << truncNumBits) - 1, false);
			min = llvm::ConstantInt::get(srcTy, 0, false);
		}

		x = lowerPMINMAX(x, min, llvm::ICmpInst::ICMP_SGT);
		x = lowerPMINMAX(x, max, llvm::ICmpInst::ICMP_SLT);
		y = lowerPMINMAX(y, min, llvm::ICmpInst::ICMP_SGT);
		y = lowerPMINMAX(y, max, llvm::ICmpInst::ICMP_SLT);

		x = ::builder->CreateTrunc(x, dstTy);
		y = ::builder->CreateTrunc(y, dstTy);

		llvm::SmallVector<uint32_t, 16> index(srcTy->getNumElements() * 2);
		std::iota(index.begin(), index.end(), 0);

		return ::builder->CreateShuffleVector(x, y, index);
	}

	llvm::Value *lowerSignMask(llvm::Value *x, llvm::Type *retTy)
	{
		llvm::VectorType *ty = llvm::cast<llvm::VectorType>(x->getType());
		llvm::Constant *zero = llvm::ConstantInt::get(ty, 0);
		llvm::Value *cmp = ::builder->CreateICmpSLT(x, zero);

		llvm::Value *ret = ::builder->CreateZExt(
			::builder->CreateExtractElement(cmp, static_cast<uint64_t>(0)), retTy);
		for (uint64_t i = 1, n = ty->getNumElements(); i < n; ++i)
		{
			llvm::Value *elem = ::builder->CreateZExt(
				::builder->CreateExtractElement(cmp, i), retTy);
			ret = ::builder->CreateOr(ret, ::builder->CreateShl(elem, i));
		}
		return ret;
	}

	llvm::Value *lowerFPSignMask(llvm::Value *x, llvm::Type *retTy)
	{
		llvm::VectorType *ty = llvm::cast<llvm::VectorType>(x->getType());
		llvm::Constant *zero = llvm::ConstantFP::get(ty, 0);
		llvm::Value *cmp = ::builder->CreateFCmpULT(x, zero);

		llvm::Value *ret = ::builder->CreateZExt(
			::builder->CreateExtractElement(cmp, static_cast<uint64_t>(0)), retTy);
		for (uint64_t i = 1, n = ty->getNumElements(); i < n; ++i)
		{
			llvm::Value *elem = ::builder->CreateZExt(
				::builder->CreateExtractElement(cmp, i), retTy);
			ret = ::builder->CreateOr(ret, ::builder->CreateShl(elem, i));
		}
		return ret;
	}
#endif  // !defined(__i386__) && !defined(__x86_64__)
#endif  // REACTOR_LLVM_VERSION >= 7
}

namespace rr
{
#if REACTOR_LLVM_VERSION < 7
	class LLVMReactorJIT
	{
	private:
		std::string arch;
		llvm::SmallVector<std::string, 16> mattrs;
		llvm::ExecutionEngine *executionEngine;
		LLVMRoutineManager *routineManager;

	public:
		LLVMReactorJIT(const std::string &arch_,
		               const llvm::SmallVectorImpl<std::string> &mattrs_) :
			arch(arch_),
			mattrs(mattrs_.begin(), mattrs_.end()),
			executionEngine(nullptr),
			routineManager(nullptr)
		{
		}

		void startSession()
		{
			std::string error;

			::module = new llvm::Module("", *::context);

			routineManager = new LLVMRoutineManager();

			llvm::TargetMachine *targetMachine =
				llvm::EngineBuilder::selectTarget(
					::module, arch, "", mattrs, llvm::Reloc::Default,
					llvm::CodeModel::JITDefault, &error);

			executionEngine = llvm::JIT::createJIT(
				::module, &error, routineManager, llvm::CodeGenOpt::Aggressive,
				true, targetMachine);
		}

		void endSession()
		{
			delete executionEngine;
			executionEngine = nullptr;
			routineManager = nullptr;

			::function = nullptr;
			::module = nullptr;
		}

		LLVMRoutine *acquireRoutine(llvm::Function *func)
		{
			void *entry = executionEngine->getPointerToFunction(::function);
			return routineManager->acquireRoutine(entry);
		}

		void optimize(llvm::Module *module)
		{
			static llvm::PassManager *passManager = nullptr;

			if(!passManager)
			{
				passManager = new llvm::PassManager();

				passManager->add(new llvm::TargetData(*executionEngine->getTargetData()));
				passManager->add(llvm::createScalarReplAggregatesPass());

				for(int pass = 0; pass < 10 && optimization[pass] != Disabled; pass++)
				{
					switch(optimization[pass])
					{
					case Disabled:                                                                       break;
					case CFGSimplification:    passManager->add(llvm::createCFGSimplificationPass());    break;
					case LICM:                 passManager->add(llvm::createLICMPass());                 break;
					case AggressiveDCE:        passManager->add(llvm::createAggressiveDCEPass());        break;
					case GVN:                  passManager->add(llvm::createGVNPass());                  break;
					case InstructionCombining: passManager->add(llvm::createInstructionCombiningPass()); break;
					case Reassociate:          passManager->add(llvm::createReassociatePass());          break;
					case DeadStoreElimination: passManager->add(llvm::createDeadStoreEliminationPass()); break;
					case SCCP:                 passManager->add(llvm::createSCCPPass());                 break;
					case ScalarReplAggregates: passManager->add(llvm::createScalarReplAggregatesPass()); break;
					default:
						assert(false);
					}
				}
			}

			passManager->run(*::module);
		}
	};
#else
	class ExternalFunctionSymbolResolver
	{
	private:
		using FunctionMap = std::unordered_map<std::string, void *>;
		FunctionMap func_;

	public:
		ExternalFunctionSymbolResolver()
		{
			func_.emplace("floorf", reinterpret_cast<void*>(floorf));
			func_.emplace("nearbyintf", reinterpret_cast<void*>(nearbyintf));
			func_.emplace("truncf", reinterpret_cast<void*>(truncf));
		}

		void *findSymbol(const std::string &name) const
		{
			FunctionMap::const_iterator it = func_.find(name);
			return (it != func_.end()) ? it->second : nullptr;
		}
	};

	class LLVMReactorJIT
	{
	private:
		using ObjLayer = llvm::orc::RTDyldObjectLinkingLayer;
		using CompileLayer = llvm::orc::IRCompileLayer<ObjLayer, llvm::orc::SimpleCompiler>;

		llvm::orc::ExecutionSession session;
		ExternalFunctionSymbolResolver externalSymbolResolver;
		std::shared_ptr<llvm::orc::SymbolResolver> resolver;
		std::unique_ptr<llvm::TargetMachine> targetMachine;
		const llvm::DataLayout dataLayout;
		ObjLayer objLayer;
		CompileLayer compileLayer;
		size_t emittedFunctionsNum;

	public:
		LLVMReactorJIT(const char *arch, const llvm::SmallVectorImpl<std::string>& mattrs,
					   const llvm::TargetOptions &targetOpts):
			resolver(createLegacyLookupResolver(
				session,
				[this](const std::string &name) {
					void *func = externalSymbolResolver.findSymbol(name);
					if (func != nullptr)
					{
						return llvm::JITSymbol(
							reinterpret_cast<uintptr_t>(func), llvm::JITSymbolFlags::Absolute);
					}

					return objLayer.findSymbol(name, true);
				},
				[](llvm::Error err) {
					if (err)
					{
						// TODO: Log the symbol resolution errors.
						return;
					}
				})),
			targetMachine(llvm::EngineBuilder()
				.setMArch(arch)
				.setMAttrs(mattrs)
				.setTargetOptions(targetOpts)
				.selectTarget()),
			dataLayout(targetMachine->createDataLayout()),
			objLayer(
				session,
				[this](llvm::orc::VModuleKey) {
					return ObjLayer::Resources{
						std::make_shared<llvm::SectionMemoryManager>(),
						resolver};
				}),
			compileLayer(objLayer, llvm::orc::SimpleCompiler(*targetMachine)),
			emittedFunctionsNum(0)
		{
		}

		void startSession()
		{
			::module = new llvm::Module("", *::context);
		}

		void endSession()
		{
			::function = nullptr;
			::module = nullptr;
		}

		LLVMRoutine *acquireRoutine(llvm::Function *func)
		{
			std::string name = "f" + llvm::Twine(emittedFunctionsNum++).str();
			func->setName(name);
			func->setLinkage(llvm::GlobalValue::ExternalLinkage);
			func->setDoesNotThrow();

			std::unique_ptr<llvm::Module> mod(::module);
			::module = nullptr;
			mod->setDataLayout(dataLayout);

			auto moduleKey = session.allocateVModule();
			llvm::cantFail(compileLayer.addModule(moduleKey, std::move(mod)));

			llvm::JITSymbol symbol = compileLayer.findSymbolIn(moduleKey, name, false);

			llvm::Expected<llvm::JITTargetAddress> expectAddr = symbol.getAddress();
			if (!expectAddr)
			{
				return nullptr;
			}

			void *addr = reinterpret_cast<void *>(static_cast<intptr_t>(expectAddr.get()));
			return new LLVMRoutine(addr, releaseRoutineCallback, this, moduleKey);
		}

		void optimize(llvm::Module *module)
		{
			std::unique_ptr<llvm::legacy::PassManager> passManager(
				new llvm::legacy::PassManager());

			passManager->add(llvm::createSROAPass());

			for(int pass = 0; pass < 10 && optimization[pass] != Disabled; pass++)
			{
				switch(optimization[pass])
				{
				case Disabled:                                                                       break;
				case CFGSimplification:    passManager->add(llvm::createCFGSimplificationPass());    break;
				case LICM:                 passManager->add(llvm::createLICMPass());                 break;
				case AggressiveDCE:        passManager->add(llvm::createAggressiveDCEPass());        break;
				case GVN:                  passManager->add(llvm::createGVNPass());                  break;
				case InstructionCombining: passManager->add(llvm::createInstructionCombiningPass()); break;
				case Reassociate:          passManager->add(llvm::createReassociatePass());          break;
				case DeadStoreElimination: passManager->add(llvm::createDeadStoreEliminationPass()); break;
				case SCCP:                 passManager->add(llvm::createSCCPPass());                 break;
				case ScalarReplAggregates: passManager->add(llvm::createSROAPass());                 break;
				default:
				                           assert(false);
				}
			}

			passManager->run(*::module);
		}

	private:
		void releaseRoutineModule(llvm::orc::VModuleKey moduleKey)
		{
			llvm::cantFail(compileLayer.removeModule(moduleKey));
		}

		static void releaseRoutineCallback(LLVMReactorJIT *jit, uint64_t moduleKey)
		{
			jit->releaseRoutineModule(moduleKey);
		}
	};
#endif

	Optimization optimization[10] = {InstructionCombining, Disabled};

	enum EmulatedType
	{
		Type_v2i32,
		Type_v4i16,
		Type_v2i16,
		Type_v8i8,
		Type_v4i8,
		Type_v2f32,
		EmulatedTypeCount
	};

	llvm::Type *T(Type *t)
	{
		uintptr_t type = reinterpret_cast<uintptr_t>(t);
		if(type < EmulatedTypeCount)
		{
			// Use 128-bit vectors to implement logically shorter ones.
			switch(type)
			{
			case Type_v2i32: return T(Int4::getType());
			case Type_v4i16: return T(Short8::getType());
			case Type_v2i16: return T(Short8::getType());
			case Type_v8i8:  return T(Byte16::getType());
			case Type_v4i8:  return T(Byte16::getType());
			case Type_v2f32: return T(Float4::getType());
			default: assert(false);
			}
		}

		return reinterpret_cast<llvm::Type*>(t);
	}

	inline Type *T(llvm::Type *t)
	{
		return reinterpret_cast<Type*>(t);
	}

	Type *T(EmulatedType t)
	{
		return reinterpret_cast<Type*>(t);
	}

	inline llvm::Value *V(Value *t)
	{
		return reinterpret_cast<llvm::Value*>(t);
	}

	inline Value *V(llvm::Value *t)
	{
		return reinterpret_cast<Value*>(t);
	}

	inline std::vector<llvm::Type*> &T(std::vector<Type*> &t)
	{
		return reinterpret_cast<std::vector<llvm::Type*>&>(t);
	}

	inline llvm::BasicBlock *B(BasicBlock *t)
	{
		return reinterpret_cast<llvm::BasicBlock*>(t);
	}

	inline BasicBlock *B(llvm::BasicBlock *t)
	{
		return reinterpret_cast<BasicBlock*>(t);
	}

	static size_t typeSize(Type *type)
	{
		uintptr_t t = reinterpret_cast<uintptr_t>(type);
		if(t < EmulatedTypeCount)
		{
			switch(t)
			{
			case Type_v2i32: return 8;
			case Type_v4i16: return 8;
			case Type_v2i16: return 4;
			case Type_v8i8:  return 8;
			case Type_v4i8:  return 4;
			case Type_v2f32: return 8;
			default: assert(false);
			}
		}

		return T(type)->getPrimitiveSizeInBits() / 8;
	}

	static unsigned int elementCount(Type *type)
	{
		uintptr_t t = reinterpret_cast<uintptr_t>(type);
		if(t < EmulatedTypeCount)
		{
			switch(t)
			{
			case Type_v2i32: return 2;
			case Type_v4i16: return 4;
			case Type_v2i16: return 2;
			case Type_v8i8:  return 8;
			case Type_v4i8:  return 4;
			case Type_v2f32: return 2;
			default: assert(false);
			}
		}

		return llvm::cast<llvm::VectorType>(T(type))->getNumElements();
	}

	Nucleus::Nucleus()
	{
		::codegenMutex.lock();   // Reactor and LLVM are currently not thread safe

		llvm::InitializeNativeTarget();

#if REACTOR_LLVM_VERSION >= 7
		llvm::InitializeNativeTargetAsmPrinter();
		llvm::InitializeNativeTargetAsmParser();
#endif

		if(!::context)
		{
			::context = new llvm::LLVMContext();
		}

		#if defined(__x86_64__)
			static const char arch[] = "x86-64";
		#elif defined(__i386__)
			static const char arch[] = "x86";
		#elif defined(__aarch64__)
			static const char arch[] = "arm64";
		#elif defined(__arm__)
			static const char arch[] = "arm";
		#elif defined(__mips__)
			static const char arch[] = "mipsel";
		#else
		#error "unknown architecture"
		#endif

		llvm::SmallVector<std::string, 1> mattrs;
#if defined(__i386__) || defined(__x86_64__)
		mattrs.push_back(CPUID::supportsMMX()    ? "+mmx"    : "-mmx");
		mattrs.push_back(CPUID::supportsCMOV()   ? "+cmov"   : "-cmov");
		mattrs.push_back(CPUID::supportsSSE()    ? "+sse"    : "-sse");
		mattrs.push_back(CPUID::supportsSSE2()   ? "+sse2"   : "-sse2");
		mattrs.push_back(CPUID::supportsSSE3()   ? "+sse3"   : "-sse3");
		mattrs.push_back(CPUID::supportsSSSE3()  ? "+ssse3"  : "-ssse3");
#if REACTOR_LLVM_VERSION < 7
		mattrs.push_back(CPUID::supportsSSE4_1() ? "+sse41"  : "-sse41");
#else
		mattrs.push_back(CPUID::supportsSSE4_1() ? "+sse4.1" : "-sse4.1");
#endif
#elif defined(__arm__)
#if __ARM_ARCH >= 8
		mattrs.push_back("+armv8-a");
#else
		// armv7-a requires compiler-rt routines; otherwise, compiled kernel
		// might fail to link.
#endif
#endif

#if REACTOR_LLVM_VERSION < 7
		llvm::JITEmitDebugInfo = false;
		llvm::UnsafeFPMath = true;
		// llvm::NoInfsFPMath = true;
		// llvm::NoNaNsFPMath = true;
#else
		llvm::TargetOptions targetOpts;
		targetOpts.UnsafeFPMath = false;
		// targetOpts.NoInfsFPMath = true;
		// targetOpts.NoNaNsFPMath = true;
#endif

		if(!::reactorJIT)
		{
#if REACTOR_LLVM_VERSION < 7
			::reactorJIT = new LLVMReactorJIT(arch, mattrs);
#else
			::reactorJIT = new LLVMReactorJIT(arch, mattrs, targetOpts);
#endif
		}

		::reactorJIT->startSession();

		if(!::builder)
		{
			::builder = new llvm::IRBuilder<>(*::context);

			#if defined(_WIN32) && REACTOR_LLVM_VERSION < 7
				HMODULE CodeAnalyst = LoadLibrary("CAJitNtfyLib.dll");
				if(CodeAnalyst)
				{
					CodeAnalystInitialize = (bool(*)())GetProcAddress(CodeAnalyst, "CAJIT_Initialize");
					CodeAnalystCompleteJITLog = (void(*)())GetProcAddress(CodeAnalyst, "CAJIT_CompleteJITLog");
					CodeAnalystLogJITCode = (bool(*)(const void*, unsigned int, const wchar_t*))GetProcAddress(CodeAnalyst, "CAJIT_LogJITCode");

					CodeAnalystInitialize();
				}
			#endif
		}
	}

	Nucleus::~Nucleus()
	{
		::reactorJIT->endSession();

		::codegenMutex.unlock();
	}

	Routine *Nucleus::acquireRoutine(const wchar_t *name, bool runOptimizations)
	{
		if(::builder->GetInsertBlock()->empty() || !::builder->GetInsertBlock()->back().isTerminator())
		{
			llvm::Type *type = ::function->getReturnType();

			if(type->isVoidTy())
			{
				createRetVoid();
			}
			else
			{
				createRet(V(llvm::UndefValue::get(type)));
			}
		}

		if(false)
		{
#if REACTOR_LLVM_VERSION < 7
			std::string error;
#else
			std::error_code error;
#endif
			llvm::raw_fd_ostream file("llvm-dump-unopt.txt", error);
			::module->print(file, 0);
		}

		if(runOptimizations)
		{
			optimize();
		}

		if(false)
		{
#if REACTOR_LLVM_VERSION < 7
			std::string error;
#else
			std::error_code error;
#endif
			llvm::raw_fd_ostream file("llvm-dump-opt.txt", error);
			::module->print(file, 0);
		}

		LLVMRoutine *routine = ::reactorJIT->acquireRoutine(::function);

#if defined(_WIN32) && REACTOR_LLVM_VERSION < 7
		if(CodeAnalystLogJITCode)
		{
			CodeAnalystLogJITCode(routine->getEntry(), routine->getCodeSize(), name);
		}
#endif

		return routine;
	}

	void Nucleus::optimize()
	{
		::reactorJIT->optimize(::module);
	}

	Value *Nucleus::allocateStackVariable(Type *type, int arraySize)
	{
		// Need to allocate it in the entry block for mem2reg to work
		llvm::BasicBlock &entryBlock = ::function->getEntryBlock();

		llvm::Instruction *declaration;

		if(arraySize)
		{
#if REACTOR_LLVM_VERSION < 7
			declaration = new llvm::AllocaInst(T(type), V(Nucleus::createConstantInt(arraySize)));
#else
			declaration = new llvm::AllocaInst(T(type), 0, V(Nucleus::createConstantInt(arraySize)));
#endif
		}
		else
		{
#if REACTOR_LLVM_VERSION < 7
			declaration = new llvm::AllocaInst(T(type), (llvm::Value*)nullptr);
#else
			declaration = new llvm::AllocaInst(T(type), 0, (llvm::Value*)nullptr);
#endif
		}

		entryBlock.getInstList().push_front(declaration);

		return V(declaration);
	}

	BasicBlock *Nucleus::createBasicBlock()
	{
		return B(llvm::BasicBlock::Create(*::context, "", ::function));
	}

	BasicBlock *Nucleus::getInsertBlock()
	{
		return B(::builder->GetInsertBlock());
	}

	void Nucleus::setInsertBlock(BasicBlock *basicBlock)
	{
	//	assert(::builder->GetInsertBlock()->back().isTerminator());
		::builder->SetInsertPoint(B(basicBlock));
	}

	void Nucleus::createFunction(Type *ReturnType, std::vector<Type*> &Params)
	{
		llvm::FunctionType *functionType = llvm::FunctionType::get(T(ReturnType), T(Params), false);
		::function = llvm::Function::Create(functionType, llvm::GlobalValue::InternalLinkage, "", ::module);
		::function->setCallingConv(llvm::CallingConv::C);

		#if defined(_WIN32) && REACTOR_LLVM_VERSION >= 7
			// FIXME(capn):
			// On Windows, stack memory is committed in increments of 4 kB pages, with the last page
			// having a trap which allows the OS to grow the stack. For functions with a stack frame
			// larger than 4 kB this can cause an issue when a variable is accessed beyond the guard
			// page. Therefore the compiler emits a call to __chkstk in the function prolog to probe
			// the stack and ensure all pages have been committed. This is currently broken in LLVM
			// JIT, but we can prevent emitting the stack probe call:
			::function->addFnAttr("stack-probe-size", "1048576");
		#endif

		::builder->SetInsertPoint(llvm::BasicBlock::Create(*::context, "", ::function));
	}

	Value *Nucleus::getArgument(unsigned int index)
	{
		llvm::Function::arg_iterator args = ::function->arg_begin();

		while(index)
		{
			args++;
			index--;
		}

		return V(&*args);
	}

	void Nucleus::createRetVoid()
	{
		::builder->CreateRetVoid();
	}

	void Nucleus::createRet(Value *v)
	{
		::builder->CreateRet(V(v));
	}

	void Nucleus::createBr(BasicBlock *dest)
	{
		::builder->CreateBr(B(dest));
	}

	void Nucleus::createCondBr(Value *cond, BasicBlock *ifTrue, BasicBlock *ifFalse)
	{
		::builder->CreateCondBr(V(cond), B(ifTrue), B(ifFalse));
	}

	Value *Nucleus::createAdd(Value *lhs, Value *rhs)
	{
		return V(::builder->CreateAdd(V(lhs), V(rhs)));
	}

	Value *Nucleus::createSub(Value *lhs, Value *rhs)
	{
		return V(::builder->CreateSub(V(lhs), V(rhs)));
	}

	Value *Nucleus::createMul(Value *lhs, Value *rhs)
	{
		return V(::builder->CreateMul(V(lhs), V(rhs)));
	}

	Value *Nucleus::createUDiv(Value *lhs, Value *rhs)
	{
		return V(::builder->CreateUDiv(V(lhs), V(rhs)));
	}

	Value *Nucleus::createSDiv(Value *lhs, Value *rhs)
	{
		return V(::builder->CreateSDiv(V(lhs), V(rhs)));
	}

	Value *Nucleus::createFAdd(Value *lhs, Value *rhs)
	{
		return V(::builder->CreateFAdd(V(lhs), V(rhs)));
	}

	Value *Nucleus::createFSub(Value *lhs, Value *rhs)
	{
		return V(::builder->CreateFSub(V(lhs), V(rhs)));
	}

	Value *Nucleus::createFMul(Value *lhs, Value *rhs)
	{
		return V(::builder->CreateFMul(V(lhs), V(rhs)));
	}

	Value *Nucleus::createFDiv(Value *lhs, Value *rhs)
	{
		return V(::builder->CreateFDiv(V(lhs), V(rhs)));
	}

	Value *Nucleus::createURem(Value *lhs, Value *rhs)
	{
		return V(::builder->CreateURem(V(lhs), V(rhs)));
	}

	Value *Nucleus::createSRem(Value *lhs, Value *rhs)
	{
		return V(::builder->CreateSRem(V(lhs), V(rhs)));
	}

	Value *Nucleus::createFRem(Value *lhs, Value *rhs)
	{
		return V(::builder->CreateFRem(V(lhs), V(rhs)));
	}

	Value *Nucleus::createShl(Value *lhs, Value *rhs)
	{
		return V(::builder->CreateShl(V(lhs), V(rhs)));
	}

	Value *Nucleus::createLShr(Value *lhs, Value *rhs)
	{
		return V(::builder->CreateLShr(V(lhs), V(rhs)));
	}

	Value *Nucleus::createAShr(Value *lhs, Value *rhs)
	{
		return V(::builder->CreateAShr(V(lhs), V(rhs)));
	}

	Value *Nucleus::createAnd(Value *lhs, Value *rhs)
	{
		return V(::builder->CreateAnd(V(lhs), V(rhs)));
	}

	Value *Nucleus::createOr(Value *lhs, Value *rhs)
	{
		return V(::builder->CreateOr(V(lhs), V(rhs)));
	}

	Value *Nucleus::createXor(Value *lhs, Value *rhs)
	{
		return V(::builder->CreateXor(V(lhs), V(rhs)));
	}

	Value *Nucleus::createNeg(Value *v)
	{
		return V(::builder->CreateNeg(V(v)));
	}

	Value *Nucleus::createFNeg(Value *v)
	{
		return V(::builder->CreateFNeg(V(v)));
	}

	Value *Nucleus::createNot(Value *v)
	{
		return V(::builder->CreateNot(V(v)));
	}

	Value *Nucleus::createLoad(Value *ptr, Type *type, bool isVolatile, unsigned int alignment)
	{
		uintptr_t t = reinterpret_cast<uintptr_t>(type);
		if(t < EmulatedTypeCount)
		{
			switch(t)
			{
			case Type_v2i32:
			case Type_v4i16:
			case Type_v8i8:
			case Type_v2f32:
				return createBitCast(
					createInsertElement(
						V(llvm::UndefValue::get(llvm::VectorType::get(T(Long::getType()), 2))),
						createLoad(createBitCast(ptr, Pointer<Long>::getType()), Long::getType(), isVolatile, alignment),
						0),
					type);
			case Type_v2i16:
			case Type_v4i8:
				if(alignment != 0)   // Not a local variable (all vectors are 128-bit).
				{
					Value *u = V(llvm::UndefValue::get(llvm::VectorType::get(T(Long::getType()), 2)));
					Value *i = createLoad(createBitCast(ptr, Pointer<Int>::getType()), Int::getType(), isVolatile, alignment);
					i = createZExt(i, Long::getType());
					Value *v = createInsertElement(u, i, 0);
					return createBitCast(v, type);
				}
				break;
			default:
				assert(false);
			}
		}

		assert(V(ptr)->getType()->getContainedType(0) == T(type));
		return V(::builder->Insert(new llvm::LoadInst(V(ptr), "", isVolatile, alignment)));
	}

	Value *Nucleus::createStore(Value *value, Value *ptr, Type *type, bool isVolatile, unsigned int alignment)
	{
		uintptr_t t = reinterpret_cast<uintptr_t>(type);
		if(t < EmulatedTypeCount)
		{
			switch(t)
			{
			case Type_v2i32:
			case Type_v4i16:
			case Type_v8i8:
			case Type_v2f32:
				createStore(
					createExtractElement(
						createBitCast(value, T(llvm::VectorType::get(T(Long::getType()), 2))), Long::getType(), 0),
					createBitCast(ptr, Pointer<Long>::getType()),
					Long::getType(), isVolatile, alignment);
				return value;
			case Type_v2i16:
			case Type_v4i8:
				if(alignment != 0)   // Not a local variable (all vectors are 128-bit).
				{
					createStore(
						createExtractElement(createBitCast(value, Int4::getType()), Int::getType(), 0),
						createBitCast(ptr, Pointer<Int>::getType()),
						Int::getType(), isVolatile, alignment);
					return value;
				}
				break;
			default:
				assert(false);
			}
		}

		assert(V(ptr)->getType()->getContainedType(0) == T(type));
		::builder->Insert(new llvm::StoreInst(V(value), V(ptr), isVolatile, alignment));
		return value;
	}

	Value *Nucleus::createGEP(Value *ptr, Type *type, Value *index, bool unsignedIndex)
	{
		if(sizeof(void*) == 8)
		{
			if(unsignedIndex)
			{
				index = createZExt(index, Long::getType());
			}
			else
			{
				index = createSExt(index, Long::getType());
			}

			index = createMul(index, createConstantLong((int64_t)typeSize(type)));
		}
		else
		{
			index = createMul(index, createConstantInt((int)typeSize(type)));
		}

		assert(V(ptr)->getType()->getContainedType(0) == T(type));
		return createBitCast(
			V(::builder->CreateGEP(V(createBitCast(ptr, T(llvm::PointerType::get(T(Byte::getType()), 0)))), V(index))),
			T(llvm::PointerType::get(T(type), 0)));
	}

	Value *Nucleus::createAtomicAdd(Value *ptr, Value *value)
	{
		return V(::builder->CreateAtomicRMW(llvm::AtomicRMWInst::Add, V(ptr), V(value), llvm::AtomicOrdering::SequentiallyConsistent));
	}

	Value *Nucleus::createTrunc(Value *v, Type *destType)
	{
		return V(::builder->CreateTrunc(V(v), T(destType)));
	}

	Value *Nucleus::createZExt(Value *v, Type *destType)
	{
		return V(::builder->CreateZExt(V(v), T(destType)));
	}

	Value *Nucleus::createSExt(Value *v, Type *destType)
	{
		return V(::builder->CreateSExt(V(v), T(destType)));
	}

	Value *Nucleus::createFPToSI(Value *v, Type *destType)
	{
		return V(::builder->CreateFPToSI(V(v), T(destType)));
	}

	Value *Nucleus::createSIToFP(Value *v, Type *destType)
	{
		return V(::builder->CreateSIToFP(V(v), T(destType)));
	}

	Value *Nucleus::createFPTrunc(Value *v, Type *destType)
	{
		return V(::builder->CreateFPTrunc(V(v), T(destType)));
	}

	Value *Nucleus::createFPExt(Value *v, Type *destType)
	{
		return V(::builder->CreateFPExt(V(v), T(destType)));
	}

	Value *Nucleus::createBitCast(Value *v, Type *destType)
	{
		// Bitcasts must be between types of the same logical size. But with emulated narrow vectors we need
		// support for casting between scalars and wide vectors. Emulate them by writing to the stack and
		// reading back as the destination type.
		if(!V(v)->getType()->isVectorTy() && T(destType)->isVectorTy())
		{
			Value *readAddress = allocateStackVariable(destType);
			Value *writeAddress = createBitCast(readAddress, T(llvm::PointerType::get(V(v)->getType(), 0)));
			createStore(v, writeAddress, T(V(v)->getType()));
			return createLoad(readAddress, destType);
		}
		else if(V(v)->getType()->isVectorTy() && !T(destType)->isVectorTy())
		{
			Value *writeAddress = allocateStackVariable(T(V(v)->getType()));
			createStore(v, writeAddress, T(V(v)->getType()));
			Value *readAddress = createBitCast(writeAddress, T(llvm::PointerType::get(T(destType), 0)));
			return createLoad(readAddress, destType);
		}

		return V(::builder->CreateBitCast(V(v), T(destType)));
	}

	Value *Nucleus::createICmpEQ(Value *lhs, Value *rhs)
	{
		return V(::builder->CreateICmpEQ(V(lhs), V(rhs)));
	}

	Value *Nucleus::createICmpNE(Value *lhs, Value *rhs)
	{
		return V(::builder->CreateICmpNE(V(lhs), V(rhs)));
	}

	Value *Nucleus::createICmpUGT(Value *lhs, Value *rhs)
	{
		return V(::builder->CreateICmpUGT(V(lhs), V(rhs)));
	}

	Value *Nucleus::createICmpUGE(Value *lhs, Value *rhs)
	{
		return V(::builder->CreateICmpUGE(V(lhs), V(rhs)));
	}

	Value *Nucleus::createICmpULT(Value *lhs, Value *rhs)
	{
		return V(::builder->CreateICmpULT(V(lhs), V(rhs)));
	}

	Value *Nucleus::createICmpULE(Value *lhs, Value *rhs)
	{
		return V(::builder->CreateICmpULE(V(lhs), V(rhs)));
	}

	Value *Nucleus::createICmpSGT(Value *lhs, Value *rhs)
	{
		return V(::builder->CreateICmpSGT(V(lhs), V(rhs)));
	}

	Value *Nucleus::createICmpSGE(Value *lhs, Value *rhs)
	{
		return V(::builder->CreateICmpSGE(V(lhs), V(rhs)));
	}

	Value *Nucleus::createICmpSLT(Value *lhs, Value *rhs)
	{
		return V(::builder->CreateICmpSLT(V(lhs), V(rhs)));
	}

	Value *Nucleus::createICmpSLE(Value *lhs, Value *rhs)
	{
		return V(::builder->CreateICmpSLE(V(lhs), V(rhs)));
	}

	Value *Nucleus::createFCmpOEQ(Value *lhs, Value *rhs)
	{
		return V(::builder->CreateFCmpOEQ(V(lhs), V(rhs)));
	}

	Value *Nucleus::createFCmpOGT(Value *lhs, Value *rhs)
	{
		return V(::builder->CreateFCmpOGT(V(lhs), V(rhs)));
	}

	Value *Nucleus::createFCmpOGE(Value *lhs, Value *rhs)
	{
		return V(::builder->CreateFCmpOGE(V(lhs), V(rhs)));
	}

	Value *Nucleus::createFCmpOLT(Value *lhs, Value *rhs)
	{
		return V(::builder->CreateFCmpOLT(V(lhs), V(rhs)));
	}

	Value *Nucleus::createFCmpOLE(Value *lhs, Value *rhs)
	{
		return V(::builder->CreateFCmpOLE(V(lhs), V(rhs)));
	}

	Value *Nucleus::createFCmpONE(Value *lhs, Value *rhs)
	{
		return V(::builder->CreateFCmpONE(V(lhs), V(rhs)));
	}

	Value *Nucleus::createFCmpORD(Value *lhs, Value *rhs)
	{
		return V(::builder->CreateFCmpORD(V(lhs), V(rhs)));
	}

	Value *Nucleus::createFCmpUNO(Value *lhs, Value *rhs)
	{
		return V(::builder->CreateFCmpUNO(V(lhs), V(rhs)));
	}

	Value *Nucleus::createFCmpUEQ(Value *lhs, Value *rhs)
	{
		return V(::builder->CreateFCmpUEQ(V(lhs), V(rhs)));
	}

	Value *Nucleus::createFCmpUGT(Value *lhs, Value *rhs)
	{
		return V(::builder->CreateFCmpUGT(V(lhs), V(rhs)));
	}

	Value *Nucleus::createFCmpUGE(Value *lhs, Value *rhs)
	{
		return V(::builder->CreateFCmpUGE(V(lhs), V(rhs)));
	}

	Value *Nucleus::createFCmpULT(Value *lhs, Value *rhs)
	{
		return V(::builder->CreateFCmpULT(V(lhs), V(rhs)));
	}

	Value *Nucleus::createFCmpULE(Value *lhs, Value *rhs)
	{
		return V(::builder->CreateFCmpULE(V(lhs), V(rhs)));
	}

	Value *Nucleus::createFCmpUNE(Value *lhs, Value *rhs)
	{
		return V(::builder->CreateFCmpULE(V(lhs), V(rhs)));
	}

	Value *Nucleus::createExtractElement(Value *vector, Type *type, int index)
	{
		assert(V(vector)->getType()->getContainedType(0) == T(type));
		return V(::builder->CreateExtractElement(V(vector), V(createConstantInt(index))));
	}

	Value *Nucleus::createInsertElement(Value *vector, Value *element, int index)
	{
		return V(::builder->CreateInsertElement(V(vector), V(element), V(createConstantInt(index))));
	}

	Value *Nucleus::createShuffleVector(Value *v1, Value *v2, const int *select)
	{
		int size = llvm::cast<llvm::VectorType>(V(v1)->getType())->getNumElements();
		const int maxSize = 16;
		llvm::Constant *swizzle[maxSize];
		assert(size <= maxSize);

		for(int i = 0; i < size; i++)
		{
			swizzle[i] = llvm::ConstantInt::get(llvm::Type::getInt32Ty(*::context), select[i]);
		}

		llvm::Value *shuffle = llvm::ConstantVector::get(llvm::ArrayRef<llvm::Constant*>(swizzle, size));

		return V(::builder->CreateShuffleVector(V(v1), V(v2), shuffle));
	}

	Value *Nucleus::createSelect(Value *c, Value *ifTrue, Value *ifFalse)
	{
		return V(::builder->CreateSelect(V(c), V(ifTrue), V(ifFalse)));
	}

	SwitchCases *Nucleus::createSwitch(Value *control, BasicBlock *defaultBranch, unsigned numCases)
	{
		return reinterpret_cast<SwitchCases*>(::builder->CreateSwitch(V(control), B(defaultBranch), numCases));
	}

	void Nucleus::addSwitchCase(SwitchCases *switchCases, int label, BasicBlock *branch)
	{
		llvm::SwitchInst *sw = reinterpret_cast<llvm::SwitchInst *>(switchCases);
		sw->addCase(llvm::ConstantInt::get(llvm::Type::getInt32Ty(*::context), label, true), B(branch));
	}

	void Nucleus::createUnreachable()
	{
		::builder->CreateUnreachable();
	}

	static Value *createSwizzle4(Value *val, unsigned char select)
	{
		int swizzle[4] =
		{
			(select >> 0) & 0x03,
			(select >> 2) & 0x03,
			(select >> 4) & 0x03,
			(select >> 6) & 0x03,
		};

		return Nucleus::createShuffleVector(val, val, swizzle);
	}

	static Value *createMask4(Value *lhs, Value *rhs, unsigned char select)
	{
		bool mask[4] = {false, false, false, false};

		mask[(select >> 0) & 0x03] = true;
		mask[(select >> 2) & 0x03] = true;
		mask[(select >> 4) & 0x03] = true;
		mask[(select >> 6) & 0x03] = true;

		int swizzle[4] =
		{
			mask[0] ? 4 : 0,
			mask[1] ? 5 : 1,
			mask[2] ? 6 : 2,
			mask[3] ? 7 : 3,
		};

		return Nucleus::createShuffleVector(lhs, rhs, swizzle);
	}

	Type *Nucleus::getPointerType(Type *ElementType)
	{
		return T(llvm::PointerType::get(T(ElementType), 0));
	}

	Value *Nucleus::createNullValue(Type *Ty)
	{
		return V(llvm::Constant::getNullValue(T(Ty)));
	}

	Value *Nucleus::createConstantLong(int64_t i)
	{
		return V(llvm::ConstantInt::get(llvm::Type::getInt64Ty(*::context), i, true));
	}

	Value *Nucleus::createConstantInt(int i)
	{
		return V(llvm::ConstantInt::get(llvm::Type::getInt32Ty(*::context), i, true));
	}

	Value *Nucleus::createConstantInt(unsigned int i)
	{
		return V(llvm::ConstantInt::get(llvm::Type::getInt32Ty(*::context), i, false));
	}

	Value *Nucleus::createConstantBool(bool b)
	{
		return V(llvm::ConstantInt::get(llvm::Type::getInt1Ty(*::context), b));
	}

	Value *Nucleus::createConstantByte(signed char i)
	{
		return V(llvm::ConstantInt::get(llvm::Type::getInt8Ty(*::context), i, true));
	}

	Value *Nucleus::createConstantByte(unsigned char i)
	{
		return V(llvm::ConstantInt::get(llvm::Type::getInt8Ty(*::context), i, false));
	}

	Value *Nucleus::createConstantShort(short i)
	{
		return V(llvm::ConstantInt::get(llvm::Type::getInt16Ty(*::context), i, true));
	}

	Value *Nucleus::createConstantShort(unsigned short i)
	{
		return V(llvm::ConstantInt::get(llvm::Type::getInt16Ty(*::context), i, false));
	}

	Value *Nucleus::createConstantFloat(float x)
	{
		return V(llvm::ConstantFP::get(T(Float::getType()), x));
	}

	Value *Nucleus::createNullPointer(Type *Ty)
	{
		return V(llvm::ConstantPointerNull::get(llvm::PointerType::get(T(Ty), 0)));
	}

	Value *Nucleus::createConstantVector(const int64_t *constants, Type *type)
	{
		assert(llvm::isa<llvm::VectorType>(T(type)));
		const int numConstants = elementCount(type);                                       // Number of provided constants for the (emulated) type.
		const int numElements = llvm::cast<llvm::VectorType>(T(type))->getNumElements();   // Number of elements of the underlying vector type.
		assert(numElements <= 16 && numConstants <= numElements);
		llvm::Constant *constantVector[16];

		for(int i = 0; i < numElements; i++)
		{
			constantVector[i] = llvm::ConstantInt::get(T(type)->getContainedType(0), constants[i % numConstants]);
		}

		return V(llvm::ConstantVector::get(llvm::ArrayRef<llvm::Constant*>(constantVector, numElements)));
	}

	Value *Nucleus::createConstantVector(const double *constants, Type *type)
	{
		assert(llvm::isa<llvm::VectorType>(T(type)));
		const int numConstants = elementCount(type);                                       // Number of provided constants for the (emulated) type.
		const int numElements = llvm::cast<llvm::VectorType>(T(type))->getNumElements();   // Number of elements of the underlying vector type.
		assert(numElements <= 8 && numConstants <= numElements);
		llvm::Constant *constantVector[8];

		for(int i = 0; i < numElements; i++)
		{
			constantVector[i] = llvm::ConstantFP::get(T(type)->getContainedType(0), constants[i % numConstants]);
		}

		return V(llvm::ConstantVector::get(llvm::ArrayRef<llvm::Constant*>(constantVector, numElements)));
	}

	Type *Void::getType()
	{
		return T(llvm::Type::getVoidTy(*::context));
	}

	Bool::Bool(Argument<Bool> argument)
	{
		storeValue(argument.value);
	}

	Bool::Bool(bool x)
	{
		storeValue(Nucleus::createConstantBool(x));
	}

	Bool::Bool(RValue<Bool> rhs)
	{
		storeValue(rhs.value);
	}

	Bool::Bool(const Bool &rhs)
	{
		Value *value = rhs.loadValue();
		storeValue(value);
	}

	Bool::Bool(const Reference<Bool> &rhs)
	{
		Value *value = rhs.loadValue();
		storeValue(value);
	}

	RValue<Bool> Bool::operator=(RValue<Bool> rhs)
	{
		storeValue(rhs.value);

		return rhs;
	}

	RValue<Bool> Bool::operator=(const Bool &rhs)
	{
		Value *value = rhs.loadValue();
		storeValue(value);

		return RValue<Bool>(value);
	}

	RValue<Bool> Bool::operator=(const Reference<Bool> &rhs)
	{
		Value *value = rhs.loadValue();
		storeValue(value);

		return RValue<Bool>(value);
	}

	RValue<Bool> operator!(RValue<Bool> val)
	{
		return RValue<Bool>(Nucleus::createNot(val.value));
	}

	RValue<Bool> operator&&(RValue<Bool> lhs, RValue<Bool> rhs)
	{
		return RValue<Bool>(Nucleus::createAnd(lhs.value, rhs.value));
	}

	RValue<Bool> operator||(RValue<Bool> lhs, RValue<Bool> rhs)
	{
		return RValue<Bool>(Nucleus::createOr(lhs.value, rhs.value));
	}

	Type *Bool::getType()
	{
		return T(llvm::Type::getInt1Ty(*::context));
	}

	Byte::Byte(Argument<Byte> argument)
	{
		storeValue(argument.value);
	}

	Byte::Byte(RValue<Int> cast)
	{
		Value *integer = Nucleus::createTrunc(cast.value, Byte::getType());

		storeValue(integer);
	}

	Byte::Byte(RValue<UInt> cast)
	{
		Value *integer = Nucleus::createTrunc(cast.value, Byte::getType());

		storeValue(integer);
	}

	Byte::Byte(RValue<UShort> cast)
	{
		Value *integer = Nucleus::createTrunc(cast.value, Byte::getType());

		storeValue(integer);
	}

	Byte::Byte(int x)
	{
		storeValue(Nucleus::createConstantByte((unsigned char)x));
	}

	Byte::Byte(unsigned char x)
	{
		storeValue(Nucleus::createConstantByte(x));
	}

	Byte::Byte(RValue<Byte> rhs)
	{
		storeValue(rhs.value);
	}

	Byte::Byte(const Byte &rhs)
	{
		Value *value = rhs.loadValue();
		storeValue(value);
	}

	Byte::Byte(const Reference<Byte> &rhs)
	{
		Value *value = rhs.loadValue();
		storeValue(value);
	}

	RValue<Byte> Byte::operator=(RValue<Byte> rhs)
	{
		storeValue(rhs.value);

		return rhs;
	}

	RValue<Byte> Byte::operator=(const Byte &rhs)
	{
		Value *value = rhs.loadValue();
		storeValue(value);

		return RValue<Byte>(value);
	}

	RValue<Byte> Byte::operator=(const Reference<Byte> &rhs)
	{
		Value *value = rhs.loadValue();
		storeValue(value);

		return RValue<Byte>(value);
	}

	RValue<Byte> operator+(RValue<Byte> lhs, RValue<Byte> rhs)
	{
		return RValue<Byte>(Nucleus::createAdd(lhs.value, rhs.value));
	}

	RValue<Byte> operator-(RValue<Byte> lhs, RValue<Byte> rhs)
	{
		return RValue<Byte>(Nucleus::createSub(lhs.value, rhs.value));
	}

	RValue<Byte> operator*(RValue<Byte> lhs, RValue<Byte> rhs)
	{
		return RValue<Byte>(Nucleus::createMul(lhs.value, rhs.value));
	}

	RValue<Byte> operator/(RValue<Byte> lhs, RValue<Byte> rhs)
	{
		return RValue<Byte>(Nucleus::createUDiv(lhs.value, rhs.value));
	}

	RValue<Byte> operator%(RValue<Byte> lhs, RValue<Byte> rhs)
	{
		return RValue<Byte>(Nucleus::createURem(lhs.value, rhs.value));
	}

	RValue<Byte> operator&(RValue<Byte> lhs, RValue<Byte> rhs)
	{
		return RValue<Byte>(Nucleus::createAnd(lhs.value, rhs.value));
	}

	RValue<Byte> operator|(RValue<Byte> lhs, RValue<Byte> rhs)
	{
		return RValue<Byte>(Nucleus::createOr(lhs.value, rhs.value));
	}

	RValue<Byte> operator^(RValue<Byte> lhs, RValue<Byte> rhs)
	{
		return RValue<Byte>(Nucleus::createXor(lhs.value, rhs.value));
	}

	RValue<Byte> operator<<(RValue<Byte> lhs, RValue<Byte> rhs)
	{
		return RValue<Byte>(Nucleus::createShl(lhs.value, rhs.value));
	}

	RValue<Byte> operator>>(RValue<Byte> lhs, RValue<Byte> rhs)
	{
		return RValue<Byte>(Nucleus::createLShr(lhs.value, rhs.value));
	}

	RValue<Byte> operator+=(Byte &lhs, RValue<Byte> rhs)
	{
		return lhs = lhs + rhs;
	}

	RValue<Byte> operator-=(Byte &lhs, RValue<Byte> rhs)
	{
		return lhs = lhs - rhs;
	}

	RValue<Byte> operator*=(Byte &lhs, RValue<Byte> rhs)
	{
		return lhs = lhs * rhs;
	}

	RValue<Byte> operator/=(Byte &lhs, RValue<Byte> rhs)
	{
		return lhs = lhs / rhs;
	}

	RValue<Byte> operator%=(Byte &lhs, RValue<Byte> rhs)
	{
		return lhs = lhs % rhs;
	}

	RValue<Byte> operator&=(Byte &lhs, RValue<Byte> rhs)
	{
		return lhs = lhs & rhs;
	}

	RValue<Byte> operator|=(Byte &lhs, RValue<Byte> rhs)
	{
		return lhs = lhs | rhs;
	}

	RValue<Byte> operator^=(Byte &lhs, RValue<Byte> rhs)
	{
		return lhs = lhs ^ rhs;
	}

	RValue<Byte> operator<<=(Byte &lhs, RValue<Byte> rhs)
	{
		return lhs = lhs << rhs;
	}

	RValue<Byte> operator>>=(Byte &lhs, RValue<Byte> rhs)
	{
		return lhs = lhs >> rhs;
	}

	RValue<Byte> operator+(RValue<Byte> val)
	{
		return val;
	}

	RValue<Byte> operator-(RValue<Byte> val)
	{
		return RValue<Byte>(Nucleus::createNeg(val.value));
	}

	RValue<Byte> operator~(RValue<Byte> val)
	{
		return RValue<Byte>(Nucleus::createNot(val.value));
	}

	RValue<Byte> operator++(Byte &val, int)   // Post-increment
	{
		RValue<Byte> res = val;

		Value *inc = Nucleus::createAdd(res.value, Nucleus::createConstantByte((unsigned char)1));
		val.storeValue(inc);

		return res;
	}

	const Byte &operator++(Byte &val)   // Pre-increment
	{
		Value *inc = Nucleus::createAdd(val.loadValue(), Nucleus::createConstantByte((unsigned char)1));
		val.storeValue(inc);

		return val;
	}

	RValue<Byte> operator--(Byte &val, int)   // Post-decrement
	{
		RValue<Byte> res = val;

		Value *inc = Nucleus::createSub(res.value, Nucleus::createConstantByte((unsigned char)1));
		val.storeValue(inc);

		return res;
	}

	const Byte &operator--(Byte &val)   // Pre-decrement
	{
		Value *inc = Nucleus::createSub(val.loadValue(), Nucleus::createConstantByte((unsigned char)1));
		val.storeValue(inc);

		return val;
	}

	RValue<Bool> operator<(RValue<Byte> lhs, RValue<Byte> rhs)
	{
		return RValue<Bool>(Nucleus::createICmpULT(lhs.value, rhs.value));
	}

	RValue<Bool> operator<=(RValue<Byte> lhs, RValue<Byte> rhs)
	{
		return RValue<Bool>(Nucleus::createICmpULE(lhs.value, rhs.value));
	}

	RValue<Bool> operator>(RValue<Byte> lhs, RValue<Byte> rhs)
	{
		return RValue<Bool>(Nucleus::createICmpUGT(lhs.value, rhs.value));
	}

	RValue<Bool> operator>=(RValue<Byte> lhs, RValue<Byte> rhs)
	{
		return RValue<Bool>(Nucleus::createICmpUGE(lhs.value, rhs.value));
	}

	RValue<Bool> operator!=(RValue<Byte> lhs, RValue<Byte> rhs)
	{
		return RValue<Bool>(Nucleus::createICmpNE(lhs.value, rhs.value));
	}

	RValue<Bool> operator==(RValue<Byte> lhs, RValue<Byte> rhs)
	{
		return RValue<Bool>(Nucleus::createICmpEQ(lhs.value, rhs.value));
	}

	Type *Byte::getType()
	{
		return T(llvm::Type::getInt8Ty(*::context));
	}

	SByte::SByte(Argument<SByte> argument)
	{
		storeValue(argument.value);
	}

	SByte::SByte(RValue<Int> cast)
	{
		Value *integer = Nucleus::createTrunc(cast.value, SByte::getType());

		storeValue(integer);
	}

	SByte::SByte(RValue<Short> cast)
	{
		Value *integer = Nucleus::createTrunc(cast.value, SByte::getType());

		storeValue(integer);
	}

	SByte::SByte(signed char x)
	{
		storeValue(Nucleus::createConstantByte(x));
	}

	SByte::SByte(RValue<SByte> rhs)
	{
		storeValue(rhs.value);
	}

	SByte::SByte(const SByte &rhs)
	{
		Value *value = rhs.loadValue();
		storeValue(value);
	}

	SByte::SByte(const Reference<SByte> &rhs)
	{
		Value *value = rhs.loadValue();
		storeValue(value);
	}

	RValue<SByte> SByte::operator=(RValue<SByte> rhs)
	{
		storeValue(rhs.value);

		return rhs;
	}

	RValue<SByte> SByte::operator=(const SByte &rhs)
	{
		Value *value = rhs.loadValue();
		storeValue(value);

		return RValue<SByte>(value);
	}

	RValue<SByte> SByte::operator=(const Reference<SByte> &rhs)
	{
		Value *value = rhs.loadValue();
		storeValue(value);

		return RValue<SByte>(value);
	}

	RValue<SByte> operator+(RValue<SByte> lhs, RValue<SByte> rhs)
	{
		return RValue<SByte>(Nucleus::createAdd(lhs.value, rhs.value));
	}

	RValue<SByte> operator-(RValue<SByte> lhs, RValue<SByte> rhs)
	{
		return RValue<SByte>(Nucleus::createSub(lhs.value, rhs.value));
	}

	RValue<SByte> operator*(RValue<SByte> lhs, RValue<SByte> rhs)
	{
		return RValue<SByte>(Nucleus::createMul(lhs.value, rhs.value));
	}

	RValue<SByte> operator/(RValue<SByte> lhs, RValue<SByte> rhs)
	{
		return RValue<SByte>(Nucleus::createSDiv(lhs.value, rhs.value));
	}

	RValue<SByte> operator%(RValue<SByte> lhs, RValue<SByte> rhs)
	{
		return RValue<SByte>(Nucleus::createSRem(lhs.value, rhs.value));
	}

	RValue<SByte> operator&(RValue<SByte> lhs, RValue<SByte> rhs)
	{
		return RValue<SByte>(Nucleus::createAnd(lhs.value, rhs.value));
	}

	RValue<SByte> operator|(RValue<SByte> lhs, RValue<SByte> rhs)
	{
		return RValue<SByte>(Nucleus::createOr(lhs.value, rhs.value));
	}

	RValue<SByte> operator^(RValue<SByte> lhs, RValue<SByte> rhs)
	{
		return RValue<SByte>(Nucleus::createXor(lhs.value, rhs.value));
	}

	RValue<SByte> operator<<(RValue<SByte> lhs, RValue<SByte> rhs)
	{
		return RValue<SByte>(Nucleus::createShl(lhs.value, rhs.value));
	}

	RValue<SByte> operator>>(RValue<SByte> lhs, RValue<SByte> rhs)
	{
		return RValue<SByte>(Nucleus::createAShr(lhs.value, rhs.value));
	}

	RValue<SByte> operator+=(SByte &lhs, RValue<SByte> rhs)
	{
		return lhs = lhs + rhs;
	}

	RValue<SByte> operator-=(SByte &lhs, RValue<SByte> rhs)
	{
		return lhs = lhs - rhs;
	}

	RValue<SByte> operator*=(SByte &lhs, RValue<SByte> rhs)
	{
		return lhs = lhs * rhs;
	}

	RValue<SByte> operator/=(SByte &lhs, RValue<SByte> rhs)
	{
		return lhs = lhs / rhs;
	}

	RValue<SByte> operator%=(SByte &lhs, RValue<SByte> rhs)
	{
		return lhs = lhs % rhs;
	}

	RValue<SByte> operator&=(SByte &lhs, RValue<SByte> rhs)
	{
		return lhs = lhs & rhs;
	}

	RValue<SByte> operator|=(SByte &lhs, RValue<SByte> rhs)
	{
		return lhs = lhs | rhs;
	}

	RValue<SByte> operator^=(SByte &lhs, RValue<SByte> rhs)
	{
		return lhs = lhs ^ rhs;
	}

	RValue<SByte> operator<<=(SByte &lhs, RValue<SByte> rhs)
	{
		return lhs = lhs << rhs;
	}

	RValue<SByte> operator>>=(SByte &lhs, RValue<SByte> rhs)
	{
		return lhs = lhs >> rhs;
	}

	RValue<SByte> operator+(RValue<SByte> val)
	{
		return val;
	}

	RValue<SByte> operator-(RValue<SByte> val)
	{
		return RValue<SByte>(Nucleus::createNeg(val.value));
	}

	RValue<SByte> operator~(RValue<SByte> val)
	{
		return RValue<SByte>(Nucleus::createNot(val.value));
	}

	RValue<SByte> operator++(SByte &val, int)   // Post-increment
	{
		RValue<SByte> res = val;

		Value *inc = Nucleus::createAdd(res.value, Nucleus::createConstantByte((signed char)1));
		val.storeValue(inc);

		return res;
	}

	const SByte &operator++(SByte &val)   // Pre-increment
	{
		Value *inc = Nucleus::createAdd(val.loadValue(), Nucleus::createConstantByte((signed char)1));
		val.storeValue(inc);

		return val;
	}

	RValue<SByte> operator--(SByte &val, int)   // Post-decrement
	{
		RValue<SByte> res = val;

		Value *inc = Nucleus::createSub(res.value, Nucleus::createConstantByte((signed char)1));
		val.storeValue(inc);

		return res;
	}

	const SByte &operator--(SByte &val)   // Pre-decrement
	{
		Value *inc = Nucleus::createSub(val.loadValue(), Nucleus::createConstantByte((signed char)1));
		val.storeValue(inc);

		return val;
	}

	RValue<Bool> operator<(RValue<SByte> lhs, RValue<SByte> rhs)
	{
		return RValue<Bool>(Nucleus::createICmpSLT(lhs.value, rhs.value));
	}

	RValue<Bool> operator<=(RValue<SByte> lhs, RValue<SByte> rhs)
	{
		return RValue<Bool>(Nucleus::createICmpSLE(lhs.value, rhs.value));
	}

	RValue<Bool> operator>(RValue<SByte> lhs, RValue<SByte> rhs)
	{
		return RValue<Bool>(Nucleus::createICmpSGT(lhs.value, rhs.value));
	}

	RValue<Bool> operator>=(RValue<SByte> lhs, RValue<SByte> rhs)
	{
		return RValue<Bool>(Nucleus::createICmpSGE(lhs.value, rhs.value));
	}

	RValue<Bool> operator!=(RValue<SByte> lhs, RValue<SByte> rhs)
	{
		return RValue<Bool>(Nucleus::createICmpNE(lhs.value, rhs.value));
	}

	RValue<Bool> operator==(RValue<SByte> lhs, RValue<SByte> rhs)
	{
		return RValue<Bool>(Nucleus::createICmpEQ(lhs.value, rhs.value));
	}

	Type *SByte::getType()
	{
		return T(llvm::Type::getInt8Ty(*::context));
	}

	Short::Short(Argument<Short> argument)
	{
		storeValue(argument.value);
	}

	Short::Short(RValue<Int> cast)
	{
		Value *integer = Nucleus::createTrunc(cast.value, Short::getType());

		storeValue(integer);
	}

	Short::Short(short x)
	{
		storeValue(Nucleus::createConstantShort(x));
	}

	Short::Short(RValue<Short> rhs)
	{
		storeValue(rhs.value);
	}

	Short::Short(const Short &rhs)
	{
		Value *value = rhs.loadValue();
		storeValue(value);
	}

	Short::Short(const Reference<Short> &rhs)
	{
		Value *value = rhs.loadValue();
		storeValue(value);
	}

	RValue<Short> Short::operator=(RValue<Short> rhs)
	{
		storeValue(rhs.value);

		return rhs;
	}

	RValue<Short> Short::operator=(const Short &rhs)
	{
		Value *value = rhs.loadValue();
		storeValue(value);

		return RValue<Short>(value);
	}

	RValue<Short> Short::operator=(const Reference<Short> &rhs)
	{
		Value *value = rhs.loadValue();
		storeValue(value);

		return RValue<Short>(value);
	}

	RValue<Short> operator+(RValue<Short> lhs, RValue<Short> rhs)
	{
		return RValue<Short>(Nucleus::createAdd(lhs.value, rhs.value));
	}

	RValue<Short> operator-(RValue<Short> lhs, RValue<Short> rhs)
	{
		return RValue<Short>(Nucleus::createSub(lhs.value, rhs.value));
	}

	RValue<Short> operator*(RValue<Short> lhs, RValue<Short> rhs)
	{
		return RValue<Short>(Nucleus::createMul(lhs.value, rhs.value));
	}

	RValue<Short> operator/(RValue<Short> lhs, RValue<Short> rhs)
	{
		return RValue<Short>(Nucleus::createSDiv(lhs.value, rhs.value));
	}

	RValue<Short> operator%(RValue<Short> lhs, RValue<Short> rhs)
	{
		return RValue<Short>(Nucleus::createSRem(lhs.value, rhs.value));
	}

	RValue<Short> operator&(RValue<Short> lhs, RValue<Short> rhs)
	{
		return RValue<Short>(Nucleus::createAnd(lhs.value, rhs.value));
	}

	RValue<Short> operator|(RValue<Short> lhs, RValue<Short> rhs)
	{
		return RValue<Short>(Nucleus::createOr(lhs.value, rhs.value));
	}

	RValue<Short> operator^(RValue<Short> lhs, RValue<Short> rhs)
	{
		return RValue<Short>(Nucleus::createXor(lhs.value, rhs.value));
	}

	RValue<Short> operator<<(RValue<Short> lhs, RValue<Short> rhs)
	{
		return RValue<Short>(Nucleus::createShl(lhs.value, rhs.value));
	}

	RValue<Short> operator>>(RValue<Short> lhs, RValue<Short> rhs)
	{
		return RValue<Short>(Nucleus::createAShr(lhs.value, rhs.value));
	}

	RValue<Short> operator+=(Short &lhs, RValue<Short> rhs)
	{
		return lhs = lhs + rhs;
	}

	RValue<Short> operator-=(Short &lhs, RValue<Short> rhs)
	{
		return lhs = lhs - rhs;
	}

	RValue<Short> operator*=(Short &lhs, RValue<Short> rhs)
	{
		return lhs = lhs * rhs;
	}

	RValue<Short> operator/=(Short &lhs, RValue<Short> rhs)
	{
		return lhs = lhs / rhs;
	}

	RValue<Short> operator%=(Short &lhs, RValue<Short> rhs)
	{
		return lhs = lhs % rhs;
	}

	RValue<Short> operator&=(Short &lhs, RValue<Short> rhs)
	{
		return lhs = lhs & rhs;
	}

	RValue<Short> operator|=(Short &lhs, RValue<Short> rhs)
	{
		return lhs = lhs | rhs;
	}

	RValue<Short> operator^=(Short &lhs, RValue<Short> rhs)
	{
		return lhs = lhs ^ rhs;
	}

	RValue<Short> operator<<=(Short &lhs, RValue<Short> rhs)
	{
		return lhs = lhs << rhs;
	}

	RValue<Short> operator>>=(Short &lhs, RValue<Short> rhs)
	{
		return lhs = lhs >> rhs;
	}

	RValue<Short> operator+(RValue<Short> val)
	{
		return val;
	}

	RValue<Short> operator-(RValue<Short> val)
	{
		return RValue<Short>(Nucleus::createNeg(val.value));
	}

	RValue<Short> operator~(RValue<Short> val)
	{
		return RValue<Short>(Nucleus::createNot(val.value));
	}

	RValue<Short> operator++(Short &val, int)   // Post-increment
	{
		RValue<Short> res = val;

		Value *inc = Nucleus::createAdd(res.value, Nucleus::createConstantShort((short)1));
		val.storeValue(inc);

		return res;
	}

	const Short &operator++(Short &val)   // Pre-increment
	{
		Value *inc = Nucleus::createAdd(val.loadValue(), Nucleus::createConstantShort((short)1));
		val.storeValue(inc);

		return val;
	}

	RValue<Short> operator--(Short &val, int)   // Post-decrement
	{
		RValue<Short> res = val;

		Value *inc = Nucleus::createSub(res.value, Nucleus::createConstantShort((short)1));
		val.storeValue(inc);

		return res;
	}

	const Short &operator--(Short &val)   // Pre-decrement
	{
		Value *inc = Nucleus::createSub(val.loadValue(), Nucleus::createConstantShort((short)1));
		val.storeValue(inc);

		return val;
	}

	RValue<Bool> operator<(RValue<Short> lhs, RValue<Short> rhs)
	{
		return RValue<Bool>(Nucleus::createICmpSLT(lhs.value, rhs.value));
	}

	RValue<Bool> operator<=(RValue<Short> lhs, RValue<Short> rhs)
	{
		return RValue<Bool>(Nucleus::createICmpSLE(lhs.value, rhs.value));
	}

	RValue<Bool> operator>(RValue<Short> lhs, RValue<Short> rhs)
	{
		return RValue<Bool>(Nucleus::createICmpSGT(lhs.value, rhs.value));
	}

	RValue<Bool> operator>=(RValue<Short> lhs, RValue<Short> rhs)
	{
		return RValue<Bool>(Nucleus::createICmpSGE(lhs.value, rhs.value));
	}

	RValue<Bool> operator!=(RValue<Short> lhs, RValue<Short> rhs)
	{
		return RValue<Bool>(Nucleus::createICmpNE(lhs.value, rhs.value));
	}

	RValue<Bool> operator==(RValue<Short> lhs, RValue<Short> rhs)
	{
		return RValue<Bool>(Nucleus::createICmpEQ(lhs.value, rhs.value));
	}

	Type *Short::getType()
	{
		return T(llvm::Type::getInt16Ty(*::context));
	}

	UShort::UShort(Argument<UShort> argument)
	{
		storeValue(argument.value);
	}

	UShort::UShort(RValue<UInt> cast)
	{
		Value *integer = Nucleus::createTrunc(cast.value, UShort::getType());

		storeValue(integer);
	}

	UShort::UShort(RValue<Int> cast)
	{
		Value *integer = Nucleus::createTrunc(cast.value, UShort::getType());

		storeValue(integer);
	}

	UShort::UShort(unsigned short x)
	{
		storeValue(Nucleus::createConstantShort(x));
	}

	UShort::UShort(RValue<UShort> rhs)
	{
		storeValue(rhs.value);
	}

	UShort::UShort(const UShort &rhs)
	{
		Value *value = rhs.loadValue();
		storeValue(value);
	}

	UShort::UShort(const Reference<UShort> &rhs)
	{
		Value *value = rhs.loadValue();
		storeValue(value);
	}

	RValue<UShort> UShort::operator=(RValue<UShort> rhs)
	{
		storeValue(rhs.value);

		return rhs;
	}

	RValue<UShort> UShort::operator=(const UShort &rhs)
	{
		Value *value = rhs.loadValue();
		storeValue(value);

		return RValue<UShort>(value);
	}

	RValue<UShort> UShort::operator=(const Reference<UShort> &rhs)
	{
		Value *value = rhs.loadValue();
		storeValue(value);

		return RValue<UShort>(value);
	}

	RValue<UShort> operator+(RValue<UShort> lhs, RValue<UShort> rhs)
	{
		return RValue<UShort>(Nucleus::createAdd(lhs.value, rhs.value));
	}

	RValue<UShort> operator-(RValue<UShort> lhs, RValue<UShort> rhs)
	{
		return RValue<UShort>(Nucleus::createSub(lhs.value, rhs.value));
	}

	RValue<UShort> operator*(RValue<UShort> lhs, RValue<UShort> rhs)
	{
		return RValue<UShort>(Nucleus::createMul(lhs.value, rhs.value));
	}

	RValue<UShort> operator/(RValue<UShort> lhs, RValue<UShort> rhs)
	{
		return RValue<UShort>(Nucleus::createUDiv(lhs.value, rhs.value));
	}

	RValue<UShort> operator%(RValue<UShort> lhs, RValue<UShort> rhs)
	{
		return RValue<UShort>(Nucleus::createURem(lhs.value, rhs.value));
	}

	RValue<UShort> operator&(RValue<UShort> lhs, RValue<UShort> rhs)
	{
		return RValue<UShort>(Nucleus::createAnd(lhs.value, rhs.value));
	}

	RValue<UShort> operator|(RValue<UShort> lhs, RValue<UShort> rhs)
	{
		return RValue<UShort>(Nucleus::createOr(lhs.value, rhs.value));
	}

	RValue<UShort> operator^(RValue<UShort> lhs, RValue<UShort> rhs)
	{
		return RValue<UShort>(Nucleus::createXor(lhs.value, rhs.value));
	}

	RValue<UShort> operator<<(RValue<UShort> lhs, RValue<UShort> rhs)
	{
		return RValue<UShort>(Nucleus::createShl(lhs.value, rhs.value));
	}

	RValue<UShort> operator>>(RValue<UShort> lhs, RValue<UShort> rhs)
	{
		return RValue<UShort>(Nucleus::createLShr(lhs.value, rhs.value));
	}

	RValue<UShort> operator+=(UShort &lhs, RValue<UShort> rhs)
	{
		return lhs = lhs + rhs;
	}

	RValue<UShort> operator-=(UShort &lhs, RValue<UShort> rhs)
	{
		return lhs = lhs - rhs;
	}

	RValue<UShort> operator*=(UShort &lhs, RValue<UShort> rhs)
	{
		return lhs = lhs * rhs;
	}

	RValue<UShort> operator/=(UShort &lhs, RValue<UShort> rhs)
	{
		return lhs = lhs / rhs;
	}

	RValue<UShort> operator%=(UShort &lhs, RValue<UShort> rhs)
	{
		return lhs = lhs % rhs;
	}

	RValue<UShort> operator&=(UShort &lhs, RValue<UShort> rhs)
	{
		return lhs = lhs & rhs;
	}

	RValue<UShort> operator|=(UShort &lhs, RValue<UShort> rhs)
	{
		return lhs = lhs | rhs;
	}

	RValue<UShort> operator^=(UShort &lhs, RValue<UShort> rhs)
	{
		return lhs = lhs ^ rhs;
	}

	RValue<UShort> operator<<=(UShort &lhs, RValue<UShort> rhs)
	{
		return lhs = lhs << rhs;
	}

	RValue<UShort> operator>>=(UShort &lhs, RValue<UShort> rhs)
	{
		return lhs = lhs >> rhs;
	}

	RValue<UShort> operator+(RValue<UShort> val)
	{
		return val;
	}

	RValue<UShort> operator-(RValue<UShort> val)
	{
		return RValue<UShort>(Nucleus::createNeg(val.value));
	}

	RValue<UShort> operator~(RValue<UShort> val)
	{
		return RValue<UShort>(Nucleus::createNot(val.value));
	}

	RValue<UShort> operator++(UShort &val, int)   // Post-increment
	{
		RValue<UShort> res = val;

		Value *inc = Nucleus::createAdd(res.value, Nucleus::createConstantShort((unsigned short)1));
		val.storeValue(inc);

		return res;
	}

	const UShort &operator++(UShort &val)   // Pre-increment
	{
		Value *inc = Nucleus::createAdd(val.loadValue(), Nucleus::createConstantShort((unsigned short)1));
		val.storeValue(inc);

		return val;
	}

	RValue<UShort> operator--(UShort &val, int)   // Post-decrement
	{
		RValue<UShort> res = val;

		Value *inc = Nucleus::createSub(res.value, Nucleus::createConstantShort((unsigned short)1));
		val.storeValue(inc);

		return res;
	}

	const UShort &operator--(UShort &val)   // Pre-decrement
	{
		Value *inc = Nucleus::createSub(val.loadValue(), Nucleus::createConstantShort((unsigned short)1));
		val.storeValue(inc);

		return val;
	}

	RValue<Bool> operator<(RValue<UShort> lhs, RValue<UShort> rhs)
	{
		return RValue<Bool>(Nucleus::createICmpULT(lhs.value, rhs.value));
	}

	RValue<Bool> operator<=(RValue<UShort> lhs, RValue<UShort> rhs)
	{
		return RValue<Bool>(Nucleus::createICmpULE(lhs.value, rhs.value));
	}

	RValue<Bool> operator>(RValue<UShort> lhs, RValue<UShort> rhs)
	{
		return RValue<Bool>(Nucleus::createICmpUGT(lhs.value, rhs.value));
	}

	RValue<Bool> operator>=(RValue<UShort> lhs, RValue<UShort> rhs)
	{
		return RValue<Bool>(Nucleus::createICmpUGE(lhs.value, rhs.value));
	}

	RValue<Bool> operator!=(RValue<UShort> lhs, RValue<UShort> rhs)
	{
		return RValue<Bool>(Nucleus::createICmpNE(lhs.value, rhs.value));
	}

	RValue<Bool> operator==(RValue<UShort> lhs, RValue<UShort> rhs)
	{
		return RValue<Bool>(Nucleus::createICmpEQ(lhs.value, rhs.value));
	}

	Type *UShort::getType()
	{
		return T(llvm::Type::getInt16Ty(*::context));
	}

	Byte4::Byte4(RValue<Byte8> cast)