Google Git Google Git
Sign in
swiftshader/SwiftShader/104f72dbb79fc2faf28fa544c048cc65c52286c8/./third_party/llvm-16.0/llvm/lib/Target/X86/X86SpeculativeLoadHardening.cpp
blob: 2a896314875d29376c8400e37e7e44c80a4156fc [file]
//====- X86SpeculativeLoadHardening.cpp - A Spectre v1 mitigation ---------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
/// \file
///
/// Provide a pass which mitigates speculative execution attacks which operate
/// by speculating incorrectly past some predicate (a type check, bounds check,
/// or other condition) to reach a load with invalid inputs and leak the data
/// accessed by that load using a side channel out of the speculative domain.
///
/// For details on the attacks, see the first variant in both the Project Zero
/// writeup and the Spectre paper:
/// https://googleprojectzero.blogspot.com/2018/01/reading-privileged-memory-with-side.html
/// https://spectreattack.com/spectre.pdf
///
//===----------------------------------------------------------------------===//
#include "X86.h"
#include "X86InstrBuilder.h"
#include "X86InstrInfo.h"
#include "X86Subtarget.h"
#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/ScopeExit.h"
#include "llvm/ADT/SmallPtrSet.h"
#include "llvm/ADT/SmallSet.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/SparseBitVector.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/CodeGen/MachineBasicBlock.h"
#include "llvm/CodeGen/MachineConstantPool.h"
#include "llvm/CodeGen/MachineFunction.h"
#include "llvm/CodeGen/MachineFunctionPass.h"
#include "llvm/CodeGen/MachineInstr.h"
#include "llvm/CodeGen/MachineInstrBuilder.h"
#include "llvm/CodeGen/MachineModuleInfo.h"
#include "llvm/CodeGen/MachineOperand.h"
#include "llvm/CodeGen/MachineRegisterInfo.h"
#include "llvm/CodeGen/MachineSSAUpdater.h"
#include "llvm/CodeGen/TargetInstrInfo.h"
#include "llvm/CodeGen/TargetRegisterInfo.h"
#include "llvm/CodeGen/TargetSchedule.h"
#include "llvm/CodeGen/TargetSubtargetInfo.h"
#include "llvm/IR/DebugLoc.h"
#include "llvm/MC/MCSchedule.h"
#include "llvm/Pass.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/raw_ostream.h"
#include "llvm/Target/TargetMachine.h"
#include <algorithm>
#include <cassert>
#include <iterator>
#include <optional>
#include <utility>
using namespace llvm;
#define PASS_KEY "x86-slh"
#define DEBUG_TYPE PASS_KEY
STATISTIC(NumCondBranchesTraced, "Number of conditional branches traced");
STATISTIC(NumBranchesUntraced, "Number of branches unable to trace");
STATISTIC(NumAddrRegsHardened,
"Number of address mode used registers hardaned");
STATISTIC(NumPostLoadRegsHardened,
"Number of post-load register values hardened");
STATISTIC(NumCallsOrJumpsHardened,
"Number of calls or jumps requiring extra hardening");
STATISTIC(NumInstsInserted, "Number of instructions inserted");
STATISTIC(NumLFENCEsInserted, "Number of lfence instructions inserted");
static cl::opt<bool> EnableSpeculativeLoadHardening(
"x86-speculative-load-hardening",
cl::desc("Force enable speculative load hardening"), cl::init(false),
cl::Hidden);
static cl::opt<bool> HardenEdgesWithLFENCE(
PASS_KEY "-lfence",
cl::desc(
"Use LFENCE along each conditional edge to harden against speculative "
"loads rather than conditional movs and poisoned pointers."),
cl::init(false), cl::Hidden);
static cl::opt<bool> EnablePostLoadHardening(
PASS_KEY "-post-load",
cl::desc("Harden the value loaded *after* it is loaded by "
"flushing the loaded bits to 1. This is hard to do "
"in general but can be done easily for GPRs."),
cl::init(true), cl::Hidden);
static cl::opt<bool> FenceCallAndRet(
PASS_KEY "-fence-call-and-ret",
cl::desc("Use a full speculation fence to harden both call and ret edges "
"rather than a lighter weight mitigation."),
cl::init(false), cl::Hidden);
static cl::opt<bool> HardenInterprocedurally(
PASS_KEY "-ip",
cl::desc("Harden interprocedurally by passing our state in and out of "
"functions in the high bits of the stack pointer."),
cl::init(true), cl::Hidden);
static cl::opt<bool>
HardenLoads(PASS_KEY "-loads",
cl::desc("Sanitize loads from memory. When disable, no "
"significant security is provided."),
cl::init(true), cl::Hidden);
static cl::opt<bool> HardenIndirectCallsAndJumps(
PASS_KEY "-indirect",
cl::desc("Harden indirect calls and jumps against using speculatively "
"stored attacker controlled addresses. This is designed to "
"mitigate Spectre v1.2 style attacks."),
cl::init(true), cl::Hidden);
namespace {
class X86SpeculativeLoadHardeningPass : public MachineFunctionPass {
public:
X86SpeculativeLoadHardeningPass() : MachineFunctionPass(ID) { }
StringRef getPassName() const override {
return "X86 speculative load hardening";
}
bool runOnMachineFunction(MachineFunction &MF) override;
void getAnalysisUsage(AnalysisUsage &AU) const override;
/// Pass identification, replacement for typeid.
static char ID;
private:
/// The information about a block's conditional terminators needed to trace
/// our predicate state through the exiting edges.
struct BlockCondInfo {
MachineBasicBlock *MBB;
// We mostly have one conditional branch, and in extremely rare cases have
// two. Three and more are so rare as to be unimportant for compile time.
SmallVector<MachineInstr *, 2> CondBrs;
MachineInstr *UncondBr;
};
/// Manages the predicate state traced through the program.
struct PredState {
unsigned InitialReg = 0;
unsigned PoisonReg = 0;
const TargetRegisterClass *RC;
MachineSSAUpdater SSA;
PredState(MachineFunction &MF, const TargetRegisterClass *RC)
: RC(RC), SSA(MF) {}
};
const X86Subtarget *Subtarget = nullptr;
MachineRegisterInfo *MRI = nullptr;
const X86InstrInfo *TII = nullptr;
const TargetRegisterInfo *TRI = nullptr;
std::optional<PredState> PS;
void hardenEdgesWithLFENCE(MachineFunction &MF);
SmallVector<BlockCondInfo, 16> collectBlockCondInfo(MachineFunction &MF);
SmallVector<MachineInstr *, 16>
tracePredStateThroughCFG(MachineFunction &MF, ArrayRef<BlockCondInfo> Infos);
void unfoldCallAndJumpLoads(MachineFunction &MF);
SmallVector<MachineInstr *, 16>
tracePredStateThroughIndirectBranches(MachineFunction &MF);
void tracePredStateThroughBlocksAndHarden(MachineFunction &MF);
unsigned saveEFLAGS(MachineBasicBlock &MBB,
MachineBasicBlock::iterator InsertPt,
const DebugLoc &Loc);
void restoreEFLAGS(MachineBasicBlock &MBB,
MachineBasicBlock::iterator InsertPt, const DebugLoc &Loc,
Register Reg);
void mergePredStateIntoSP(MachineBasicBlock &MBB,
MachineBasicBlock::iterator InsertPt,
const DebugLoc &Loc, unsigned PredStateReg);
unsigned extractPredStateFromSP(MachineBasicBlock &MBB,
MachineBasicBlock::iterator InsertPt,
const DebugLoc &Loc);
void
hardenLoadAddr(MachineInstr &MI, MachineOperand &BaseMO,
MachineOperand &IndexMO,
SmallDenseMap<unsigned, unsigned, 32> &AddrRegToHardenedReg);
MachineInstr *
sinkPostLoadHardenedInst(MachineInstr &MI,
SmallPtrSetImpl<MachineInstr *> &HardenedInstrs);
bool canHardenRegister(Register Reg);
unsigned hardenValueInRegister(Register Reg, MachineBasicBlock &MBB,
MachineBasicBlock::iterator InsertPt,
const DebugLoc &Loc);
unsigned hardenPostLoad(MachineInstr &MI);
void hardenReturnInstr(MachineInstr &MI);
void tracePredStateThroughCall(MachineInstr &MI);
void hardenIndirectCallOrJumpInstr(
MachineInstr &MI,
SmallDenseMap<unsigned, unsigned, 32> &AddrRegToHardenedReg);
};
} // end anonymous namespace
char X86SpeculativeLoadHardeningPass::ID = 0;
void X86SpeculativeLoadHardeningPass::getAnalysisUsage(
AnalysisUsage &AU) const {
MachineFunctionPass::getAnalysisUsage(AU);
}
static MachineBasicBlock &splitEdge(MachineBasicBlock &MBB,
MachineBasicBlock &Succ, int SuccCount,
MachineInstr *Br, MachineInstr *&UncondBr,
const X86InstrInfo &TII) {
assert(!Succ.isEHPad() && "Shouldn't get edges to EH pads!");
MachineFunction &MF = *MBB.getParent();
MachineBasicBlock &NewMBB = *MF.CreateMachineBasicBlock();
// We have to insert the new block immediately after the current one as we
// don't know what layout-successor relationships the successor has and we
// may not be able to (and generally don't want to) try to fix those up.
MF.insert(std::next(MachineFunction::iterator(&MBB)), &NewMBB);
// Update the branch instruction if necessary.
if (Br) {
assert(Br->getOperand(0).getMBB() == &Succ &&
"Didn't start with the right target!");
Br->getOperand(0).setMBB(&NewMBB);
// If this successor was reached through a branch rather than fallthrough,
// we might have *broken* fallthrough and so need to inject a new
// unconditional branch.
if (!UncondBr) {
MachineBasicBlock &OldLayoutSucc =
*std::next(MachineFunction::iterator(&NewMBB));
assert(MBB.isSuccessor(&OldLayoutSucc) &&
"Without an unconditional branch, the old layout successor should "
"be an actual successor!");
auto BrBuilder =
BuildMI(&MBB, DebugLoc(), TII.get(X86::JMP_1)).addMBB(&OldLayoutSucc);
// Update the unconditional branch now that we've added one.
UncondBr = &*BrBuilder;
}
// Insert unconditional "jump Succ" instruction in the new block if
// necessary.
if (!NewMBB.isLayoutSuccessor(&Succ)) {
SmallVector<MachineOperand, 4> Cond;
TII.insertBranch(NewMBB, &Succ, nullptr, Cond, Br->getDebugLoc());
}
} else {
assert(!UncondBr &&
"Cannot have a branchless successor and an unconditional branch!");
assert(NewMBB.isLayoutSuccessor(&Succ) &&
"A non-branch successor must have been a layout successor before "
"and now is a layout successor of the new block.");
}
// If this is the only edge to the successor, we can just replace it in the
// CFG. Otherwise we need to add a new entry in the CFG for the new
// successor.
if (SuccCount == 1) {
MBB.replaceSuccessor(&Succ, &NewMBB);
} else {
MBB.splitSuccessor(&Succ, &NewMBB);
}
// Hook up the edge from the new basic block to the old successor in the CFG.
NewMBB.addSuccessor(&Succ);
// Fix PHI nodes in Succ so they refer to NewMBB instead of MBB.
for (MachineInstr &MI : Succ) {
if (!MI.isPHI())
break;
for (int OpIdx = 1, NumOps = MI.getNumOperands(); OpIdx < NumOps;
OpIdx += 2) {
MachineOperand &OpV = MI.getOperand(OpIdx);
MachineOperand &OpMBB = MI.getOperand(OpIdx + 1);
assert(OpMBB.isMBB() && "Block operand to a PHI is not a block!");
if (OpMBB.getMBB() != &MBB)
continue;
// If this is the last edge to the succesor, just replace MBB in the PHI
if (SuccCount == 1) {
OpMBB.setMBB(&NewMBB);
break;
}
// Otherwise, append a new pair of operands for the new incoming edge.
MI.addOperand(MF, OpV);
MI.addOperand(MF, MachineOperand::CreateMBB(&NewMBB));
break;
}
}
// Inherit live-ins from the successor
for (auto &LI : Succ.liveins())
NewMBB.addLiveIn(LI);
LLVM_DEBUG(dbgs() << " Split edge from '" << MBB.getName() << "' to '"
<< Succ.getName() << "'.\n");
return NewMBB;
}
/// Removing duplicate PHI operands to leave the PHI in a canonical and
/// predictable form.
///
/// FIXME: It's really frustrating that we have to do this, but SSA-form in MIR
/// isn't what you might expect. We may have multiple entries in PHI nodes for
/// a single predecessor. This makes CFG-updating extremely complex, so here we
/// simplify all PHI nodes to a model even simpler than the IR's model: exactly
/// one entry per predecessor, regardless of how many edges there are.
static void canonicalizePHIOperands(MachineFunction &MF) {
SmallPtrSet<MachineBasicBlock *, 4> Preds;
SmallVector<int, 4> DupIndices;
for (auto &MBB : MF)
for (auto &MI : MBB) {
if (!MI.isPHI())
break;
// First we scan the operands of the PHI looking for duplicate entries
// a particular predecessor. We retain the operand index of each duplicate
// entry found.
for (int OpIdx = 1, NumOps = MI.getNumOperands(); OpIdx < NumOps;
OpIdx += 2)
if (!Preds.insert(MI.getOperand(OpIdx + 1).getMBB()).second)
DupIndices.push_back(OpIdx);
// Now walk the duplicate indices, removing both the block and value. Note
// that these are stored as a vector making this element-wise removal
// :w
// potentially quadratic.
//
// FIXME: It is really frustrating that we have to use a quadratic
// removal algorithm here. There should be a better way, but the use-def
// updates required make that impossible using the public API.
//
// Note that we have to process these backwards so that we don't
// invalidate other indices with each removal.
while (!DupIndices.empty()) {
int OpIdx = DupIndices.pop_back_val();
// Remove both the block and value operand, again in reverse order to
// preserve indices.
MI.removeOperand(OpIdx + 1);
MI.removeOperand(OpIdx);
}
Preds.clear();
}
}
/// Helper to scan a function for loads vulnerable to misspeculation that we
/// want to harden.
///
/// We use this to avoid making changes to functions where there is nothing we
/// need to do to harden against misspeculation.
static bool hasVulnerableLoad(MachineFunction &MF) {
for (MachineBasicBlock &MBB : MF) {
for (MachineInstr &MI : MBB) {
// Loads within this basic block after an LFENCE are not at risk of
// speculatively executing with invalid predicates from prior control
// flow. So break out of this block but continue scanning the function.
if (MI.getOpcode() == X86::LFENCE)
break;
// Looking for loads only.
if (!MI.mayLoad())
continue;
// An MFENCE is modeled as a load but isn't vulnerable to misspeculation.
if (MI.getOpcode() == X86::MFENCE)
continue;
// We found a load.
return true;
}
}
// No loads found.
return false;
}
bool X86SpeculativeLoadHardeningPass::runOnMachineFunction(
MachineFunction &MF) {
LLVM_DEBUG(dbgs() << "********** " << getPassName() << " : " << MF.getName()
<< " **********\n");
// Only run if this pass is forced enabled or we detect the relevant function
// attribute requesting SLH.
if (!EnableSpeculativeLoadHardening &&
!MF.getFunction().hasFnAttribute(Attribute::SpeculativeLoadHardening))
return false;
Subtarget = &MF.getSubtarget<X86Subtarget>();
MRI = &MF.getRegInfo();
TII = Subtarget->getInstrInfo();
TRI = Subtarget->getRegisterInfo();
// FIXME: Support for 32-bit.
PS.emplace(MF, &X86::GR64_NOSPRegClass);
if (MF.begin() == MF.end())
// Nothing to do for a degenerate empty function...
return false;
// We support an alternative hardening technique based on a debug flag.
if (HardenEdgesWithLFENCE) {
hardenEdgesWithLFENCE(MF);
return true;
}
// Create a dummy debug loc to use for all the generated code here.
DebugLoc Loc;
MachineBasicBlock &Entry = *MF.begin();
auto EntryInsertPt = Entry.SkipPHIsLabelsAndDebug(Entry.begin());
// Do a quick scan to see if we have any checkable loads.
bool HasVulnerableLoad = hasVulnerableLoad(MF);
// See if we have any conditional branching blocks that we will need to trace
// predicate state through.
SmallVector<BlockCondInfo, 16> Infos = collectBlockCondInfo(MF);
// If we have no interesting conditions or loads, nothing to do here.
if (!HasVulnerableLoad && Infos.empty())
return true;
// The poison value is required to be an all-ones value for many aspects of
// this mitigation.
const int PoisonVal = -1;
PS->PoisonReg = MRI->createVirtualRegister(PS->RC);
BuildMI(Entry, EntryInsertPt, Loc, TII->get(X86::MOV64ri32), PS->PoisonReg)
.addImm(PoisonVal);
++NumInstsInserted;
// If we have loads being hardened and we've asked for call and ret edges to
// get a full fence-based mitigation, inject that fence.
if (HasVulnerableLoad && FenceCallAndRet) {
// We need to insert an LFENCE at the start of the function to suspend any
// incoming misspeculation from the caller. This helps two-fold: the caller
// may not have been protected as this code has been, and this code gets to
// not take any specific action to protect across calls.
// FIXME: We could skip this for functions which unconditionally return
// a constant.
BuildMI(Entry, EntryInsertPt, Loc, TII->get(X86::LFENCE));
++NumInstsInserted;
++NumLFENCEsInserted;
}
// If we guarded the entry with an LFENCE and have no conditionals to protect
// in blocks, then we're done.
if (FenceCallAndRet && Infos.empty())
// We may have changed the function's code at this point to insert fences.
return true;
// For every basic block in the function which can b
if (HardenInterprocedurally && !FenceCallAndRet) {
// Set up the predicate state by extracting it from the incoming stack
// pointer so we pick up any misspeculation in our caller.
PS->InitialReg = extractPredStateFromSP(Entry, EntryInsertPt, Loc);
} else {
// Otherwise, just build the predicate state itself by zeroing a register
// as we don't need any initial state.
PS->InitialReg = MRI->createVirtualRegister(PS->RC);
Register PredStateSubReg = MRI->createVirtualRegister(&X86::GR32RegClass);
auto ZeroI = BuildMI(Entry, EntryInsertPt, Loc, TII->get(X86::MOV32r0),
PredStateSubReg);
++NumInstsInserted;
MachineOperand *ZeroEFLAGSDefOp =
ZeroI->findRegisterDefOperand(X86::EFLAGS);
assert(ZeroEFLAGSDefOp && ZeroEFLAGSDefOp->isImplicit() &&
"Must have an implicit def of EFLAGS!");
ZeroEFLAGSDefOp->setIsDead(true);
BuildMI(Entry, EntryInsertPt, Loc, TII->get(X86::SUBREG_TO_REG),
PS->InitialReg)
.addImm(0)
.addReg(PredStateSubReg)
.addImm(X86::sub_32bit);
}
// We're going to need to trace predicate state throughout the function's
// CFG. Prepare for this by setting up our initial state of PHIs with unique
// predecessor entries and all the initial predicate state.
canonicalizePHIOperands(MF);
// Track the updated values in an SSA updater to rewrite into SSA form at the
// end.
PS->SSA.Initialize(PS->InitialReg);
PS->SSA.AddAvailableValue(&Entry, PS->InitialReg);
// Trace through the CFG.
auto CMovs = tracePredStateThroughCFG(MF, Infos);
// We may also enter basic blocks in this function via exception handling
// control flow. Here, if we are hardening interprocedurally, we need to
// re-capture the predicate state from the throwing code. In the Itanium ABI,
// the throw will always look like a call to __cxa_throw and will have the
// predicate state in the stack pointer, so extract fresh predicate state from
// the stack pointer and make it available in SSA.
// FIXME: Handle non-itanium ABI EH models.
if (HardenInterprocedurally) {
for (MachineBasicBlock &MBB : MF) {
assert(!MBB.isEHScopeEntry() && "Only Itanium ABI EH supported!");
assert(!MBB.isEHFuncletEntry() && "Only Itanium ABI EH supported!");
assert(!MBB.isCleanupFuncletEntry() && "Only Itanium ABI EH supported!");
if (!MBB.isEHPad())
continue;
PS->SSA.AddAvailableValue(
&MBB,
extractPredStateFromSP(MBB, MBB.SkipPHIsAndLabels(MBB.begin()), Loc));
}
}
if (HardenIndirectCallsAndJumps) {
// If we are going to harden calls and jumps we need to unfold their memory
// operands.
unfoldCallAndJumpLoads(MF);
// Then we trace predicate state through the indirect branches.
auto IndirectBrCMovs = tracePredStateThroughIndirectBranches(MF);
CMovs.append(IndirectBrCMovs.begin(), IndirectBrCMovs.end());
}
// Now that we have the predicate state available at the start of each block
// in the CFG, trace it through each block, hardening vulnerable instructions
// as we go.
tracePredStateThroughBlocksAndHarden(MF);
// Now rewrite all the uses of the pred state using the SSA updater to insert
// PHIs connecting the state between blocks along the CFG edges.
for (MachineInstr *CMovI : CMovs)
for (MachineOperand &Op : CMovI->operands()) {
if (!Op.isReg() || Op.getReg() != PS->InitialReg)
continue;
PS->SSA.RewriteUse(Op);
}
LLVM_DEBUG(dbgs() << "Final speculative load hardened function:\n"; MF.dump();
dbgs() << "\n"; MF.verify(this));
return true;
}
/// Implements the naive hardening approach of putting an LFENCE after every
/// potentially mis-predicted control flow construct.
///
/// We include this as an alternative mostly for the purpose of comparison. The
/// performance impact of this is expected to be extremely severe and not
/// practical for any real-world users.
void X86SpeculativeLoadHardeningPass::hardenEdgesWithLFENCE(
MachineFunction &MF) {
// First, we scan the function looking for blocks that are reached along edges
// that we might want to harden.
SmallSetVector<MachineBasicBlock *, 8> Blocks;
for (MachineBasicBlock &MBB : MF) {
// If there are no or only one successor, nothing to do here.
if (MBB.succ_size() <= 1)
continue;
// Skip blocks unless their terminators start with a branch. Other
// terminators don't seem interesting for guarding against misspeculation.
auto TermIt = MBB.getFirstTerminator();
if (TermIt == MBB.end() || !TermIt->isBranch())
continue;
// Add all the non-EH-pad succossors to the blocks we want to harden. We
// skip EH pads because there isn't really a condition of interest on
// entering.
for (MachineBasicBlock *SuccMBB : MBB.successors())
if (!SuccMBB->isEHPad())
Blocks.insert(SuccMBB);
}
for (MachineBasicBlock *MBB : Blocks) {
auto InsertPt = MBB->SkipPHIsAndLabels(MBB->begin());
BuildMI(*MBB, InsertPt, DebugLoc(), TII->get(X86::LFENCE));
++NumInstsInserted;
++NumLFENCEsInserted;
}
}
SmallVector<X86SpeculativeLoadHardeningPass::BlockCondInfo, 16>
X86SpeculativeLoadHardeningPass::collectBlockCondInfo(MachineFunction &MF) {
SmallVector<BlockCondInfo, 16> Infos;
// Walk the function and build up a summary for each block's conditions that
// we need to trace through.
for (MachineBasicBlock &MBB : MF) {
// If there are no or only one successor, nothing to do here.
if (MBB.succ_size() <= 1)
continue;
// We want to reliably handle any conditional branch terminators in the
// MBB, so we manually analyze the branch. We can handle all of the
// permutations here, including ones that analyze branch cannot.
//
// The approach is to walk backwards across the terminators, resetting at
// any unconditional non-indirect branch, and track all conditional edges
// to basic blocks as well as the fallthrough or unconditional successor
// edge. For each conditional edge, we track the target and the opposite
// condition code in order to inject a "no-op" cmov into that successor
// that will harden the predicate. For the fallthrough/unconditional
// edge, we inject a separate cmov for each conditional branch with
// matching condition codes. This effectively implements an "and" of the
// condition flags, even if there isn't a single condition flag that would
// directly implement that. We don't bother trying to optimize either of
// these cases because if such an optimization is possible, LLVM should
// have optimized the conditional *branches* in that way already to reduce
// instruction count. This late, we simply assume the minimal number of
// branch instructions is being emitted and use that to guide our cmov
// insertion.
BlockCondInfo Info = {&MBB, {}, nullptr};
// Now walk backwards through the terminators and build up successors they
// reach and the conditions.
for (MachineInstr &MI : llvm::reverse(MBB)) {
// Once we've handled all the terminators, we're done.
if (!MI.isTerminator())
break;
// If we see a non-branch terminator, we can't handle anything so bail.
if (!MI.isBranch()) {
Info.CondBrs.clear();
break;
}
// If we see an unconditional branch, reset our state, clear any
// fallthrough, and set this is the "else" successor.
if (MI.getOpcode() == X86::JMP_1) {
Info.CondBrs.clear();
Info.UncondBr = &MI;
continue;
}
// If we get an invalid condition, we have an indirect branch or some
// other unanalyzable "fallthrough" case. We model this as a nullptr for
// the destination so we can still guard any conditional successors.
// Consider code sequences like:
// ```
// jCC L1
// jmpq *%rax
// ```
// We still want to harden the edge to `L1`.
if (X86::getCondFromBranch(MI) == X86::COND_INVALID) {
Info.CondBrs.clear();
Info.UncondBr = &MI;
continue;
}
// We have a vanilla conditional branch, add it to our list.
Info.CondBrs.push_back(&MI);
}
if (Info.CondBrs.empty()) {
++NumBranchesUntraced;
LLVM_DEBUG(dbgs() << "WARNING: unable to secure successors of block:\n";
MBB.dump());
continue;
}
Infos.push_back(Info);
}
return Infos;
}
/// Trace the predicate state through the CFG, instrumenting each conditional
/// branch such that misspeculation through an edge will poison the predicate
/// state.
///
/// Returns the list of inserted CMov instructions so that they can have their
/// uses of the predicate state rewritten into proper SSA form once it is
/// complete.
SmallVector<MachineInstr *, 16>
X86SpeculativeLoadHardeningPass::tracePredStateThroughCFG(
MachineFunction &MF, ArrayRef<BlockCondInfo> Infos) {
// Collect the inserted cmov instructions so we can rewrite their uses of the
// predicate state into SSA form.
SmallVector<MachineInstr *, 16> CMovs;
// Now walk all of the basic blocks looking for ones that end in conditional
// jumps where we need to update this register along each edge.
for (const BlockCondInfo &Info : Infos) {
MachineBasicBlock &MBB = *Info.MBB;
const SmallVectorImpl<MachineInstr *> &CondBrs = Info.CondBrs;
MachineInstr *UncondBr = Info.UncondBr;
LLVM_DEBUG(dbgs() << "Tracing predicate through block: " << MBB.getName()
<< "\n");
++NumCondBranchesTraced;
// Compute the non-conditional successor as either the target of any
// unconditional branch or the layout successor.
MachineBasicBlock *UncondSucc =
UncondBr ? (UncondBr->getOpcode() == X86::JMP_1
? UncondBr->getOperand(0).getMBB()
: nullptr)
: &*std::next(MachineFunction::iterator(&MBB));
// Count how many edges there are to any given successor.
SmallDenseMap<MachineBasicBlock *, int> SuccCounts;
if (UncondSucc)
++SuccCounts[UncondSucc];
for (auto *CondBr : CondBrs)
++SuccCounts[CondBr->getOperand(0).getMBB()];
// A lambda to insert cmov instructions into a block checking all of the
// condition codes in a sequence.
auto BuildCheckingBlockForSuccAndConds =
[&](MachineBasicBlock &MBB, MachineBasicBlock &Succ, int SuccCount,
MachineInstr *Br, MachineInstr *&UncondBr,
ArrayRef<X86::CondCode> Conds) {
// First, we split the edge to insert the checking block into a safe
// location.
auto &CheckingMBB =
(SuccCount == 1 && Succ.pred_size() == 1)
? Succ
: splitEdge(MBB, Succ, SuccCount, Br, UncondBr, *TII);
bool LiveEFLAGS = Succ.isLiveIn(X86::EFLAGS);
if (!LiveEFLAGS)
CheckingMBB.addLiveIn(X86::EFLAGS);
// Now insert the cmovs to implement the checks.
auto InsertPt = CheckingMBB.begin();
assert((InsertPt == CheckingMBB.end() || !InsertPt->isPHI()) &&
"Should never have a PHI in the initial checking block as it "
"always has a single predecessor!");
// We will wire each cmov to each other, but need to start with the
// incoming pred state.
unsigned CurStateReg = PS->InitialReg;
for (X86::CondCode Cond : Conds) {
int PredStateSizeInBytes = TRI->getRegSizeInBits(*PS->RC) / 8;
auto CMovOp = X86::getCMovOpcode(PredStateSizeInBytes);
Register UpdatedStateReg = MRI->createVirtualRegister(PS->RC);
// Note that we intentionally use an empty debug location so that
// this picks up the preceding location.
auto CMovI = BuildMI(CheckingMBB, InsertPt, DebugLoc(),
TII->get(CMovOp), UpdatedStateReg)
.addReg(CurStateReg)
.addReg(PS->PoisonReg)
.addImm(Cond);
// If this is the last cmov and the EFLAGS weren't originally
// live-in, mark them as killed.
if (!LiveEFLAGS && Cond == Conds.back())
CMovI->findRegisterUseOperand(X86::EFLAGS)->setIsKill(true);
++NumInstsInserted;
LLVM_DEBUG(dbgs() << " Inserting cmov: "; CMovI->dump();
dbgs() << "\n");
// The first one of the cmovs will be using the top level
// `PredStateReg` and need to get rewritten into SSA form.
if (CurStateReg == PS->InitialReg)
CMovs.push_back(&*CMovI);
// The next cmov should start from this one's def.
CurStateReg = UpdatedStateReg;
}
// And put the last one into the available values for SSA form of our
// predicate state.
PS->SSA.AddAvailableValue(&CheckingMBB, CurStateReg);
};
std::vector<X86::CondCode> UncondCodeSeq;
for (auto *CondBr : CondBrs) {
MachineBasicBlock &Succ = *CondBr->getOperand(0).getMBB();
int &SuccCount = SuccCounts[&Succ];
X86::CondCode Cond = X86::getCondFromBranch(*CondBr);
X86::CondCode InvCond = X86::GetOppositeBranchCondition(Cond);
UncondCodeSeq.push_back(Cond);
BuildCheckingBlockForSuccAndConds(MBB, Succ, SuccCount, CondBr, UncondBr,
{InvCond});
// Decrement the successor count now that we've split one of the edges.
// We need to keep the count of edges to the successor accurate in order
// to know above when to *replace* the successor in the CFG vs. just
// adding the new successor.
--SuccCount;
}
// Since we may have split edges and changed the number of successors,
// normalize the probabilities. This avoids doing it each time we split an
// edge.
MBB.normalizeSuccProbs();
// Finally, we need to insert cmovs into the "fallthrough" edge. Here, we
// need to intersect the other condition codes. We can do this by just
// doing a cmov for each one.
if (!UncondSucc)
// If we have no fallthrough to protect (perhaps it is an indirect jump?)
// just skip this and continue.
continue;
assert(SuccCounts[UncondSucc] == 1 &&
"We should never have more than one edge to the unconditional "
"successor at this point because every other edge must have been "
"split above!");
// Sort and unique the codes to minimize them.
llvm::sort(UncondCodeSeq);
UncondCodeSeq.erase(std::unique(UncondCodeSeq.begin(), UncondCodeSeq.end()),
UncondCodeSeq.end());
// Build a checking version of the successor.
BuildCheckingBlockForSuccAndConds(MBB, *UncondSucc, /*SuccCount*/ 1,
UncondBr, UncondBr, UncondCodeSeq);
}
return CMovs;
}
/// Compute the register class for the unfolded load.
///
/// FIXME: This should probably live in X86InstrInfo, potentially by adding
/// a way to unfold into a newly created vreg rather than requiring a register
/// input.
static const TargetRegisterClass *
getRegClassForUnfoldedLoad(MachineFunction &MF, const X86InstrInfo &TII,
unsigned Opcode) {
unsigned Index;
unsigned UnfoldedOpc = TII.getOpcodeAfterMemoryUnfold(
Opcode, /*UnfoldLoad*/ true, /*UnfoldStore*/ false, &Index);
const MCInstrDesc &MCID = TII.get(UnfoldedOpc);
return TII.getRegClass(MCID, Index, &TII.getRegisterInfo(), MF);
}
void X86SpeculativeLoadHardeningPass::unfoldCallAndJumpLoads(
MachineFunction &MF) {
for (MachineBasicBlock &MBB : MF)
// We use make_early_inc_range here so we can remove instructions if needed
// without disturbing the iteration.
for (MachineInstr &MI : llvm::make_early_inc_range(MBB.instrs())) {
// Must either be a call or a branch.
if (!MI.isCall() && !MI.isBranch())
continue;
// We only care about loading variants of these instructions.
if (!MI.mayLoad())
continue;
switch (MI.getOpcode()) {
default: {
LLVM_DEBUG(
dbgs() << "ERROR: Found an unexpected loading branch or call "
"instruction:\n";
MI.dump(); dbgs() << "\n");
report_fatal_error("Unexpected loading branch or call!");
}
case X86::FARCALL16m:
case X86::FARCALL32m:
case X86::FARCALL64m:
case X86::FARJMP16m:
case X86::FARJMP32m:
case X86::FARJMP64m:
// We cannot mitigate far jumps or calls, but we also don't expect them
// to be vulnerable to Spectre v1.2 style attacks.
continue;
case X86::CALL16m:
case X86::CALL16m_NT:
case X86::CALL32m:
case X86::CALL32m_NT:
case X86::CALL64m:
case X86::CALL64m_NT:
case X86::JMP16m:
case X86::JMP16m_NT:
case X86::JMP32m:
case X86::JMP32m_NT:
case X86::JMP64m:
case X86::JMP64m_NT:
case X86::TAILJMPm64:
case X86::TAILJMPm64_REX:
case X86::TAILJMPm:
case X86::TCRETURNmi64:
case X86::TCRETURNmi: {
// Use the generic unfold logic now that we know we're dealing with
// expected instructions.
// FIXME: We don't have test coverage for all of these!
auto *UnfoldedRC = getRegClassForUnfoldedLoad(MF, *TII, MI.getOpcode());
if (!UnfoldedRC) {
LLVM_DEBUG(dbgs()
<< "ERROR: Unable to unfold load from instruction:\n";
MI.dump(); dbgs() << "\n");
report_fatal_error("Unable to unfold load!");
}
Register Reg = MRI->createVirtualRegister(UnfoldedRC);
SmallVector<MachineInstr *, 2> NewMIs;
// If we were able to compute an unfolded reg class, any failure here
// is just a programming error so just assert.
bool Unfolded =
TII->unfoldMemoryOperand(MF, MI, Reg, /*UnfoldLoad*/ true,
/*UnfoldStore*/ false, NewMIs);
(void)Unfolded;
assert(Unfolded &&
"Computed unfolded register class but failed to unfold");
// Now stitch the new instructions into place and erase the old one.
for (auto *NewMI : NewMIs)
MBB.insert(MI.getIterator(), NewMI);
// Update the call site info.
if (MI.isCandidateForCallSiteEntry())
MF.eraseCallSiteInfo(&MI);
MI.eraseFromParent();
LLVM_DEBUG({
dbgs() << "Unfolded load successfully into:\n";
for (auto *NewMI : NewMIs) {
NewMI->dump();
dbgs() << "\n";
}
});
continue;
}
}
llvm_unreachable("Escaped switch with default!");
}
}
/// Trace the predicate state through indirect branches, instrumenting them to
/// poison the state if a target is reached that does not match the expected
/// target.
///
/// This is designed to mitigate Spectre variant 1 attacks where an indirect
/// branch is trained to predict a particular target and then mispredicts that
/// target in a way that can leak data. Despite using an indirect branch, this
/// is really a variant 1 style attack: it does not steer execution to an
/// arbitrary or attacker controlled address, and it does not require any
/// special code executing next to the victim. This attack can also be mitigated
/// through retpolines, but those require either replacing indirect branches
/// with conditional direct branches or lowering them through a device that
/// blocks speculation. This mitigation can replace these retpoline-style
/// mitigations for jump tables and other indirect branches within a function
/// when variant 2 isn't a risk while allowing limited speculation. Indirect
/// calls, however, cannot be mitigated through this technique without changing
/// the ABI in a fundamental way.
SmallVector<MachineInstr *, 16>
X86SpeculativeLoadHardeningPass::tracePredStateThroughIndirectBranches(
MachineFunction &MF) {
// We use the SSAUpdater to insert PHI nodes for the target addresses of
// indirect branches. We don't actually need the full power of the SSA updater
// in this particular case as we always have immediately available values, but
// this avoids us having to re-implement the PHI construction logic.
MachineSSAUpdater TargetAddrSSA(MF);
TargetAddrSSA.Initialize(MRI->createVirtualRegister(&X86::GR64RegClass));
// Track which blocks were terminated with an indirect branch.
SmallPtrSet<MachineBasicBlock *, 4> IndirectTerminatedMBBs;
// We need to know what blocks end up reached via indirect branches. We
// expect this to be a subset of those whose address is taken and so track it
// directly via the CFG.
SmallPtrSet<MachineBasicBlock *, 4> IndirectTargetMBBs;
// Walk all the blocks which end in an indirect branch and make the
// target address available.
for (MachineBasicBlock &MBB : MF) {
// Find the last terminator.
auto MII = MBB.instr_rbegin();
while (MII != MBB.instr_rend() && MII->isDebugInstr())
++MII;
if (MII == MBB.instr_rend())
continue;
MachineInstr &TI = *MII;
if (!TI.isTerminator() || !TI.isBranch())
// No terminator or non-branch terminator.
continue;
unsigned TargetReg;
switch (TI.getOpcode()) {
default:
// Direct branch or conditional branch (leading to fallthrough).
continue;
case X86::FARJMP16m:
case X86::FARJMP32m:
case X86::FARJMP64m:
// We cannot mitigate far jumps or calls, but we also don't expect them
// to be vulnerable to Spectre v1.2 or v2 (self trained) style attacks.
continue;
case X86::JMP16m:
case X86::JMP16m_NT:
case X86::JMP32m:
case X86::JMP32m_NT:
case X86::JMP64m:
case X86::JMP64m_NT:
// Mostly as documentation.
report_fatal_error("Memory operand jumps should have been unfolded!");
case X86::JMP16r:
report_fatal_error(
"Support for 16-bit indirect branches is not implemented.");
case X86::JMP32r:
report_fatal_error(
"Support for 32-bit indirect branches is not implemented.");
case X86::JMP64r:
TargetReg = TI.getOperand(0).getReg();
}
// We have definitely found an indirect branch. Verify that there are no
// preceding conditional branches as we don't yet support that.
if (llvm::any_of(MBB.terminators(), [&](MachineInstr &OtherTI) {
return !OtherTI.isDebugInstr() && &OtherTI != &TI;
})) {
LLVM_DEBUG({
dbgs() << "ERROR: Found other terminators in a block with an indirect "
"branch! This is not yet supported! Terminator sequence:\n";
for (MachineInstr &MI : MBB.terminators()) {
MI.dump();
dbgs() << '\n';
}
});
report_fatal_error("Unimplemented terminator sequence!");
}
// Make the target register an available value for this block.
TargetAddrSSA.AddAvailableValue(&MBB, TargetReg);
IndirectTerminatedMBBs.insert(&MBB);
// Add all the successors to our target candidates.
for (MachineBasicBlock *Succ : MBB.successors())
IndirectTargetMBBs.insert(Succ);
}
// Keep track of the cmov instructions we insert so we can return them.
SmallVector<MachineInstr *, 16> CMovs;
// If we didn't find any indirect branches with targets, nothing to do here.
if (IndirectTargetMBBs.empty())
return CMovs;
// We found indirect branches and targets that need to be instrumented to
// harden loads within them. Walk the blocks of the function (to get a stable
// ordering) and instrument each target of an indirect branch.
for (MachineBasicBlock &MBB : MF) {
// Skip the blocks that aren't candidate targets.
if (!IndirectTargetMBBs.count(&MBB))
continue;
// We don't expect EH pads to ever be reached via an indirect branch. If
// this is desired for some reason, we could simply skip them here rather
// than asserting.
assert(!MBB.isEHPad() &&
"Unexpected EH pad as target of an indirect branch!");
// We should never end up threading EFLAGS into a block to harden
// conditional jumps as there would be an additional successor via the
// indirect branch. As a consequence, all such edges would be split before
// reaching here, and the inserted block will handle the EFLAGS-based
// hardening.
assert(!MBB.isLiveIn(X86::EFLAGS) &&
"Cannot check within a block that already has live-in EFLAGS!");
// We can't handle having non-indirect edges into this block unless this is
// the only successor and we can synthesize the necessary target address.
for (MachineBasicBlock *Pred : MBB.predecessors()) {
// If we've already handled this by extracting the target directly,
// nothing to do.
if (IndirectTerminatedMBBs.count(Pred))
continue;
// Otherwise, we have to be the only successor. We generally expect this
// to be true as conditional branches should have had a critical edge
// split already. We don't however need to worry about EH pad successors
// as they'll happily ignore the target and their hardening strategy is
// resilient to all ways in which they could be reached speculatively.
if (!llvm::all_of(Pred->successors(), [&](MachineBasicBlock *Succ) {
return Succ->isEHPad() || Succ == &MBB;
})) {
LLVM_DEBUG({
dbgs() << "ERROR: Found conditional entry to target of indirect "
"branch!\n";
Pred->dump();
MBB.dump();
});
report_fatal_error("Cannot harden a conditional entry to a target of "
"an indirect branch!");
}
// Now we need to compute the address of this block and install it as a
// synthetic target in the predecessor. We do this at the bottom of the
// predecessor.
auto InsertPt = Pred->getFirstTerminator();
Register TargetReg = MRI->createVirtualRegister(&X86::GR64RegClass);
if (MF.getTarget().getCodeModel() == CodeModel::Small &&
!Subtarget->isPositionIndependent()) {
// Directly materialize it into an immediate.
auto AddrI = BuildMI(*Pred, InsertPt, DebugLoc(),
TII->get(X86::MOV64ri32), TargetReg)
.addMBB(&MBB);
++NumInstsInserted;
(void)AddrI;
LLVM_DEBUG(dbgs() << " Inserting mov: "; AddrI->dump();
dbgs() << "\n");
} else {
auto AddrI = BuildMI(*Pred, InsertPt, DebugLoc(), TII->get(X86::LEA64r),
TargetReg)
.addReg(/*Base*/ X86::RIP)
.addImm(/*Scale*/ 1)
.addReg(/*Index*/ 0)
.addMBB(&MBB)
.addReg(/*Segment*/ 0);
++NumInstsInserted;
(void)AddrI;
LLVM_DEBUG(dbgs() << " Inserting lea: "; AddrI->dump();
dbgs() << "\n");
}
// And make this available.
TargetAddrSSA.AddAvailableValue(Pred, TargetReg);
}
// Materialize the needed SSA value of the target. Note that we need the
// middle of the block as this block might at the bottom have an indirect
// branch back to itself. We can do this here because at this point, every
// predecessor of this block has an available value. This is basically just
// automating the construction of a PHI node for this target.
Register TargetReg = TargetAddrSSA.GetValueInMiddleOfBlock(&MBB);
// Insert a comparison of the incoming target register with this block's
// address. This also requires us to mark the block as having its address
// taken explicitly.
MBB.setMachineBlockAddressTaken();
auto InsertPt = MBB.SkipPHIsLabelsAndDebug(MBB.begin());
if (MF.getTarget().getCodeModel() == CodeModel::Small &&
!Subtarget->isPositionIndependent()) {
// Check directly against a relocated immediate when we can.
auto CheckI = BuildMI(MBB, InsertPt, DebugLoc(), TII->get(X86::CMP64ri32))
.addReg(TargetReg, RegState::Kill)
.addMBB(&MBB);
++NumInstsInserted;
(void)CheckI;
LLVM_DEBUG(dbgs() << " Inserting cmp: "; CheckI->dump(); dbgs() << "\n");
} else {
// Otherwise compute the address into a register first.
Register AddrReg = MRI->createVirtualRegister(&X86::GR64RegClass);
auto AddrI =
BuildMI(MBB, InsertPt, DebugLoc(), TII->get(X86::LEA64r), AddrReg)
.addReg(/*Base*/ X86::RIP)
.addImm(/*Scale*/ 1)
.addReg(/*Index*/ 0)
.addMBB(&MBB)
.addReg(/*Segment*/ 0);
++NumInstsInserted;
(void)AddrI;
LLVM_DEBUG(dbgs() << " Inserting lea: "; AddrI->dump(); dbgs() << "\n");
auto CheckI = BuildMI(MBB, InsertPt, DebugLoc(), TII->get(X86::CMP64rr))
.addReg(TargetReg, RegState::Kill)
.addReg(AddrReg, RegState::Kill);
++NumInstsInserted;
(void)CheckI;
LLVM_DEBUG(dbgs() << " Inserting cmp: "; CheckI->dump(); dbgs() << "\n");
}
// Now cmov over the predicate if the comparison wasn't equal.
int PredStateSizeInBytes = TRI->getRegSizeInBits(*PS->RC) / 8;
auto CMovOp = X86::getCMovOpcode(PredStateSizeInBytes);
Register UpdatedStateReg = MRI->createVirtualRegister(PS->RC);
auto CMovI =
BuildMI(MBB, InsertPt, DebugLoc(), TII->get(CMovOp), UpdatedStateReg)
.addReg(PS->InitialReg)
.addReg(PS->PoisonReg)
.addImm(X86::COND_NE);
CMovI->findRegisterUseOperand(X86::EFLAGS)->setIsKill(true);
++NumInstsInserted;
LLVM_DEBUG(dbgs() << " Inserting cmov: "; CMovI->dump(); dbgs() << "\n");
CMovs.push_back(&*CMovI);
// And put the new value into the available values for SSA form of our
// predicate state.
PS->SSA.AddAvailableValue(&MBB, UpdatedStateReg);
}
// Return all the newly inserted cmov instructions of the predicate state.
return CMovs;
}
// Returns true if the MI has EFLAGS as a register def operand and it's live,
// otherwise it returns false
static bool isEFLAGSDefLive(const MachineInstr &MI) {
if (const MachineOperand *DefOp = MI.findRegisterDefOperand(X86::EFLAGS)) {
return !DefOp->isDead();
}
return false;
}
static bool isEFLAGSLive(MachineBasicBlock &MBB, MachineBasicBlock::iterator I,
const TargetRegisterInfo &TRI) {
// Check if EFLAGS are alive by seeing if there is a def of them or they
// live-in, and then seeing if that def is in turn used.
for (MachineInstr &MI : llvm::reverse(llvm::make_range(MBB.begin(), I))) {
if (MachineOperand *DefOp = MI.findRegisterDefOperand(X86::EFLAGS)) {
// If the def is dead, then EFLAGS is not live.
if (DefOp->isDead())
return false;
// Otherwise we've def'ed it, and it is live.
return true;
}
// While at this instruction, also check if we use and kill EFLAGS
// which means it isn't live.
if (MI.killsRegister(X86::EFLAGS, &TRI))
return false;
}
// If we didn't find anything conclusive (neither definitely alive or
// definitely dead) return whether it lives into the block.
return MBB.isLiveIn(X86::EFLAGS);
}
/// Trace the predicate state through each of the blocks in the function,
/// hardening everything necessary along the way.
///
/// We call this routine once the initial predicate state has been established
/// for each basic block in the function in the SSA updater. This routine traces
/// it through the instructions within each basic block, and for non-returning
/// blocks informs the SSA updater about the final state that lives out of the
/// block. Along the way, it hardens any vulnerable instruction using the
/// currently valid predicate state. We have to do these two things together
/// because the SSA updater only works across blocks. Within a block, we track
/// the current predicate state directly and update it as it changes.
///
/// This operates in two passes over each block. First, we analyze the loads in
/// the block to determine which strategy will be used to harden them: hardening
/// the address or hardening the loaded value when loaded into a register
/// amenable to hardening. We have to process these first because the two
/// strategies may interact -- later hardening may change what strategy we wish
/// to use. We also will analyze data dependencies between loads and avoid
/// hardening those loads that are data dependent on a load with a hardened
/// address. We also skip hardening loads already behind an LFENCE as that is
/// sufficient to harden them against misspeculation.
///
/// Second, we actively trace the predicate state through the block, applying
/// the hardening steps we determined necessary in the first pass as we go.
///
/// These two passes are applied to each basic block. We operate one block at a
/// time to simplify reasoning about reachability and sequencing.
void X86SpeculativeLoadHardeningPass::tracePredStateThroughBlocksAndHarden(
MachineFunction &MF) {
SmallPtrSet<MachineInstr *, 16> HardenPostLoad;
SmallPtrSet<MachineInstr *, 16> HardenLoadAddr;
SmallSet<unsigned, 16> HardenedAddrRegs;
SmallDenseMap<unsigned, unsigned, 32> AddrRegToHardenedReg;
// Track the set of load-dependent registers through the basic block. Because
// the values of these registers have an existing data dependency on a loaded
// value which we would have checked, we can omit any checks on them.
SparseBitVector<> LoadDepRegs;
for (MachineBasicBlock &MBB : MF) {
// The first pass over the block: collect all the loads which can have their
// loaded value hardened and all the loads that instead need their address
// hardened. During this walk we propagate load dependence for address
// hardened loads and also look for LFENCE to stop hardening wherever
// possible. When deciding whether or not to harden the loaded value or not,
// we check to see if any registers used in the address will have been
// hardened at this point and if so, harden any remaining address registers
// as that often successfully re-uses hardened addresses and minimizes
// instructions.
//
// FIXME: We should consider an aggressive mode where we continue to keep as
// many loads value hardened even when some address register hardening would
// be free (due to reuse).
//
// Note that we only need this pass if we are actually hardening loads.
if (HardenLoads)
for (MachineInstr &MI : MBB) {
// We naively assume that all def'ed registers of an instruction have
// a data dependency on all of their operands.
// FIXME: Do a more careful analysis of x86 to build a conservative
// model here.
if (llvm::any_of(MI.uses(), [&](MachineOperand &Op) {
return Op.isReg() && LoadDepRegs.test(Op.getReg());
}))
for (MachineOperand &Def : MI.defs())
if (Def.isReg())
LoadDepRegs.set(Def.getReg());
// Both Intel and AMD are guiding that they will change the semantics of
// LFENCE to be a speculation barrier, so if we see an LFENCE, there is
// no more need to guard things in this block.
if (MI.getOpcode() == X86::LFENCE)
break;
// If this instruction cannot load, nothing to do.
if (!MI.mayLoad())
continue;
// Some instructions which "load" are trivially safe or unimportant.
if (MI.getOpcode() == X86::MFENCE)
continue;
// Extract the memory operand information about this instruction.
// FIXME: This doesn't handle loading pseudo instructions which we often
// could handle with similarly generic logic. We probably need to add an
// MI-layer routine similar to the MC-layer one we use here which maps
// pseudos much like this maps real instructions.
const MCInstrDesc &Desc = MI.getDesc();
int MemRefBeginIdx = X86II::getMemoryOperandNo(Desc.TSFlags);
if (MemRefBeginIdx < 0) {
LLVM_DEBUG(dbgs()
<< "WARNING: unable to harden loading instruction: ";
MI.dump());
continue;
}
MemRefBeginIdx += X86II::getOperandBias(Desc);
MachineOperand &BaseMO =
MI.getOperand(MemRefBeginIdx + X86::AddrBaseReg);
MachineOperand &IndexMO =
MI.getOperand(MemRefBeginIdx + X86::AddrIndexReg);
// If we have at least one (non-frame-index, non-RIP) register operand,
// and neither operand is load-dependent, we need to check the load.
unsigned BaseReg = 0, IndexReg = 0;
if (!BaseMO.isFI() && BaseMO.getReg() != X86::RIP &&
BaseMO.getReg() != X86::NoRegister)
BaseReg = BaseMO.getReg();
if (IndexMO.getReg() != X86::NoRegister)
IndexReg = IndexMO.getReg();
if (!BaseReg && !IndexReg)
// No register operands!
continue;
// If any register operand is dependent, this load is dependent and we
// needn't check it.
// FIXME: Is this true in the case where we are hardening loads after
// they complete? Unclear, need to investigate.
if ((BaseReg && LoadDepRegs.test(BaseReg)) ||
(IndexReg && LoadDepRegs.test(IndexReg)))
continue;
// If post-load hardening is enabled, this load is compatible with
// post-load hardening, and we aren't already going to harden one of the
// address registers, queue it up to be hardened post-load. Notably,
// even once hardened this won't introduce a useful dependency that
// could prune out subsequent loads.
if (EnablePostLoadHardening && X86InstrInfo::isDataInvariantLoad(MI) &&
!isEFLAGSDefLive(MI) && MI.getDesc().getNumDefs() == 1 &&
MI.getOperand(0).isReg() &&
canHardenRegister(MI.getOperand(0).getReg()) &&
!HardenedAddrRegs.count(BaseReg) &&
!HardenedAddrRegs.count(IndexReg)) {
HardenPostLoad.insert(&MI);
HardenedAddrRegs.insert(MI.getOperand(0).getReg());
continue;
}
// Record this instruction for address hardening and record its register
// operands as being address-hardened.
HardenLoadAddr.insert(&MI);
if (BaseReg)
HardenedAddrRegs.insert(BaseReg);
if (IndexReg)
HardenedAddrRegs.insert(IndexReg);
for (MachineOperand &Def : MI.defs())
if (Def.isReg())
LoadDepRegs.set(Def.getReg());
}
// Now re-walk the instructions in the basic block, and apply whichever
// hardening strategy we have elected. Note that we do this in a second
// pass specifically so that we have the complete set of instructions for
// which we will do post-load hardening and can defer it in certain
// circumstances.
for (MachineInstr &MI : MBB) {
if (HardenLoads) {
// We cannot both require hardening the def of a load and its address.
assert(!(HardenLoadAddr.count(&MI) && HardenPostLoad.count(&MI)) &&
"Requested to harden both the address and def of a load!");
// Check if this is a load whose address needs to be hardened.
if (HardenLoadAddr.erase(&MI)) {
const MCInstrDesc &Desc = MI.getDesc();
int MemRefBeginIdx = X86II::getMemoryOperandNo(Desc.TSFlags);
assert(MemRefBeginIdx >= 0 && "Cannot have an invalid index here!");
MemRefBeginIdx += X86II::getOperandBias(Desc);
MachineOperand &BaseMO =
MI.getOperand(MemRefBeginIdx + X86::AddrBaseReg);
MachineOperand &IndexMO =
MI.getOperand(MemRefBeginIdx + X86::AddrIndexReg);
hardenLoadAddr(MI, BaseMO, IndexMO, AddrRegToHardenedReg);
continue;
}
// Test if this instruction is one of our post load instructions (and
// remove it from the set if so).
if (HardenPostLoad.erase(&MI)) {
assert(!MI.isCall() && "Must not try to post-load harden a call!");
// If this is a data-invariant load and there is no EFLAGS
// interference, we want to try and sink any hardening as far as
// possible.
if (X86InstrInfo::isDataInvariantLoad(MI) && !isEFLAGSDefLive(MI)) {
// Sink the instruction we'll need to harden as far as we can down
// the graph.
MachineInstr *SunkMI = sinkPostLoadHardenedInst(MI, HardenPostLoad);
// If we managed to sink this instruction, update everything so we
// harden that instruction when we reach it in the instruction
// sequence.
if (SunkMI != &MI) {
// If in sinking there was no instruction needing to be hardened,
// we're done.
if (!SunkMI)
continue;
// Otherwise, add this to the set of defs we harden.
HardenPostLoad.insert(SunkMI);
continue;
}
}
unsigned HardenedReg = hardenPostLoad(MI);
// Mark the resulting hardened register as such so we don't re-harden.
AddrRegToHardenedReg[HardenedReg] = HardenedReg;
continue;
}
// Check for an indirect call or branch that may need its input hardened
// even if we couldn't find the specific load used, or were able to
// avoid hardening it for some reason. Note that here we cannot break
// out afterward as we may still need to handle any call aspect of this
// instruction.
if ((MI.isCall() || MI.isBranch()) && HardenIndirectCallsAndJumps)
hardenIndirectCallOrJumpInstr(MI, AddrRegToHardenedReg);
}
// After we finish hardening loads we handle interprocedural hardening if
// enabled and relevant for this instruction.
if (!HardenInterprocedurally)
continue;
if (!MI.isCall() && !MI.isReturn())
continue;
// If this is a direct return (IE, not a tail call) just directly harden
// it.
if (MI.isReturn() && !MI.isCall()) {
hardenReturnInstr(MI);
continue;
}
// Otherwise we have a call. We need to handle transferring the predicate
// state into a call and recovering it after the call returns (unless this
// is a tail call).
assert(MI.isCall() && "Should only reach here for calls!");
tracePredStateThroughCall(MI);
}
HardenPostLoad.clear();
HardenLoadAddr.clear();
HardenedAddrRegs.clear();
AddrRegToHardenedReg.clear();
// Currently, we only track data-dependent loads within a basic block.
// FIXME: We should see if this is necessary or if we could be more
// aggressive here without opening up attack avenues.
LoadDepRegs.clear();
}
}
/// Save EFLAGS into the returned GPR. This can in turn be restored with
/// `restoreEFLAGS`.
///
/// Note that LLVM can only lower very simple patterns of saved and restored
/// EFLAGS registers. The restore should always be within the same basic block
/// as the save so that no PHI nodes are inserted.
unsigned X86SpeculativeLoadHardeningPass::saveEFLAGS(
MachineBasicBlock &MBB, MachineBasicBlock::iterator InsertPt,
const DebugLoc &Loc) {
// FIXME: Hard coding this to a 32-bit register class seems weird, but matches
// what instruction selection does.
Register Reg = MRI->createVirtualRegister(&X86::GR32RegClass);
// We directly copy the FLAGS register and rely on later lowering to clean
// this up into the appropriate setCC instructions.
BuildMI(MBB, InsertPt, Loc, TII->get(X86::COPY), Reg).addReg(X86::EFLAGS);
++NumInstsInserted;
return Reg;
}
/// Restore EFLAGS from the provided GPR. This should be produced by
/// `saveEFLAGS`.
///
/// This must be done within the same basic block as the save in order to
/// reliably lower.
void X86SpeculativeLoadHardeningPass::restoreEFLAGS(
MachineBasicBlock &MBB, MachineBasicBlock::iterator InsertPt,
const DebugLoc &Loc, Register Reg) {
BuildMI(MBB, InsertPt, Loc, TII->get(X86::COPY), X86::EFLAGS).addReg(Reg);
++NumInstsInserted;
}
/// Takes the current predicate state (in a register) and merges it into the
/// stack pointer. The state is essentially a single bit, but we merge this in
/// a way that won't form non-canonical pointers and also will be preserved
/// across normal stack adjustments.
void X86SpeculativeLoadHardeningPass::mergePredStateIntoSP(
MachineBasicBlock &MBB, MachineBasicBlock::iterator InsertPt,
const DebugLoc &Loc, unsigned PredStateReg) {
Register TmpReg = MRI->createVirtualRegister(PS->RC);
// FIXME: This hard codes a shift distance based on the number of bits needed
// to stay canonical on 64-bit. We should compute this somehow and support
// 32-bit as part of that.
auto ShiftI = BuildMI(MBB, InsertPt, Loc, TII->get(X86::SHL64ri), TmpReg)
.addReg(PredStateReg, RegState::Kill)
.addImm(47);
ShiftI->addRegisterDead(X86::EFLAGS, TRI);
++NumInstsInserted;
auto OrI = BuildMI(MBB, InsertPt, Loc, TII->get(X86::OR64rr), X86::RSP)
.addReg(X86::RSP)
.addReg(TmpReg, RegState::Kill);
OrI->addRegisterDead(X86::EFLAGS, TRI);
++NumInstsInserted;
}
/// Extracts the predicate state stored in the high bits of the stack pointer.
unsigned X86SpeculativeLoadHardeningPass::extractPredStateFromSP(
MachineBasicBlock &MBB, MachineBasicBlock::iterator InsertPt,
const DebugLoc &Loc) {
Register PredStateReg = MRI->createVirtualRegister(PS->RC);
Register TmpReg = MRI->createVirtualRegister(PS->RC);
// We know that the stack pointer will have any preserved predicate state in
// its high bit. We just want to smear this across the other bits. Turns out,
// this is exactly what an arithmetic right shift does.
BuildMI(MBB, InsertPt, Loc, TII->get(TargetOpcode::COPY), TmpReg)
.addReg(X86::RSP);
auto ShiftI =
BuildMI(MBB, InsertPt, Loc, TII->get(X86::SAR64ri), PredStateReg)
.addReg(TmpReg, RegState::Kill)
.addImm(TRI->getRegSizeInBits(*PS->RC) - 1);
ShiftI->addRegisterDead(X86::EFLAGS, TRI);
++NumInstsInserted;
return PredStateReg;
}
void X86SpeculativeLoadHardeningPass::hardenLoadAddr(
MachineInstr &MI, MachineOperand &BaseMO, MachineOperand &IndexMO,
SmallDenseMap<unsigned, unsigned, 32> &AddrRegToHardenedReg) {
MachineBasicBlock &MBB = *MI.getParent();
const DebugLoc &Loc = MI.getDebugLoc();
// Check if EFLAGS are alive by seeing if there is a def of them or they
// live-in, and then seeing if that def is in turn used.
bool EFLAGSLive = isEFLAGSLive(MBB, MI.getIterator(), *TRI);
SmallVector<MachineOperand *, 2> HardenOpRegs;
if (BaseMO.isFI()) {
// A frame index is never a dynamically controllable load, so only
// harden it if we're covering fixed address loads as well.
LLVM_DEBUG(
dbgs() << " Skipping hardening base of explicit stack frame load: ";
MI.dump(); dbgs() << "\n");
} else if (BaseMO.getReg() == X86::RSP) {
// Some idempotent atomic operations are lowered directly to a locked
// OR with 0 to the top of stack(or slightly offset from top) which uses an
// explicit RSP register as the base.
assert(IndexMO.getReg() == X86::NoRegister &&
"Explicit RSP access with dynamic index!");
LLVM_DEBUG(
dbgs() << " Cannot harden base of explicit RSP offset in a load!");
} else if (BaseMO.getReg() == X86::RIP ||
BaseMO.getReg() == X86::NoRegister) {
// For both RIP-relative addressed loads or absolute loads, we cannot
// meaningfully harden them because the address being loaded has no
// dynamic component.
//
// FIXME: When using a segment base (like TLS does) we end up with the
// dynamic address being the base plus -1 because we can't mutate the
// segment register here. This allows the signed 32-bit offset to point at
// valid segment-relative addresses and load them successfully.
LLVM_DEBUG(
dbgs() << " Cannot harden base of "
<< (BaseMO.getReg() == X86::RIP ? "RIP-relative" : "no-base")
<< " address in a load!");
} else {
assert(BaseMO.isReg() &&
"Only allowed to have a frame index or register base.");
HardenOpRegs.push_back(&BaseMO);
}
if (IndexMO.getReg() != X86::NoRegister &&
(HardenOpRegs.empty() ||
HardenOpRegs.front()->getReg() != IndexMO.getReg()))
HardenOpRegs.push_back(&IndexMO);
assert((HardenOpRegs.size() == 1 || HardenOpRegs.size() == 2) &&
"Should have exactly one or two registers to harden!");
assert((HardenOpRegs.size() == 1 ||
HardenOpRegs[0]->getReg() != HardenOpRegs[1]->getReg()) &&
"Should not have two of the same registers!");
// Remove any registers that have alreaded been checked.
llvm::erase_if(HardenOpRegs, [&](MachineOperand *Op) {
// See if this operand's register has already been checked.
auto It = AddrRegToHardenedReg.find(Op->getReg());
if (It == AddrRegToHardenedReg.end())
// Not checked, so retain this one.
return false;
// Otherwise, we can directly update this operand and remove it.
Op->setReg(It->second);
return true;
});
// If there are none left, we're done.
if (HardenOpRegs.empty())
return;
// Compute the current predicate state.
Register StateReg = PS->SSA.GetValueAtEndOfBlock(&MBB);
auto InsertPt = MI.getIterator();
// If EFLAGS are live and we don't have access to instructions that avoid
// clobbering EFLAGS we need to save and restore them. This in turn makes
// the EFLAGS no longer live.
unsigned FlagsReg = 0;
if (EFLAGSLive && !Subtarget->hasBMI2()) {
EFLAGSLive = false;
FlagsReg = saveEFLAGS(MBB, InsertPt, Loc);
}
for (MachineOperand *Op : HardenOpRegs) {
Register OpReg = Op->getReg();
auto *OpRC = MRI->getRegClass(OpReg);
Register TmpReg = MRI->createVirtualRegister(OpRC);
// If this is a vector register, we'll need somewhat custom logic to handle
// hardening it.
if (!Subtarget->hasVLX() && (OpRC->hasSuperClassEq(&X86::VR128RegClass) ||
OpRC->hasSuperClassEq(&X86::VR256RegClass))) {
assert(Subtarget->hasAVX2() && "AVX2-specific register classes!");
bool Is128Bit = OpRC->hasSuperClassEq(&X86::VR128RegClass);
// Move our state into a vector register.
// FIXME: We could skip this at the cost of longer encodings with AVX-512
// but that doesn't seem likely worth it.
Register VStateReg = MRI->createVirtualRegister(&X86::VR128RegClass);
auto MovI =
BuildMI(MBB, InsertPt, Loc, TII->get(X86::VMOV64toPQIrr), VStateReg)
.addReg(StateReg);
(void)MovI;
++NumInstsInserted;
LLVM_DEBUG(dbgs() << " Inserting mov: "; MovI->dump(); dbgs() << "\n");
// Broadcast it across the vector register.
Register VBStateReg = MRI->createVirtualRegister(OpRC);
auto BroadcastI = BuildMI(MBB, InsertPt, Loc,
TII->get(Is128Bit ? X86::VPBROADCASTQrr
: X86::VPBROADCASTQYrr),
VBStateReg)
.addReg(VStateReg);
(void)BroadcastI;
++NumInstsInserted;
LLVM_DEBUG(dbgs() << " Inserting broadcast: "; BroadcastI->dump();
dbgs() << "\n");
// Merge our potential poison state into the value with a vector or.
auto OrI =
BuildMI(MBB, InsertPt, Loc,
TII->get(Is128Bit ? X86::VPORrr : X86::VPORYrr), TmpReg)
.addReg(VBStateReg)
.addReg(OpReg);
(void)OrI;
++NumInstsInserted;
LLVM_DEBUG(dbgs() << " Inserting or: "; OrI->dump(); dbgs() << "\n");
} else if (OpRC->hasSuperClassEq(&X86::VR128XRegClass) ||
OpRC->hasSuperClassEq(&X86::VR256XRegClass) ||
OpRC->hasSuperClassEq(&X86::VR512RegClass)) {
assert(Subtarget->hasAVX512() && "AVX512-specific register classes!");
bool Is128Bit = OpRC->hasSuperClassEq(&X86::VR128XRegClass);
bool Is256Bit = OpRC->hasSuperClassEq(&X86::VR256XRegClass);
if (Is128Bit || Is256Bit)
assert(Subtarget->hasVLX() && "AVX512VL-specific register classes!");
// Broadcast our state into a vector register.
Register VStateReg = MRI->createVirtualRegister(OpRC);
unsigned BroadcastOp = Is128Bit ? X86::VPBROADCASTQrZ128rr
: Is256Bit ? X86::VPBROADCASTQrZ256rr
: X86::VPBROADCASTQrZrr;
auto BroadcastI =
BuildMI(MBB, InsertPt, Loc, TII->get(BroadcastOp), VStateReg)
.addReg(StateReg);
(void)BroadcastI;
++NumInstsInserted;
LLVM_DEBUG(dbgs() << " Inserting broadcast: "; BroadcastI->dump();
dbgs() << "\n");
// Merge our potential poison state into the value with a vector or.
unsigned OrOp = Is128Bit ? X86::VPORQZ128rr
: Is256Bit ? X86::VPORQZ256rr : X86::VPORQZrr;
auto OrI = BuildMI(MBB, InsertPt, Loc, TII->get(OrOp), TmpReg)
.addReg(VStateReg)
.addReg(OpReg);
(void)OrI;
++NumInstsInserted;
LLVM_DEBUG(dbgs() << " Inserting or: "; OrI->dump(); dbgs() << "\n");
} else {
// FIXME: Need to support GR32 here for 32-bit code.
assert(OpRC->hasSuperClassEq(&X86::GR64RegClass) &&
"Not a supported register class for address hardening!");
if (!EFLAGSLive) {
// Merge our potential poison state into the value with an or.
auto OrI = BuildMI(MBB, InsertPt, Loc, TII->get(X86::OR64rr), TmpReg)
.addReg(StateReg)
.addReg(OpReg);
OrI->addRegisterDead(X86::EFLAGS, TRI);
++NumInstsInserted;
LLVM_DEBUG(dbgs() << " Inserting or: "; OrI->dump(); dbgs() << "\n");
} else {
// We need to avoid touching EFLAGS so shift out all but the least
// significant bit using the instruction that doesn't update flags.
auto ShiftI =
BuildMI(MBB, InsertPt, Loc, TII->get(X86::SHRX64rr), TmpReg)
.addReg(OpReg)
.addReg(StateReg);
(void)ShiftI;
++NumInstsInserted;
LLVM_DEBUG(dbgs() << " Inserting shrx: "; ShiftI->dump();
dbgs() << "\n");
}
}
// Record this register as checked and update the operand.
assert(!AddrRegToHardenedReg.count(Op->getReg()) &&
"Should not have checked this register yet!");
AddrRegToHardenedReg[Op->getReg()] = TmpReg;
Op->setReg(TmpReg);
++NumAddrRegsHardened;
}
// And restore the flags if needed.
if (FlagsReg)
restoreEFLAGS(MBB, InsertPt, Loc, FlagsReg);
}
MachineInstr *X86SpeculativeLoadHardeningPass::sinkPostLoadHardenedInst(
MachineInstr &InitialMI, SmallPtrSetImpl<MachineInstr *> &HardenedInstrs) {
assert(X86InstrInfo::isDataInvariantLoad(InitialMI) &&
"Cannot get here with a non-invariant load!");
assert(!isEFLAGSDefLive(InitialMI) &&
"Cannot get here with a data invariant load "
"that interferes with EFLAGS!");
// See if we can sink hardening the loaded value.
auto SinkCheckToSingleUse =
[&](MachineInstr &MI) -> std::optional<MachineInstr *> {
Register DefReg = MI.getOperand(0).getReg();
// We need to find a single use which we can sink the check. We can
// primarily do this because many uses may already end up checked on their
// own.
MachineInstr *SingleUseMI = nullptr;
for (MachineInstr &UseMI : MRI->use_instructions(DefReg)) {
// If we're already going to harden this use, it is data invariant, it
// does not interfere with EFLAGS, and within our block.
if (HardenedInstrs.count(&UseMI)) {
if (!X86InstrInfo::isDataInvariantLoad(UseMI) || isEFLAGSDefLive(UseMI)) {
// If we've already decided to harden a non-load, we must have sunk
// some other post-load hardened instruction to it and it must itself
// be data-invariant.
assert(X86InstrInfo::isDataInvariant(UseMI) &&
"Data variant instruction being hardened!");
continue;
}
// Otherwise, this is a load and the load component can't be data
// invariant so check how this register is being used.
const MCInstrDesc &Desc = UseMI.getDesc();
int MemRefBeginIdx = X86II::getMemoryOperandNo(Desc.TSFlags);
assert(MemRefBeginIdx >= 0 &&
"Should always have mem references here!");
MemRefBeginIdx += X86II::getOperandBias(Desc);
MachineOperand &BaseMO =
UseMI.getOperand(MemRefBeginIdx + X86::AddrBaseReg);
MachineOperand &IndexMO =
UseMI.getOperand(MemRefBeginIdx + X86::AddrIndexReg);
if ((BaseMO.isReg() && BaseMO.getReg() == DefReg) ||
(IndexMO.isReg() && IndexMO.getReg() == DefReg))
// The load uses the register as part of its address making it not
// invariant.
return {};
continue;
}
if (SingleUseMI)
// We already have a single use, this would make two. Bail.
return {};
// If this single use isn't data invariant, isn't in this block, or has
// interfering EFLAGS, we can't sink the hardening to it.
if (!X86InstrInfo::isDataInvariant(UseMI) || UseMI.getParent() != MI.getParent() ||
isEFLAGSDefLive(UseMI))
return {};
// If this instruction defines multiple registers bail as we won't harden
// all of them.
if (UseMI.getDesc().getNumDefs() > 1)
return {};
// If this register isn't a virtual register we can't walk uses of sanely,
// just bail. Also check that its register class is one of the ones we
// can harden.
Register UseDefReg = UseMI.getOperand(0).getReg();
if (!UseDefReg.isVirtual() || !canHardenRegister(UseDefReg))
return {};
SingleUseMI = &UseMI;
}
// If SingleUseMI is still null, there is no use that needs its own
// checking. Otherwise, it is the single use that needs checking.
return {SingleUseMI};
};
MachineInstr *MI = &InitialMI;
while (std::optional<MachineInstr *> SingleUse = SinkCheckToSingleUse(*MI)) {
// Update which MI we're checking now.
MI = *SingleUse;
if (!MI)
break;
}
return MI;
}
bool X86SpeculativeLoadHardeningPass::canHardenRegister(Register Reg) {
auto *RC = MRI->getRegClass(Reg);
int RegBytes = TRI->getRegSizeInBits(*RC) / 8;
if (RegBytes > 8)
// We don't support post-load hardening of vectors.
return false;
unsigned RegIdx = Log2_32(RegBytes);
assert(RegIdx < 4 && "Unsupported register size");
// If this register class is explicitly constrained to a class that doesn't
// require REX prefix, we may not be able to satisfy that constraint when
// emitting the hardening instructions, so bail out here.
// FIXME: This seems like a pretty lame hack. The way this comes up is when we
// end up both with a NOREX and REX-only register as operands to the hardening
// instructions. It would be better to fix that code to handle this situation
// rather than hack around it in this way.
const TargetRegisterClass *NOREXRegClasses[] = {
&X86::GR8_NOREXRegClass, &X86::GR16_NOREXRegClass,
&X86::GR32_NOREXRegClass, &X86::GR64_NOREXRegClass};
if (RC == NOREXRegClasses[RegIdx])
return false;
const TargetRegisterClass *GPRRegClasses[] = {
&X86::GR8RegClass, &X86::GR16RegClass, &X86::GR32RegClass,
&X86::GR64RegClass};
return RC->hasSuperClassEq(GPRRegClasses[RegIdx]);
}
/// Harden a value in a register.
///
/// This is the low-level logic to fully harden a value sitting in a register
/// against leaking during speculative execution.
///
/// Unlike hardening an address that is used by a load, this routine is required
/// to hide *all* incoming bits in the register.
///
/// `Reg` must be a virtual register. Currently, it is required to be a GPR no
/// larger than the predicate state register. FIXME: We should support vector
/// registers here by broadcasting the predicate state.
///
/// The new, hardened virtual register is returned. It will have the same
/// register class as `Reg`.
unsigned X86SpeculativeLoadHardeningPass::hardenValueInRegister(
Register Reg, MachineBasicBlock &MBB, MachineBasicBlock::iterator InsertPt,
const DebugLoc &Loc) {
assert(canHardenRegister(Reg) && "Cannot harden this register!");
assert(Reg.isVirtual() && "Cannot harden a physical register!");
auto *RC = MRI->getRegClass(Reg);
int Bytes = TRI->getRegSizeInBits(*RC) / 8;
Register StateReg = PS->SSA.GetValueAtEndOfBlock(&MBB);
assert((Bytes == 1 || Bytes == 2 || Bytes == 4 || Bytes == 8) &&
"Unknown register size");
// FIXME: Need to teach this about 32-bit mode.
if (Bytes != 8) {
unsigned SubRegImms[] = {X86::sub_8bit, X86::sub_16bit, X86::sub_32bit};
unsigned SubRegImm = SubRegImms[Log2_32(Bytes)];
Register NarrowStateReg = MRI->createVirtualRegister(RC);
BuildMI(MBB, InsertPt, Loc, TII->get(TargetOpcode::COPY), NarrowStateReg)
.addReg(StateReg, 0, SubRegImm);
StateReg = NarrowStateReg;
}
unsigned FlagsReg = 0;
if (isEFLAGSLive(MBB, InsertPt, *TRI))
FlagsReg = saveEFLAGS(MBB, InsertPt, Loc);
Register NewReg = MRI->createVirtualRegister(RC);
unsigned OrOpCodes[] = {X86::OR8rr, X86::OR16rr, X86::OR32rr, X86::OR64rr};
unsigned OrOpCode = OrOpCodes[Log2_32(Bytes)];
auto OrI = BuildMI(MBB, InsertPt, Loc, TII->get(OrOpCode), NewReg)
.addReg(StateReg)
.addReg(Reg);
OrI->addRegisterDead(X86::EFLAGS, TRI);
++NumInstsInserted;
LLVM_DEBUG(dbgs() << " Inserting or: "; OrI->dump(); dbgs() << "\n");
if (FlagsReg)
restoreEFLAGS(MBB, InsertPt, Loc, FlagsReg);
return NewReg;
}
/// Harden a load by hardening the loaded value in the defined register.
///
/// We can harden a non-leaking load into a register without touching the
/// address by just hiding all of the loaded bits during misspeculation. We use
/// an `or` instruction to do this because we set up our poison value as all
/// ones. And the goal is just for the loaded bits to not be exposed to
/// execution and coercing them to one is sufficient.
///
/// Returns the newly hardened register.
unsigned X86SpeculativeLoadHardeningPass::hardenPostLoad(MachineInstr &MI) {
MachineBasicBlock &MBB = *MI.getParent();
const DebugLoc &Loc = MI.getDebugLoc();
auto &DefOp = MI.getOperand(0);
Register OldDefReg = DefOp.getReg();
auto *DefRC = MRI->getRegClass(OldDefReg);
// Because we want to completely replace the uses of this def'ed value with
// the hardened value, create a dedicated new register that will only be used
// to communicate the unhardened value to the hardening.
Register UnhardenedReg = MRI->createVirtualRegister(DefRC);
DefOp.setReg(UnhardenedReg);
// Now harden this register's value, getting a hardened reg that is safe to
// use. Note that we insert the instructions to compute this *after* the
// defining instruction, not before it.
unsigned HardenedReg = hardenValueInRegister(
UnhardenedReg, MBB, std::next(MI.getIterator()), Loc);
// Finally, replace the old register (which now only has the uses of the
// original def) with the hardened register.
MRI->replaceRegWith(/*FromReg*/ OldDefReg, /*ToReg*/ HardenedReg);
++NumPostLoadRegsHardened;
return HardenedReg;
}
/// Harden a return instruction.
///
/// Returns implicitly perform a load which we need to harden. Without hardening
/// this load, an attacker my speculatively write over the return address to
/// steer speculation of the return to an attacker controlled address. This is
/// called Spectre v1.1 or Bounds Check Bypass Store (BCBS) and is described in
/// this paper:
/// https://people.csail.mit.edu/vlk/spectre11.pdf
///
/// We can harden this by introducing an LFENCE that will delay any load of the
/// return address until prior instructions have retired (and thus are not being
/// speculated), or we can harden the address used by the implicit load: the
/// stack pointer.
///
/// If we are not using an LFENCE, hardening the stack pointer has an additional
/// benefit: it allows us to pass the predicate state accumulated in this
/// function back to the caller. In the absence of a BCBS attack on the return,
/// the caller will typically be resumed and speculatively executed due to the
/// Return Stack Buffer (RSB) prediction which is very accurate and has a high
/// priority. It is possible that some code from the caller will be executed
/// speculatively even during a BCBS-attacked return until the steering takes
/// effect. Whenever this happens, the caller can recover the (poisoned)
/// predicate state from the stack pointer and continue to harden loads.
void X86SpeculativeLoadHardeningPass::hardenReturnInstr(MachineInstr &MI) {
MachineBasicBlock &MBB = *MI.getParent();
const DebugLoc &Loc = MI.getDebugLoc();
auto InsertPt = MI.getIterator();
if (FenceCallAndRet)
// No need to fence here as we'll fence at the return site itself. That
// handles more cases than we can handle here.
return;
// Take our predicate state, shift it to the high 17 bits (so that we keep
// pointers canonical) and merge it into RSP. This will allow the caller to
// extract it when we return (speculatively).
mergePredStateIntoSP(MBB, InsertPt, Loc, PS->SSA.GetValueAtEndOfBlock(&MBB));
}
/// Trace the predicate state through a call.
///
/// There are several layers of this needed to handle the full complexity of
/// calls.
///
/// First, we need to send the predicate state into the called function. We do
/// this by merging it into the high bits of the stack pointer.
///
/// For tail calls, this is all we need to do.
///
/// For calls where we might return and resume the control flow, we need to
/// extract the predicate state from the high bits of the stack pointer after
/// control returns from the called function.
///
/// We also need to verify that we intended to return to this location in the
/// code. An attacker might arrange for the processor to mispredict the return
/// to this valid but incorrect return address in the program rather than the
/// correct one. See the paper on this attack, called "ret2spec" by the
/// researchers, here:
/// https://christian-rossow.de/publications/ret2spec-ccs2018.pdf
///
/// The way we verify that we returned to the correct location is by preserving
/// the expected return address across the call. One technique involves taking
/// advantage of the red-zone to load the return address from `8(%rsp)` where it
/// was left by the RET instruction when it popped `%rsp`. Alternatively, we can
/// directly save the address into a register that will be preserved across the
/// call. We compare this intended return address against the address
/// immediately following the call (the observed return address). If these
/// mismatch, we have detected misspeculation and can poison our predicate
/// state.
void X86SpeculativeLoadHardeningPass::tracePredStateThroughCall(
MachineInstr &MI) {
MachineBasicBlock &MBB = *MI.getParent();
MachineFunction &MF = *MBB.getParent();
auto InsertPt = MI.getIterator();
const DebugLoc &Loc = MI.getDebugLoc();
if (FenceCallAndRet) {
if (MI.isReturn())
// Tail call, we don't return to this function.
// FIXME: We should also handle noreturn calls.
return;
// We don't need to fence before the call because the function should fence
// in its entry. However, we do need to fence after the call returns.
// Fencing before the return doesn't correctly handle cases where the return
// itself is mispredicted.
BuildMI(MBB, std::next(InsertPt), Loc, TII->get(X86::LFENCE));
++NumInstsInserted;
++NumLFENCEsInserted;
return;
}
// First, we transfer the predicate state into the called function by merging
// it into the stack pointer. This will kill the current def of the state.
Register StateReg = PS->SSA.GetValueAtEndOfBlock(&MBB);
mergePredStateIntoSP(MBB, InsertPt, Loc, StateReg);
// If this call is also a return, it is a tail call and we don't need anything
// else to handle it so just return. Also, if there are no further
// instructions and no successors, this call does not return so we can also
// bail.
if (MI.isReturn() || (std::next(InsertPt) == MBB.end() && MBB.succ_empty()))
return;
// Create a symbol to track the return address and attach it to the call
// machine instruction. We will lower extra symbols attached to call
// instructions as label immediately following the call.
MCSymbol *RetSymbol =
MF.getContext().createTempSymbol("slh_ret_addr",
/*AlwaysAddSuffix*/ true);
MI.setPostInstrSymbol(MF, RetSymbol);
const TargetRegisterClass *AddrRC = &X86::GR64RegClass;
unsigned ExpectedRetAddrReg = 0;
// If we have no red zones or if the function returns twice (possibly without
// using the `ret` instruction) like setjmp, we need to save the expected
// return address prior to the call.
if (!Subtarget->getFrameLowering()->has128ByteRedZone(MF) ||
MF.exposesReturnsTwice()) {
// If we don't have red zones, we need to compute the expected return
// address prior to the call and store it in a register that lives across
// the call.
//
// In some ways, this is doubly satisfying as a mitigation because it will
// also successfully detect stack smashing bugs in some cases (typically,
// when a callee-saved register is used and the callee doesn't push it onto
// the stack). But that isn't our primary goal, so we only use it as
// a fallback.
//
// FIXME: It isn't clear that this is reliable in the face of
// rematerialization in the register allocator. We somehow need to force
// that to not occur for this particular instruction, and instead to spill
// or otherwise preserve the value computed *prior* to the call.
//
// FIXME: It is even less clear why MachineCSE can't just fold this when we
// end up having to use identical instructions both before and after the
// call to feed the comparison.
ExpectedRetAddrReg = MRI->createVirtualRegister(AddrRC);
if (MF.getTarget().getCodeModel() == CodeModel::Small &&
!Subtarget->isPositionIndependent()) {
BuildMI(MBB, InsertPt, Loc, TII->get(X86::MOV64ri32), ExpectedRetAddrReg)
.addSym(RetSymbol);
} else {
BuildMI(MBB, InsertPt, Loc, TII->get(X86::LEA64r), ExpectedRetAddrReg)
.addReg(/*Base*/ X86::RIP)
.addImm(/*Scale*/ 1)
.addReg(/*Index*/ 0)
.addSym(RetSymbol)
.addReg(/*Segment*/ 0);
}
}
// Step past the call to handle when it returns.
++InsertPt;
// If we didn't pre-compute the expected return address into a register, then
// red zones are enabled and the return address is still available on the
// stack immediately after the call. As the very first instruction, we load it
// into a register.
if (!ExpectedRetAddrReg) {
ExpectedRetAddrReg = MRI->createVirtualRegister(AddrRC);
BuildMI(MBB, InsertPt, Loc, TII->get(X86::MOV64rm), ExpectedRetAddrReg)
.addReg(/*Base*/ X86::RSP)
.addImm(/*Scale*/ 1)
.addReg(/*Index*/ 0)
.addImm(/*Displacement*/ -8) // The stack pointer has been popped, so
// the return address is 8-bytes past it.
.addReg(/*Segment*/ 0);
}
// Now we extract the callee's predicate state from the stack pointer.
unsigned NewStateReg = extractPredStateFromSP(MBB, InsertPt, Loc);
// Test the expected return address against our actual address. If we can
// form this basic block's address as an immediate, this is easy. Otherwise
// we compute it.
if (MF.getTarget().getCodeModel() == CodeModel::Small &&
!Subtarget->isPositionIndependent()) {
// FIXME: Could we fold this with the load? It would require careful EFLAGS
// management.
BuildMI(MBB, InsertPt, Loc, TII->get(X86::CMP64ri32))
.addReg(ExpectedRetAddrReg, RegState::Kill)
.addSym(RetSymbol);
} else {
Register ActualRetAddrReg = MRI->createVirtualRegister(AddrRC);
BuildMI(MBB, InsertPt, Loc, TII->get(X86::LEA64r), ActualRetAddrReg)
.addReg(/*Base*/ X86::RIP)
.addImm(/*Scale*/ 1)
.addReg(/*Index*/ 0)
.addSym(RetSymbol)
.addReg(/*Segment*/ 0);
BuildMI(MBB, InsertPt, Loc, TII->get(X86::CMP64rr))
.addReg(ExpectedRetAddrReg, RegState::Kill)
.addReg(ActualRetAddrReg, RegState::Kill);
}
// Now conditionally update the predicate state we just extracted if we ended
// up at a different return address than expected.
int PredStateSizeInBytes = TRI->getRegSizeInBits(*PS->RC) / 8;
auto CMovOp = X86::getCMovOpcode(PredStateSizeInBytes);
Register UpdatedStateReg = MRI->createVirtualRegister(PS->RC);
auto CMovI = BuildMI(MBB, InsertPt, Loc, TII->get(CMovOp), UpdatedStateReg)
.addReg(NewStateReg, RegState::Kill)
.addReg(PS->PoisonReg)
.addImm(X86::COND_NE);
CMovI->findRegisterUseOperand(X86::EFLAGS)->setIsKill(true);
++NumInstsInserted;
LLVM_DEBUG(dbgs() << " Inserting cmov: "; CMovI->dump(); dbgs() << "\n");
PS->SSA.AddAvailableValue(&MBB, UpdatedStateReg);
}
/// An attacker may speculatively store over a value that is then speculatively
/// loaded and used as the target of an indirect call or jump instruction. This
/// is called Spectre v1.2 or Bounds Check Bypass Store (BCBS) and is described
/// in this paper:
/// https://people.csail.mit.edu/vlk/spectre11.pdf
///
/// When this happens, the speculative execution of the call or jump will end up
/// being steered to this attacker controlled address. While most such loads
/// will be adequately hardened already, we want to ensure that they are
/// definitively treated as needing post-load hardening. While address hardening
/// is sufficient to prevent secret data from leaking to the attacker, it may
/// not be sufficient to prevent an attacker from steering speculative
/// execution. We forcibly unfolded all relevant loads above and so will always
/// have an opportunity to post-load harden here, we just need to scan for cases
/// not already flagged and add them.
void X86SpeculativeLoadHardeningPass::hardenIndirectCallOrJumpInstr(
MachineInstr &MI,
SmallDenseMap<unsigned, unsigned, 32> &AddrRegToHardenedReg) {
switch (MI.getOpcode()) {
case X86::FARCALL16m:
case X86::FARCALL32m:
case X86::FARCALL64m:
case X86::FARJMP16m:
case X86::FARJMP32m:
case X86::FARJMP64m:
// We don't need to harden either far calls or far jumps as they are
// safe from Spectre.
return;
default:
break;
}
// We should never see a loading instruction at this point, as those should
// have been unfolded.
assert(!MI.mayLoad() && "Found a lingering loading instruction!");
// If the first operand isn't a register, this is a branch or call
// instruction with an immediate operand which doesn't need to be hardened.
if (!MI.getOperand(0).isReg())
return;
// For all of these, the target register is the first operand of the
// instruction.
auto &TargetOp = MI.getOperand(0);
Register OldTargetReg = TargetOp.getReg();
// Try to lookup a hardened version of this register. We retain a reference
// here as we want to update the map to track any newly computed hardened
// register.
unsigned &HardenedTargetReg = AddrRegToHardenedReg[OldTargetReg];
// If we don't have a hardened register yet, compute one. Otherwise, just use
// the already hardened register.
//
// FIXME: It is a little suspect that we use partially hardened registers that
// only feed addresses. The complexity of partial hardening with SHRX
// continues to pile up. Should definitively measure its value and consider
// eliminating it.
if (!HardenedTargetReg)
HardenedTargetReg = hardenValueInRegister(
OldTargetReg, *MI.getParent(), MI.getIterator(), MI.getDebugLoc());
// Set the target operand to the hardened register.
TargetOp.setReg(HardenedTargetReg);
++NumCallsOrJumpsHardened;
}
INITIALIZE_PASS_BEGIN(X86SpeculativeLoadHardeningPass, PASS_KEY,
"X86 speculative load hardener", false, false)
INITIALIZE_PASS_END(X86SpeculativeLoadHardeningPass, PASS_KEY,
"X86 speculative load hardener", false, false)
FunctionPass *llvm::createX86SpeculativeLoadHardeningPass() {
return new X86SpeculativeLoadHardeningPass();
}