third_party/llvm-16.0/llvm/lib/Target/ARM/ARMFixCortexA57AES1742098Pass.cpp

//===-- ARMFixCortexA57AES1742098Pass.cpp ---------------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
// This pass works around a Cortex Core Fused AES erratum:
// - Cortex-A57 Erratum 1742098
// - Cortex-A72 Erratum 1655431
//
// The erratum may be triggered if an input vector register to AESE or AESD was
// last written by an instruction that only updated 32 bits of it. This can
// occur for either of the input registers.
//
// The workaround chosen is to update the input register using `r = VORRq r, r`,
// as this updates all 128 bits of the register unconditionally, but does not
// change the values observed in `r`, making the input safe.
//
// This pass has to be conservative in a few cases:
// - an input vector register to the AES instruction is defined outside the
//   current function, where we have to assume the register was updated in an
//   unsafe way; and
// - an input vector register to the AES instruction is updated along multiple
//   different control-flow paths, where we have to ensure all the register
//   updating instructions are safe.
//
// Both of these cases may apply to a input vector register. In either case, we
// need to ensure that, when the pass is finished, there exists a safe
// instruction between every unsafe register updating instruction and the AES
// instruction.
//
//===----------------------------------------------------------------------===//

#include "ARM.h"
#include "ARMBaseInstrInfo.h"
#include "ARMBaseRegisterInfo.h"
#include "ARMSubtarget.h"
#include "Utils/ARMBaseInfo.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/SmallPtrSet.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/StringRef.h"
#include "llvm/CodeGen/MachineBasicBlock.h"
#include "llvm/CodeGen/MachineFunction.h"
#include "llvm/CodeGen/MachineFunctionPass.h"
#include "llvm/CodeGen/MachineInstr.h"
#include "llvm/CodeGen/MachineInstrBuilder.h"
#include "llvm/CodeGen/MachineInstrBundleIterator.h"
#include "llvm/CodeGen/MachineOperand.h"
#include "llvm/CodeGen/ReachingDefAnalysis.h"
#include "llvm/CodeGen/Register.h"
#include "llvm/CodeGen/TargetRegisterInfo.h"
#include "llvm/IR/DebugLoc.h"
#include "llvm/InitializePasses.h"
#include "llvm/MC/MCInstrDesc.h"
#include "llvm/Pass.h"
#include "llvm/PassRegistry.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/raw_ostream.h"
#include <assert.h>
#include <stdint.h>

using namespace llvm;

#define DEBUG_TYPE "arm-fix-cortex-a57-aes-1742098"

//===----------------------------------------------------------------------===//

namespace {
class ARMFixCortexA57AES1742098 : public MachineFunctionPass {
public:
  static char ID;
  explicit ARMFixCortexA57AES1742098() : MachineFunctionPass(ID) {
    initializeARMFixCortexA57AES1742098Pass(*PassRegistry::getPassRegistry());
  }

  bool runOnMachineFunction(MachineFunction &F) override;

  MachineFunctionProperties getRequiredProperties() const override {
    return MachineFunctionProperties().set(
        MachineFunctionProperties::Property::NoVRegs);
  }

  StringRef getPassName() const override {
    return "ARM fix for Cortex-A57 AES Erratum 1742098";
  }

  void getAnalysisUsage(AnalysisUsage &AU) const override {
    AU.addRequired<ReachingDefAnalysis>();
    AU.setPreservesCFG();
    MachineFunctionPass::getAnalysisUsage(AU);
  }

private:
  // This is the information needed to insert the fixup in the right place.
  struct AESFixupLocation {
    MachineBasicBlock *Block;
    // The fixup instruction will be inserted *before* InsertionPt.
    MachineInstr *InsertionPt;
    MachineOperand *MOp;
  };

  void analyzeMF(MachineFunction &MF, ReachingDefAnalysis &RDA,
                 const ARMBaseRegisterInfo *TRI,
                 SmallVectorImpl<AESFixupLocation> &FixupLocsForFn) const;

  void insertAESFixup(AESFixupLocation &FixupLoc, const ARMBaseInstrInfo *TII,
                      const ARMBaseRegisterInfo *TRI) const;

  static bool isFirstAESPairInstr(MachineInstr &MI);
  static bool isSafeAESInput(MachineInstr &MI);
};
char ARMFixCortexA57AES1742098::ID = 0;

} // end anonymous namespace

INITIALIZE_PASS_BEGIN(ARMFixCortexA57AES1742098, DEBUG_TYPE,
                      "ARM fix for Cortex-A57 AES Erratum 1742098", false,
                      false)
INITIALIZE_PASS_DEPENDENCY(ReachingDefAnalysis);
INITIALIZE_PASS_END(ARMFixCortexA57AES1742098, DEBUG_TYPE,
                    "ARM fix for Cortex-A57 AES Erratum 1742098", false, false)

//===----------------------------------------------------------------------===//

bool ARMFixCortexA57AES1742098::isFirstAESPairInstr(MachineInstr &MI) {
  unsigned Opc = MI.getOpcode();
  return Opc == ARM::AESD || Opc == ARM::AESE;
}

bool ARMFixCortexA57AES1742098::isSafeAESInput(MachineInstr &MI) {
  auto CondCodeIsAL = [](MachineInstr &MI) -> bool {
    int CCIdx = MI.findFirstPredOperandIdx();
    if (CCIdx == -1)
      return false;
    return MI.getOperand(CCIdx).getImm() == (int64_t)ARMCC::AL;
  };

  switch (MI.getOpcode()) {
  // Unknown: Assume not safe.
  default:
    return false;
  // 128-bit wide AES instructions
  case ARM::AESD:
  case ARM::AESE:
  case ARM::AESMC:
  case ARM::AESIMC:
    // No CondCode.
    return true;
  // 128-bit and 64-bit wide bitwise ops (when condition = al)
  case ARM::VANDd:
  case ARM::VANDq:
  case ARM::VORRd:
  case ARM::VORRq:
  case ARM::VEORd:
  case ARM::VEORq:
  case ARM::VMVNd:
  case ARM::VMVNq:
  // VMOV of 64-bit value between D registers (when condition = al)
  case ARM::VMOVD:
  // VMOV of 64 bit value from GPRs (when condition = al)
  case ARM::VMOVDRR:
  // VMOV of immediate into D or Q registers (when condition = al)
  case ARM::VMOVv2i64:
  case ARM::VMOVv1i64:
  case ARM::VMOVv2f32:
  case ARM::VMOVv4f32:
  case ARM::VMOVv2i32:
  case ARM::VMOVv4i32:
  case ARM::VMOVv4i16:
  case ARM::VMOVv8i16:
  case ARM::VMOVv8i8:
  case ARM::VMOVv16i8:
  // Loads (when condition = al)
  // VLD Dn, [Rn, #imm]
  case ARM::VLDRD:
  // VLDM
  case ARM::VLDMDDB_UPD:
  case ARM::VLDMDIA_UPD:
  case ARM::VLDMDIA:
  // VLDn to all lanes.
  case ARM::VLD1d64:
  case ARM::VLD1q64:
  case ARM::VLD1d32:
  case ARM::VLD1q32:
  case ARM::VLD2b32:
  case ARM::VLD2d32:
  case ARM::VLD2q32:
  case ARM::VLD1d16:
  case ARM::VLD1q16:
  case ARM::VLD2d16:
  case ARM::VLD2q16:
  case ARM::VLD1d8:
  case ARM::VLD1q8:
  case ARM::VLD2b8:
  case ARM::VLD2d8:
  case ARM::VLD2q8:
  case ARM::VLD3d32:
  case ARM::VLD3q32:
  case ARM::VLD3d16:
  case ARM::VLD3q16:
  case ARM::VLD3d8:
  case ARM::VLD3q8:
  case ARM::VLD4d32:
  case ARM::VLD4q32:
  case ARM::VLD4d16:
  case ARM::VLD4q16:
  case ARM::VLD4d8:
  case ARM::VLD4q8:
  // VLD1 (single element to one lane)
  case ARM::VLD1LNd32:
  case ARM::VLD1LNd32_UPD:
  case ARM::VLD1LNd8:
  case ARM::VLD1LNd8_UPD:
  case ARM::VLD1LNd16:
  case ARM::VLD1LNd16_UPD:
  // VLD1 (single element to all lanes)
  case ARM::VLD1DUPd32:
  case ARM::VLD1DUPd32wb_fixed:
  case ARM::VLD1DUPd32wb_register:
  case ARM::VLD1DUPd16:
  case ARM::VLD1DUPd16wb_fixed:
  case ARM::VLD1DUPd16wb_register:
  case ARM::VLD1DUPd8:
  case ARM::VLD1DUPd8wb_fixed:
  case ARM::VLD1DUPd8wb_register:
  case ARM::VLD1DUPq32:
  case ARM::VLD1DUPq32wb_fixed:
  case ARM::VLD1DUPq32wb_register:
  case ARM::VLD1DUPq16:
  case ARM::VLD1DUPq16wb_fixed:
  case ARM::VLD1DUPq16wb_register:
  case ARM::VLD1DUPq8:
  case ARM::VLD1DUPq8wb_fixed:
  case ARM::VLD1DUPq8wb_register:
  // VMOV
  case ARM::VSETLNi32:
  case ARM::VSETLNi16:
  case ARM::VSETLNi8:
    return CondCodeIsAL(MI);
  };

  return false;
}

bool ARMFixCortexA57AES1742098::runOnMachineFunction(MachineFunction &F) {
  LLVM_DEBUG(dbgs() << "***** ARMFixCortexA57AES1742098 *****\n");
  auto &STI = F.getSubtarget<ARMSubtarget>();

  // Fix not requested or AES instructions not present: skip pass.
  if (!STI.hasAES() || !STI.fixCortexA57AES1742098())
    return false;

  const ARMBaseRegisterInfo *TRI = STI.getRegisterInfo();
  const ARMBaseInstrInfo *TII = STI.getInstrInfo();

  auto &RDA = getAnalysis<ReachingDefAnalysis>();

  // Analyze whole function to find instructions which need fixing up...
  SmallVector<AESFixupLocation> FixupLocsForFn{};
  analyzeMF(F, RDA, TRI, FixupLocsForFn);

  // ... and fix the instructions up all at the same time.
  bool Changed = false;
  LLVM_DEBUG(dbgs() << "Inserting " << FixupLocsForFn.size() << " fixup(s)\n");
  for (AESFixupLocation &FixupLoc : FixupLocsForFn) {
    insertAESFixup(FixupLoc, TII, TRI);
    Changed |= true;
  }

  return Changed;
}

void ARMFixCortexA57AES1742098::analyzeMF(
    MachineFunction &MF, ReachingDefAnalysis &RDA,
    const ARMBaseRegisterInfo *TRI,
    SmallVectorImpl<AESFixupLocation> &FixupLocsForFn) const {
  unsigned MaxAllowedFixups = 0;

  for (MachineBasicBlock &MBB : MF) {
    for (MachineInstr &MI : MBB) {
      if (!isFirstAESPairInstr(MI))
        continue;

      // Found an instruction to check the operands of.
      LLVM_DEBUG(dbgs() << "Found AES Pair starting: " << MI);
      assert(MI.getNumExplicitOperands() == 3 && MI.getNumExplicitDefs() == 1 &&
             "Unknown AES Instruction Format. Expected 1 def, 2 uses.");

      // A maximum of two fixups should be inserted for each AES pair (one per
      // register use).
      MaxAllowedFixups += 2;

      // Inspect all operands, choosing whether to insert a fixup.
      for (MachineOperand &MOp : MI.uses()) {
        SmallPtrSet<MachineInstr *, 1> AllDefs{};
        RDA.getGlobalReachingDefs(&MI, MOp.getReg(), AllDefs);

        // Planned Fixup: This should be added to FixupLocsForFn at most once.
        AESFixupLocation NewLoc{&MBB, &MI, &MOp};

        // In small functions with loops, this operand may be both a live-in and
        // have definitions within the function itself. These will need a fixup.
        bool IsLiveIn = MF.front().isLiveIn(MOp.getReg());

        // If the register doesn't have defining instructions, and is not a
        // live-in, then something is wrong and the fixup must always be
        // inserted to be safe.
        if (!IsLiveIn && AllDefs.size() == 0) {
          LLVM_DEBUG(dbgs()
                     << "Fixup Planned: No Defining Instrs found, not live-in: "
                     << printReg(MOp.getReg(), TRI) << "\n");
          FixupLocsForFn.emplace_back(NewLoc);
          continue;
        }

        auto IsUnsafe = [](MachineInstr *MI) -> bool {
          return !isSafeAESInput(*MI);
        };
        size_t UnsafeCount = llvm::count_if(AllDefs, IsUnsafe);

        // If there are no unsafe definitions...
        if (UnsafeCount == 0) {
          // ... and the register is not live-in ...
          if (!IsLiveIn) {
            // ... then skip the fixup.
            LLVM_DEBUG(dbgs() << "No Fixup: Defining instrs are all safe: "
                              << printReg(MOp.getReg(), TRI) << "\n");
            continue;
          }

          // Otherwise, the only unsafe "definition" is a live-in, so insert the
          // fixup at the start of the function.
          LLVM_DEBUG(dbgs()
                     << "Fixup Planned: Live-In (with safe defining instrs): "
                     << printReg(MOp.getReg(), TRI) << "\n");
          NewLoc.Block = &MF.front();
          NewLoc.InsertionPt = &*NewLoc.Block->begin();
          LLVM_DEBUG(dbgs() << "Moving Fixup for Live-In to immediately before "
                            << *NewLoc.InsertionPt);
          FixupLocsForFn.emplace_back(NewLoc);
          continue;
        }

        // If a fixup is needed in more than one place, then the best place to
        // insert it is adjacent to the use rather than introducing a fixup
        // adjacent to each def.
        //
        // FIXME: It might be better to hoist this to the start of the BB, if
        // possible.
        if (IsLiveIn || UnsafeCount > 1) {
          LLVM_DEBUG(dbgs() << "Fixup Planned: Multiple unsafe defining instrs "
                               "(including live-ins): "
                            << printReg(MOp.getReg(), TRI) << "\n");
          FixupLocsForFn.emplace_back(NewLoc);
          continue;
        }

        assert(UnsafeCount == 1 && !IsLiveIn &&
               "At this point, there should be one unsafe defining instrs "
               "and the defined register should not be a live-in.");
        SmallPtrSetIterator<MachineInstr *> It =
            llvm::find_if(AllDefs, IsUnsafe);
        assert(It != AllDefs.end() &&
               "UnsafeCount == 1 but No Unsafe MachineInstr found.");
        MachineInstr *DefMI = *It;

        LLVM_DEBUG(
            dbgs() << "Fixup Planned: Found single unsafe defining instrs for "
                   << printReg(MOp.getReg(), TRI) << ": " << *DefMI);

        // There is one unsafe defining instruction, which needs a fixup. It is
        // generally good to hoist the fixup to be adjacent to the defining
        // instruction rather than the using instruction, as the using
        // instruction may be inside a loop when the defining instruction is
        // not.
        MachineBasicBlock::iterator DefIt = DefMI;
        ++DefIt;
        if (DefIt != DefMI->getParent()->end()) {
          LLVM_DEBUG(dbgs() << "Moving Fixup to immediately after " << *DefMI
                            << "And immediately before " << *DefIt);
          NewLoc.Block = DefIt->getParent();
          NewLoc.InsertionPt = &*DefIt;
        }

        FixupLocsForFn.emplace_back(NewLoc);
      }
    }
  }

  assert(FixupLocsForFn.size() <= MaxAllowedFixups &&
         "Inserted too many fixups for this function.");
  (void)MaxAllowedFixups;
}

void ARMFixCortexA57AES1742098::insertAESFixup(
    AESFixupLocation &FixupLoc, const ARMBaseInstrInfo *TII,
    const ARMBaseRegisterInfo *TRI) const {
  MachineOperand *OperandToFixup = FixupLoc.MOp;

  assert(OperandToFixup->isReg() && "OperandToFixup must be a register");
  Register RegToFixup = OperandToFixup->getReg();

  LLVM_DEBUG(dbgs() << "Inserting VORRq of " << printReg(RegToFixup, TRI)
                    << " before: " << *FixupLoc.InsertionPt);

  // Insert the new `VORRq qN, qN, qN`. There are a few details here:
  //
  // The uses are marked as killed, even if the original use of OperandToFixup
  // is not killed, as the new instruction is clobbering the register. This is
  // safe even if there are other uses of `qN`, as the VORRq value-wise a no-op
  // (it is inserted for microarchitectural reasons).
  //
  // The def and the uses are still marked as Renamable if the original register
  // was, to avoid having to rummage through all the other uses and defs and
  // unset their renamable bits.
  unsigned Renamable = OperandToFixup->isRenamable() ? RegState::Renamable : 0;
  BuildMI(*FixupLoc.Block, FixupLoc.InsertionPt, DebugLoc(),
          TII->get(ARM::VORRq))
      .addReg(RegToFixup, RegState::Define | Renamable)
      .addReg(RegToFixup, RegState::Kill | Renamable)
      .addReg(RegToFixup, RegState::Kill | Renamable)
      .addImm((uint64_t)ARMCC::AL)
      .addReg(ARM::NoRegister);
}

// Factory function used by AArch64TargetMachine to add the pass to
// the passmanager.
FunctionPass *llvm::createARMFixCortexA57AES1742098Pass() {
  return new ARMFixCortexA57AES1742098();
}